Merge master into feat/677-aggregates

# Conflicts: # docs/Historian.md
2026-10-11 11:30:46 -04:00
parent 27ca6cd791 8e6ac4b344
commit 9120182314
66 changed files with 2054 additions and 154 deletions
@@ -125,6 +125,21 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
        }
    }

+    /// <inheritdoc />
+    /// <remarks>
+    ///     A reverse (newest-first) read is refused with <see cref="NotSupportedException"/> — the server reports
+    ///     <c>BadHistoryOperationUnsupported</c> (Gitea #675). The gateway's <c>ReadEvents</c> streams ascending only and
+    ///     rejects <c>end &lt; start</c>, so the only way to serve "the latest N" would be to drain the whole window and
+    ///     keep its tail, an unbounded read the cap exists to prevent. Nothing is sent.
+    /// </remarks>
+    public Task<HistoricalEventsResult> ReadEventsAsync(
+        string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse,
+        CancellationToken cancellationToken) =>
+        reverse
+            ? throw new NotSupportedException(
+                "The HistorianGateway backend reads events oldest-first only; a reverse (newest-first) events read is not supported.")
+            : ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken);
+
    /// <inheritdoc />
    public async Task<HistoricalEventsResult> ReadEventsAsync(
        string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
@@ -361,8 +361,18 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
    ///     <c>maxEvents + 1</c> rows are fetched; when the extra row arrives it is dropped and the continuation point is
    ///     the UTF-8 of <see cref="EventContinuation.Encode"/> for that first not-returned event.
    /// </remarks>
+    public Task<HistoricalEventsResult> ReadEventsAsync(
+        string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken) =>
+        ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, reverse: false, cancellationToken);
+
+    /// <inheritdoc />
+    /// <remarks>
+    ///     Forward is the five-argument read. Reverse (Gitea #675) is the (rounded) <c>(lo, hi]</c> ordered by
+    ///     <c>(ts, event_id)</c> DESCENDING, so <c>LIMIT</c> keeps the latest events; truncation and the continuation point
+    ///     work as forward.
+    /// </remarks>
    public async Task<HistoricalEventsResult> ReadEventsAsync(
-        string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken)
+        string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse, CancellationToken cancellationToken)
    {
        // No stored event can carry a source longer than the limit, so an over-long filter matches nothing.
        if (sourceName is not null && sourceName.Length > HistorianTagNames.MaxLength)
@@ -376,9 +386,9 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
            var bounded = maxEvents > 0 && maxEvents < int.MaxValue;
            var sql =
                $"SELECT event_id, source, ts, received_utc, message, severity FROM {_conn.EventTable} " +
-                "WHERE ts >= $1 AND ts < $2" +
+                (reverse ? "WHERE ts > $1 AND ts <= $2" : "WHERE ts >= $1 AND ts < $2") +
                (sourceName is null ? "" : " AND source = $3") +
-                " ORDER BY ts ASC, event_id ASC" +
+                (reverse ? " ORDER BY ts DESC, event_id DESC" : " ORDER BY ts ASC, event_id ASC") +
                (bounded ? (sourceName is null ? " LIMIT $3" : " LIMIT $4") : "");

            await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
@@ -137,7 +137,10 @@ internal sealed class QuestDbSchemaBootstrapper
            // The ALTER is a WAL transaction: tables() keeps the old value until it applies, so do NOT re-read to verify.
            await using (var alter = _conn.CreateCommand(conn, QuestDbTables.AlterTtl(table, _options.RetentionDays)))
                await alter.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
-            _log.LogInformation("Applied TTL {Days} day(s) to QuestDB table {Table}.", _options.RetentionDays, table);
+            if (_options.RetentionDays is { } days)
+                _log.LogInformation("Applied TTL {Days} day(s) to QuestDB table {Table}.", days, table);
+            else
+                _log.LogInformation("Cleared TTL on QuestDB table {Table}.", table);
        }
    }
 }
@@ -27,4 +27,10 @@ public static class HistorianStatusCodes
    ///     status, or a tag name longer than <see cref="HistorianTagNames.MaxLength"/>.
    /// </summary>
    public const uint BadInvalidArgument = 0x80AB0000u;
+
+    /// <summary>
+    ///     A modified-value read or HistoryUpdate item against a SQL Server schema still at version 1 (no Part 11
+    ///     modification trail, Gitea #674): refused without touching the database.
+    /// </summary>
+    public const uint BadHistoryOperationUnsupported = 0x80720000u;
 }
@@ -154,6 +154,12 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
        var bounded = maxValues > 0 && maxValues < int.MaxValue;
        try
        {
+            // #674: a v1 schema has no SampleModified. NotSupportedException is the interface's "no trail" signal (the
+            // node manager reports BadHistoryOperationUnsupported); it is not a backend failure and sends no query.
+            await _bootstrap.EnsureAsync(cancellationToken).ConfigureAwait(false);
+            if (!_bootstrap.SupportsModifiedTrail)
+                throw new NotSupportedException(
+                    $"Historian schema [{_conn.Schema}] is below version {SqlServerSchemaBootstrapper.ModifiedTrailVersion}: no modified-value history.");
            var values = new List<HistoryModifiedValue>();
            var hasMore = false;
            var tagId = await ResolveAsync(fullReference, cancellationToken).ConfigureAwait(false);
@@ -219,7 +225,7 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
            RecordSuccess();
            return new HistoryModifiedReadResult(values, hasMore);
        }
-        catch (Exception ex)
+        catch (Exception ex) when (ex is not NotSupportedException)
        {
            RecordFailure(ex);
            throw;
@@ -314,8 +320,18 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
    ///     of <c>"{EventTimeUtc.Ticks}|{EventId}"</c> of that first not-returned event (opaque to callers; only
    ///     its presence is contractual).
    /// </remarks>
+    public Task<HistoricalEventsResult> ReadEventsAsync(
+        string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken) =>
+        ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, reverse: false, cancellationToken);
+
+    /// <inheritdoc />
+    /// <remarks>
+    ///     Forward is the five-argument read. Reverse (Gitea #675) is <c>(lo, hi]</c> ordered by
+    ///     <c>(EventTimeUtc, EventId)</c> DESCENDING — the clustered index scanned backwards — so <c>TOP</c> keeps the latest
+    ///     events; truncation and the continuation point work as forward.
+    /// </remarks>
    public async Task<HistoricalEventsResult> ReadEventsAsync(
-        string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken)
+        string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse, CancellationToken cancellationToken)
    {
        // No stored event can carry a source longer than the column (the alarm writer dead-letters it rather than
        // truncating), so an over-long filter matches nothing — never a truncated prefix of someone else's source.
@@ -328,9 +344,10 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
            var bounded = maxEvents > 0 && maxEvents < int.MaxValue;
            var sql =
                $"SELECT {(bounded ? "TOP (@take) " : "")}EventId, SourceName, EventTimeUtc, ReceivedTimeUtc, Message, Severity " +
-                $"FROM {_conn.Q("Event")} WHERE EventTimeUtc >= @lo AND EventTimeUtc < @hi" +
+                $"FROM {_conn.Q("Event")} WHERE " +
+                (reverse ? "EventTimeUtc > @lo AND EventTimeUtc <= @hi" : "EventTimeUtc >= @lo AND EventTimeUtc < @hi") +
                (sourceName is null ? "" : " AND SourceName = @source") +
-                " ORDER BY EventTimeUtc ASC, EventId ASC";
+                (reverse ? " ORDER BY EventTimeUtc DESC, EventId DESC" : " ORDER BY EventTimeUtc ASC, EventId ASC");

            await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
            await using var cmd = _conn.CreateCommand(conn, sql);
@@ -27,21 +27,44 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
 ///     value from <c>deleted.*</c> (NULL for an insert), the caller's <see cref="HistoryUpdateContext"/>, the
 ///     <see cref="HistoryModificationType"/> and <c>SYSUTCDATETIME()</c>. One statement means the sample change and its
 ///     audit row commit or roll back together: a failed update leaves no audit row.</para>
+///     <para><b>Schema v1 (Gitea #674).</b> Without the trail (<see cref="SqlServerSchemaBootstrapper.SupportsModifiedTrail"/>
+///     false) a change could not be audited, so every item (and DeleteRaw's single result) reports
+///     <c>BadHistoryOperationUnsupported</c> and nothing is sent to the database beyond the one-time schema check.</para>
 /// </summary>
 internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
 {
    private readonly SqlHistorianConnection _conn;
    private readonly SqlServerTagCatalog _tags;
+    private readonly SqlServerSchemaBootstrapper _bootstrap;

    /// <summary>Creates the updater.</summary>
    /// <param name="conn">Connection helper.</param>
+    /// <param name="bootstrap">Schema bootstrapper — decides whether the modification trail exists (#674).</param>
    /// <param name="tags">Tag catalog (runs the schema bootstrap itself).</param>
-    public SqlServerHistorianDataUpdater(SqlHistorianConnection conn, SqlServerTagCatalog tags)
+    public SqlServerHistorianDataUpdater(SqlHistorianConnection conn, SqlServerSchemaBootstrapper bootstrap,
+        SqlServerTagCatalog tags)
    {
        _conn = conn;
+        _bootstrap = bootstrap;
        _tags = tags;
    }

+    /// <summary>#674: ensures the schema and reports whether the Part 11 trail exists (no SQL after the first success).</summary>
+    /// <param name="ct">Cancellation.</param>
+    /// <returns>True when the modification trail is available.</returns>
+    private async Task<bool> TrailAvailableAsync(CancellationToken ct)
+    {
+        await _bootstrap.EnsureAsync(ct).ConfigureAwait(false);
+        return _bootstrap.SupportsModifiedTrail;
+    }
+
+    private static uint[] Unsupported(int count)
+    {
+        var codes = new uint[count];
+        Array.Fill(codes, HistorianStatusCodes.BadHistoryOperationUnsupported);
+        return codes;
+    }
+
    /// <inheritdoc/>
    public async Task<IReadOnlyList<uint>> UpdateDataAsync(string tag, HistoryUpdateMode mode,
        IReadOnlyList<HistorizationValue> values, HistoryUpdateContext context, CancellationToken ct)
@@ -49,6 +72,7 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
        ArgumentNullException.ThrowIfNull(tag);
        ArgumentNullException.ThrowIfNull(values);
        ArgumentNullException.ThrowIfNull(context);
+        if (!await TrailAvailableAsync(ct).ConfigureAwait(false)) return Unsupported(values.Count);
        var plan = HistoryUpdatePlanner.Plan(tag, mode, values);
        if (plan.Winners.Count == 0) return HistoryUpdatePlanner.Complete(plan, mode, new Dictionary<long, UpsertOutcome>());

@@ -68,6 +92,7 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
    {
        ArgumentNullException.ThrowIfNull(tag);
        ArgumentNullException.ThrowIfNull(context);
+        if (!await TrailAvailableAsync(ct).ConfigureAwait(false)) return HistorianStatusCodes.BadHistoryOperationUnsupported;
        if (!HistorianTagNames.IsValid(tag)) return HistorianStatusCodes.BadInvalidArgument;
        var tagId = await _tags.TryGetTagIdAsync(tag, ct).ConfigureAwait(false);
        if (tagId is null) return HistoryUpdatePlanner.DeleteRawResult(tagKnown: false, 0);
@@ -95,6 +120,7 @@ internal sealed class SqlServerHistorianDataUpdater : IHistorianDataUpdater
        ArgumentNullException.ThrowIfNull(tag);
        ArgumentNullException.ThrowIfNull(timesUtc);
        ArgumentNullException.ThrowIfNull(context);
+        if (!await TrailAvailableAsync(ct).ConfigureAwait(false)) return Unsupported(timesUtc.Count);
        if (!HistorianTagNames.IsValid(tag))
        {
            var refused = new uint[timesUtc.Count];
@@ -18,8 +18,10 @@ public sealed partial class SqlServerHistorianOptions

    /// <summary>
    ///     <c>true</c> creates the schema and applies missing scripts on first use; <c>false</c> only verifies
-    ///     that the required schema version (<c>SqlServerSchemaBootstrapper.RequiredVersion</c>, currently 2) exists and
-    ///     fails every historian call with a clear error otherwise.
+    ///     that the base schema version (<c>SqlServerSchemaBootstrapper.RequiredVersion</c>, 1) exists and fails every
+    ///     historian call with a clear error otherwise. A verified v1 schema (no Part 11 modification trail, script 002
+    ///     not applied) is served with modified reads and HistoryUpdate refused <c>BadHistoryOperationUnsupported</c>
+    ///     and one Warning per boot (Gitea #674).
    /// </summary>
    public bool AutoCreateSchema { get; set; } = true;

@@ -99,7 +99,7 @@ public sealed class SqlServerHistorianProvider : IHistorianBackendProvider
    public IHistorianDataUpdater? CreateUpdater(IServiceProvider sp)
    {
        var s = Get(sp);
-        return new SqlServerHistorianDataUpdater(s.Conn, s.Catalog);
+        return new SqlServerHistorianDataUpdater(s.Conn, s.Bootstrap, s.Catalog);
    }

    /// <inheritdoc />
@@ -11,7 +11,8 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
 ///     <c>DELETE TOP(n)</c> batches (under the lock-escalation threshold). Samples are purged per tag so each
 ///     delete is a seek on the clustered key <c>(TagId, SourceTimeUtc)</c>; events seek on <c>CX_Event_Time</c>.
 ///     A session <c>sp_getapplock</c> (timeout 0) serialises purgers that share one database; a loser skips.
-///     To purge another table later, add it to <see cref="PurgeAsync"/>.
+///     To purge another table later, add it to <see cref="PurgeAsync"/>. On a v1 schema (no Part 11 trail, Gitea #674)
+///     the <c>SampleModified</c> purge is skipped and absent from the results.
 /// </summary>
 internal sealed class SqlServerHistorianRetention : IHistorianRetention
 {
@@ -42,6 +43,9 @@ internal sealed class SqlServerHistorianRetention : IHistorianRetention
    /// <summary>Test seam for <see cref="SqlConnection.ClearPool"/>.</summary>
    internal Action<SqlConnection> ClearPool { get; set; } = SqlConnection.ClearPool;

+    /// <summary>Test seam: the release statement. Tests point it at a failing statement to exercise the pool-clear fallback.</summary>
+    internal string ReleaseSql { get; set; } = "EXEC sp_releaseapplock @Resource = @res, @LockOwner = 'Session';";
+
    public async Task<IReadOnlyList<RetentionPurgeResult>> PurgeAsync(CancellationToken ct)
    {
        await _bootstrap.EnsureAsync(ct).ConfigureAwait(false);
@@ -69,6 +73,8 @@ internal sealed class SqlServerHistorianRetention : IHistorianRetention
                return [];
            }

+            // #674: a v1 schema has no SampleModified table — skip its purge rather than fail the whole pass.
+            var trail = _bootstrap.SupportsModifiedTrail;
            long samples = 0, modified = 0;
            var tagIds = new List<int>();
            await using (var cmd = _conn.CreateCommand(conn, $"SELECT TagId FROM {_conn.Q("Tag")}"))
@@ -82,21 +88,30 @@ internal sealed class SqlServerHistorianRetention : IHistorianRetention
                    cutoff, id, ct).ConfigureAwait(false);
                // The Part 11 modification trail (#668) ages out on the SAME source-time cutoff as the samples it
                // describes; a purge never writes to it (no trigger, no OUTPUT INTO).
-                modified += await DeleteLoopAsync(conn,
-                    $"DELETE TOP (@n) FROM {_conn.Q("SampleModified")} WHERE TagId = @id AND SourceTimeUtc < @cutoff",
-                    cutoff, id, ct).ConfigureAwait(false);
+                if (trail)
+                {
+                    modified += await DeleteLoopAsync(conn,
+                        $"DELETE TOP (@n) FROM {_conn.Q("SampleModified")} WHERE TagId = @id AND SourceTimeUtc < @cutoff",
+                        cutoff, id, ct).ConfigureAwait(false);
+                }
            }

            var events = await DeleteLoopAsync(conn,
                $"DELETE TOP (@n) FROM {_conn.Q("Event")} WHERE EventTimeUtc < @cutoff",
                cutoff, null, ct).ConfigureAwait(false);

-            return
-            [
-                new RetentionPurgeResult("Sample", samples),
-                new RetentionPurgeResult("SampleModified", modified),
-                new RetentionPurgeResult("Event", events),
-            ];
+            return trail
+                ?
+                [
+                    new RetentionPurgeResult("Sample", samples),
+                    new RetentionPurgeResult("SampleModified", modified),
+                    new RetentionPurgeResult("Event", events),
+                ]
+                :
+                [
+                    new RetentionPurgeResult("Sample", samples),
+                    new RetentionPurgeResult("Event", events),
+                ];
        }
        finally
        {
@@ -137,8 +152,7 @@ internal sealed class SqlServerHistorianRetention : IHistorianRetention
    {
        try
        {
-            await using var cmd = _conn.CreateCommand(conn,
-                "EXEC sp_releaseapplock @Resource = @res, @LockOwner = 'Session';");
+            await using var cmd = _conn.CreateCommand(conn, ReleaseSql);
            cmd.Parameters.Add(new SqlParameter("@res", SqlDbType.NVarChar, 255) { Value = LockResource });
            await cmd.ExecuteNonQueryAsync(CancellationToken.None).ConfigureAwait(false);
        }
@@ -15,15 +15,27 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.SqlServer;
 ///     SQL, so a missing schema (or a SQL outage) does not serialise every write, read and alarm event behind one
 ///     semaphore and one round-trip each.
 ///     With <c>AutoCreateSchema=false</c> it only verifies <c>MAX(Version) &gt;= </c><see cref="RequiredVersion"/>.
+///     <para><b>Schema v1 is accepted (Gitea #674).</b> A database at version 1 (no <c>SampleModified</c>) still serves
+///     live writes, raw / processed / at-time reads, alarm events and retention; only the Part 11 modification trail is
+///     unavailable — <see cref="SupportsModifiedTrail"/> is false, modified reads and HistoryUpdate report
+///     <c>BadHistoryOperationUnsupported</c> without touching SQL, and one Warning is logged per instance (= per boot: the
+///     provider shares one bootstrapper). Before #674 a v1 database refused EVERY historian call, continuous-historization
+///     drains included, until script 002 was applied by hand.</para>
 /// </summary>
 internal sealed partial class SqlServerSchemaBootstrapper
 {
    /// <summary>
-    ///     The schema version this build requires: 2 = <c>002_sample_modified.sql</c> (Part 11 modified-value
-    ///     history, Gitea #668). An existing v1 schema is upgraded in place on the next bootstrap when
-    ///     <c>AutoCreateSchema</c> is on; otherwise the script must be applied by hand.
+    ///     The schema version this build requires to serve at all: 1 = <c>001_initial.sql</c>. Gitea #674 lowered it from
+    ///     2 — a missing modification trail disables only the features that need it (<see cref="ModifiedTrailVersion"/>).
    /// </summary>
-    public const int RequiredVersion = 2;
+    public const int RequiredVersion = 1;
+
+    /// <summary>
+    ///     The schema version that carries the Part 11 modification trail: 2 = <c>002_sample_modified.sql</c> (Gitea #668).
+    ///     An auto-creating bootstrap always upgrades to it; with <c>AutoCreateSchema=false</c> the script must be applied by
+    ///     hand (and the node restarted — the verdict is taken once per instance).
+    /// </summary>
+    public const int ModifiedTrailVersion = 2;

    private const string LockResource = "OtOpcUaHistorianSchema";
    private const int LockTimeoutMs = 30_000;
@@ -34,6 +46,13 @@ internal sealed partial class SqlServerSchemaBootstrapper
    private readonly SemaphoreSlim _gate = new(1, 1);
    private readonly BootstrapFailureCache _failure; // guarded by _gate
    private volatile bool _ready;
+    private volatile bool _supportsModifiedTrail;
+
+    /// <summary>
+    ///     Whether the schema carries the Part 11 modification trail (version &gt;= <see cref="ModifiedTrailVersion"/>).
+    ///     Meaningful only after <see cref="EnsureAsync"/> has succeeded (false before). Taken once per instance.
+    /// </summary>
+    public bool SupportsModifiedTrail => _supportsModifiedTrail;

    /// <summary>Creates the bootstrapper.</summary>
    /// <param name="conn">Connection helper.</param>
@@ -99,6 +118,13 @@ internal sealed partial class SqlServerSchemaBootstrapper
            throw new HistorianSchemaMissingException(
                $"Historian schema [{_conn.Schema}] is missing version {RequiredVersion} (found {v}) and AutoCreateSchema is false. " +
                "Apply the docs/historian/sqlserver/NNN_*.sql scripts above version " + v + " in order, or enable AutoCreateSchema.");
+        _supportsModifiedTrail = v >= ModifiedTrailVersion;
+        if (!_supportsModifiedTrail)
+            _log.LogWarning("Historian schema [{Schema}] is at version {Version}, below {TrailVersion}, and AutoCreateSchema is false: " +
+                "the Part 11 modification trail is unavailable, so modified-value history reads and HistoryUpdate report " +
+                "BadHistoryOperationUnsupported and retention skips SampleModified. Live historization, reads, events and retention " +
+                "are unaffected. Apply docs/historian/sqlserver/002_sample_modified.sql and restart the node to enable them.",
+                _conn.Schema, v, ModifiedTrailVersion);
    }

    private async Task ApplyAsync(SqlConnection conn, CancellationToken ct)