Merge feat/675-reverse-event-paging: reverse event HistoryRead paging, end-only backward, Gateway MaxValue pin (#675)

This commit is contained in:
Joseph Doherty
2026-10-11 11:23:51 -04:00
20 changed files with 752 additions and 60 deletions
@@ -88,7 +88,7 @@ public sealed class HistoryReadLimitExceededException : InvalidOperationExceptio
}
/// <summary>
/// One row returned by <see cref="IHistorianDataSource.ReadEventsAsync"/> — a historical
/// One row returned by <see cref="IHistorianDataSource.ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/> — a historical
/// alarm/event record, not the OPC UA live-event stream. Fields match the minimum set the
/// Server needs to populate a <c>HistoryEventFieldList</c> for HistoryReadEvents responses.
/// </summary>
@@ -106,8 +106,8 @@ public sealed record HistoricalEvent(
string? Message,
ushort Severity);
/// <summary>Result of a <see cref="IHistorianDataSource.ReadEventsAsync"/> call.</summary>
/// <param name="Events">Events in chronological order by <c>EventTimeUtc</c>.</param>
/// <summary>Result of a <see cref="IHistorianDataSource.ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/> call.</summary>
/// <param name="Events">Events ordered by <c>EventTimeUtc</c> in the read's direction — ascending, or descending for a reverse read (#675).</param>
/// <param name="ContinuationPoint">Opaque token for the next call when more events are available; null when complete.</param>
public sealed record HistoricalEventsResult(
IReadOnlyList<HistoricalEvent> Events,
@@ -79,7 +79,7 @@ public interface IHistorianDataSource : IDisposable
/// <c>uint maxValuesPerNode</c> because their OPC UA HistoryRead surface has no
/// equivalent "use default" sentinel.
///
/// This surface declares <see cref="ReadAtTimeAsync"/> and <see cref="ReadEventsAsync"/>
/// This surface declares <see cref="ReadAtTimeAsync"/> and <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/>
/// as required members — a server-side historian owns the full read surface.
/// </remarks>
/// <param name="sourceName">The source name to filter events, or null to return events from all sources.</param>
@@ -101,6 +101,46 @@ public interface IHistorianDataSource : IDisposable
int maxEvents,
CancellationToken cancellationToken);
/// <summary>
/// Read historical alarm / event records in a chosen direction — the overload the server's HistoryReadEvents
/// paging calls (Gitea #675).
/// </summary>
/// <remarks>
/// <para>
/// The bounds may be given in either order; let <c>lo</c>/<c>hi</c> be the earlier/later of them. Forward
/// (<paramref name="reverse"/> <c>false</c>) is exactly <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/>:
/// <c>[lo, hi)</c> ascending by <c>(EventTimeUtc, EventId)</c>. Reverse is the Part 11 reverse window
/// <c>(lo, hi]</c> in DESCENDING <c>(EventTimeUtc, EventId)</c> order, so a cap keeps the LATEST
/// <paramref name="maxEvents"/> events — the mirror of forward, with the same tie-breaker reversed, so a
/// resumed reverse page that re-reads its boundary timestamp sees the boundary ties in the same order.
/// <paramref name="maxEvents"/> and the continuation-point contract are as for the forward read.
/// </para>
/// <para>
/// The default forwards a forward read to the five-argument overload and throws
/// <see cref="NotSupportedException"/> for a reverse one — a backend that cannot read newest-first (Gateway,
/// Null) — which the server reports as <c>BadHistoryOperationUnsupported</c> rather than returning the
/// earliest events in ascending order as if they were the answer.
/// </para>
/// </remarks>
/// <param name="sourceName">The source name to filter events, or null to return events from all sources.</param>
/// <param name="startUtc">One bound of the window (UTC).</param>
/// <param name="endUtc">The other bound of the window (UTC).</param>
/// <param name="maxEvents">The cap; non-positive uses the backend default (see the five-argument overload).</param>
/// <param name="reverse"><c>true</c> to read newest-first over <c>(lo, hi]</c>.</param>
/// <param name="cancellationToken">Cancellation.</param>
/// <returns>The matching events in the read's direction, with a continuation point if truncated.</returns>
/// <exception cref="NotSupportedException">A reverse read on a backend that cannot read newest-first.</exception>
Task<HistoricalEventsResult> ReadEventsAsync(
string? sourceName,
DateTime startUtc,
DateTime endUtc,
int maxEvents,
bool reverse,
CancellationToken cancellationToken) =>
reverse
? throw new NotSupportedException("This historian backend cannot read events newest-first (reverse).")
: ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken);
/// <summary>
/// Point-in-time health snapshot for diagnostics and dashboards. Pure
/// observation; never blocks on backend I/O.
@@ -125,6 +125,21 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
}
}
/// <inheritdoc />
/// <remarks>
/// A reverse (newest-first) read is refused with <see cref="NotSupportedException"/> — the server reports
/// <c>BadHistoryOperationUnsupported</c> (Gitea #675). The gateway's <c>ReadEvents</c> streams ascending only and
/// rejects <c>end &lt; start</c>, so the only way to serve "the latest N" would be to drain the whole window and
/// keep its tail, an unbounded read the cap exists to prevent. Nothing is sent.
/// </remarks>
public Task<HistoricalEventsResult> ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse,
CancellationToken cancellationToken) =>
reverse
? throw new NotSupportedException(
"The HistorianGateway backend reads events oldest-first only; a reverse (newest-first) events read is not supported.")
: ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken);
/// <inheritdoc />
public async Task<HistoricalEventsResult> ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
@@ -361,8 +361,18 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
/// <c>maxEvents + 1</c> rows are fetched; when the extra row arrives it is dropped and the continuation point is
/// the UTF-8 of <see cref="EventContinuation.Encode"/> for that first not-returned event.
/// </remarks>
public Task<HistoricalEventsResult> ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken) =>
ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, reverse: false, cancellationToken);
/// <inheritdoc />
/// <remarks>
/// Forward is the five-argument read. Reverse (Gitea #675) is the (rounded) <c>(lo, hi]</c> ordered by
/// <c>(ts, event_id)</c> DESCENDING, so <c>LIMIT</c> keeps the latest events; truncation and the continuation point
/// work as forward.
/// </remarks>
public async Task<HistoricalEventsResult> ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken)
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse, CancellationToken cancellationToken)
{
// No stored event can carry a source longer than the limit, so an over-long filter matches nothing.
if (sourceName is not null && sourceName.Length > HistorianTagNames.MaxLength)
@@ -376,9 +386,9 @@ public sealed class QuestDbHistorianDataSource : IHistorianDataSource
var bounded = maxEvents > 0 && maxEvents < int.MaxValue;
var sql =
$"SELECT event_id, source, ts, received_utc, message, severity FROM {_conn.EventTable} " +
"WHERE ts >= $1 AND ts < $2" +
(reverse ? "WHERE ts > $1 AND ts <= $2" : "WHERE ts >= $1 AND ts < $2") +
(sourceName is null ? "" : " AND source = $3") +
" ORDER BY ts ASC, event_id ASC" +
(reverse ? " ORDER BY ts DESC, event_id DESC" : " ORDER BY ts ASC, event_id ASC") +
(bounded ? (sourceName is null ? " LIMIT $3" : " LIMIT $4") : "");
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
@@ -320,8 +320,18 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
/// of <c>"{EventTimeUtc.Ticks}|{EventId}"</c> of that first not-returned event (opaque to callers; only
/// its presence is contractual).
/// </remarks>
public Task<HistoricalEventsResult> ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken) =>
ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, reverse: false, cancellationToken);
/// <inheritdoc />
/// <remarks>
/// Forward is the five-argument read. Reverse (Gitea #675) is <c>(lo, hi]</c> ordered by
/// <c>(EventTimeUtc, EventId)</c> DESCENDING — the clustered index scanned backwards — so <c>TOP</c> keeps the latest
/// events; truncation and the continuation point work as forward.
/// </remarks>
public async Task<HistoricalEventsResult> ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, CancellationToken cancellationToken)
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse, CancellationToken cancellationToken)
{
// No stored event can carry a source longer than the column (the alarm writer dead-letters it rather than
// truncating), so an over-long filter matches nothing — never a truncated prefix of someone else's source.
@@ -334,9 +344,10 @@ public sealed class SqlServerHistorianDataSource : IHistorianDataSource
var bounded = maxEvents > 0 && maxEvents < int.MaxValue;
var sql =
$"SELECT {(bounded ? "TOP (@take) " : "")}EventId, SourceName, EventTimeUtc, ReceivedTimeUtc, Message, Severity " +
$"FROM {_conn.Q("Event")} WHERE EventTimeUtc >= @lo AND EventTimeUtc < @hi" +
$"FROM {_conn.Q("Event")} WHERE " +
(reverse ? "EventTimeUtc > @lo AND EventTimeUtc <= @hi" : "EventTimeUtc >= @lo AND EventTimeUtc < @hi") +
(sourceName is null ? "" : " AND SourceName = @source") +
" ORDER BY EventTimeUtc ASC, EventId ASC";
(reverse ? " ORDER BY EventTimeUtc DESC, EventId DESC" : " ORDER BY EventTimeUtc ASC, EventId ASC");
await using var conn = await _conn.OpenAsync(cancellationToken).ConfigureAwait(false);
await using var cmd = _conn.CreateCommand(conn, sql);
@@ -46,6 +46,11 @@ namespace ZB.MOM.WW.OtOpcUa.OpcUaServer;
/// <see cref="HistoryCursorKind.Modified"/> only: the normalised window start, so a resumed page keeps the original
/// window and direction (the modified read resumes by key, not by moving the window start).
/// </param>
/// <param name="Reverse">
/// <see cref="HistoryCursorKind.Events"/> only (Gitea #675): the read is newest-first over
/// <c>(EndUtc, NextStartUtc]</c>. A resumed events read takes its direction from here, never from the resume
/// request's details. Raw keeps deriving it from the window (<c>NextStartUtc &gt; EndUtc</c>).
/// </param>
internal sealed record HistoryContinuationState(
string Tagname,
DateTime NextStartUtc,
@@ -54,7 +59,8 @@ internal sealed record HistoryContinuationState(
uint NumValuesPerNode,
HistoryCursorKind Kind = HistoryCursorKind.Raw,
long LastModifiedId = 0,
DateTime WindowStartUtc = default);
DateTime WindowStartUtc = default,
bool Reverse = false);
/// <summary>The HistoryRead kind that issued a <see cref="HistoryContinuationState"/>.</summary>
internal enum HistoryCursorKind
@@ -102,7 +102,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
/// <see cref="_alarmConditions"/> / <see cref="_notifierFolders"/>.</summary>
private readonly Dictionary<RealmKey, AlarmNotifierWiring> _alarmNotifierWiring = new();
/// <summary>Phase C: event-notifier folder NodeId-identifier → the event-history source
/// name passed to <see cref="IHistorianDataSource.ReadEventsAsync"/>. The equipment-folder NodeId
/// name passed to <see cref="IHistorianDataSource.ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/>. The equipment-folder NodeId
/// identifier IS the equipment id, which IS the sourceName, so key and value are the same string;
/// the map's presence (not its value) is what makes a folder an event-history source. Populated by
/// <see cref="EnsureFolderIsEventNotifier"/> only when a real historian is wired at promotion time,
@@ -5195,6 +5195,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
var endUtc = details.EndTime;
var numValuesPerNode = details.NumValuesPerNode;
var boundarySkip = 0;
bool reverse;
if (resumed)
{
@@ -5214,36 +5215,59 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
endUtc = state.EndUtc;
boundarySkip = state.BoundarySkipCount;
numValuesPerNode = state.NumValuesPerNode;
// The direction is the CURSOR's (Gitea #675), never re-derived from the resume request's details: a
// reverse read resumes reverse, a forward one forward.
reverse = state.Reverse;
}
else
{
// Part 11: with one bound unset, an unset EndTime (MinValue) with a StartTime means "page FORWARD from
// StartTime" — normalise it to an open upper bound, or the cursor would store MinValue and the read would
// look reversed. An unset StartTime with an EndTime and a count cap means "page BACKWARD from EndTime"
// (#675, mirroring raw): the reverse window (MinValue, EndTime]. Without a count cap an end-only read
// keeps its historical forward [MinValue, EndTime) meaning.
if (endUtc == DateTime.MinValue && startUtc != DateTime.MinValue)
endUtc = new DateTime(DateTime.MaxValue.Ticks, DateTimeKind.Utc); // Utc kind: a proto Timestamp conversion rejects Unspecified
else if (numValuesPerNode > 0 && startUtc == DateTime.MinValue && endUtc != DateTime.MinValue)
(startUtc, endUtc) = (endUtc, new DateTime(0, DateTimeKind.Utc));
// Part 11: an unset EndTime (MinValue) with a StartTime means "page forward from StartTime". Normalise it
// to an open upper bound, or the cursor would store MinValue and a backend would swap the reversed
// resume window. A resumed read is always forward (the cursor is only ever stored for a forward read),
// so its direction comes from the cursor, not from the resume request's details.
if (!resumed && endUtc == DateTime.MinValue && startUtc != DateTime.MinValue)
endUtc = new DateTime(DateTime.MaxValue.Ticks, DateTimeKind.Utc); // Utc kind: a proto Timestamp conversion rejects Unspecified
// Reverse event reads (start > end) are not paged: the cursor below is forward-only.
var forward = resumed || startUtc <= endUtc;
// Part 11: StartTime later than EndTime is a reverse read — events newest-first over (end, start].
reverse = startUtc > endUtc;
}
// NOT under the node-manager Lock — awaiting the async source is safe here.
// NumValuesPerNode==0 means "no limit — return ALL values" per OPC UA Part 4/11, so it is translated
// to UNBOUNDED (EventMaxEvents) rather than the backend's <=0 default-cap sentinel.
// to UNBOUNDED (EventMaxEvents) rather than the backend's <=0 default-cap sentinel. A resumed page re-reads its
// boundary timestamp inclusively and then drops the `boundarySkip` ties already emitted, so it asks for that
// many more rows than the cap — otherwise every resumed page would come back short by the skip.
var fetch = EventMaxEvents(numValuesPerNode);
if (numValuesPerNode > 0 && boundarySkip > 0)
fetch = (int)Math.Min(int.MaxValue - 1L, (long)fetch + boundarySkip);
var sourceResult = await _historianDataSource.ReadEventsAsync(
sourceName, startUtc, endUtc, EventMaxEvents(numValuesPerNode), ct).ConfigureAwait(false);
sourceName, startUtc, endUtc, fetch, reverse, ct).ConfigureAwait(false);
// Truncation is the SOURCE's word (non-null continuation point) — never inferred from the count, so a
// page that happens to be exactly cap-sized but complete never emits a point.
var truncated = sourceResult.ContinuationPoint is not null && numValuesPerNode > 0 && forward;
var truncated = sourceResult.ContinuationPoint is not null && numValuesPerNode > 0;
IReadOnlyList<HistoricalEvent> events = sourceResult.Events;
if (resumed && boundarySkip > 0)
{
// Drop the events at the boundary timestamp already emitted on the prior page.
// Drop the events at the boundary timestamp already emitted on the prior page. The resumed read is in
// the cursor's direction, so in both directions those ties lead the page.
var i = 0;
while (i < events.Count && i < boundarySkip && events[i].EventTimeUtc == startUtc) i++;
if (i > 0) events = events.Skip(i).ToList();
}
// The skip over-fetch can leave more than the cap (fewer boundary ties came back than were skipped — pruned
// between pages): keep the cap, and the dropped tail means the read continues.
if (numValuesPerNode > 0 && events.Count > numValuesPerNode)
{
events = events.Take((int)Math.Min(int.MaxValue, numValuesPerNode)).ToList();
truncated = true;
}
// The cursor to store, if any. Saved only AFTER the page is projected, so a projection throw can
// never orphan a stored cursor.
HistoryContinuationState? pendingCursor = null;
@@ -5256,10 +5280,14 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
// with an explicit bounded cap and page WITHIN it (same scheme as raw).
var resolution = Math.Max(1L, _historianDataSource.TimestampResolutionTicks);
var overfetchCap = (int)Math.Min(int.MaxValue - 1L, (long)Math.Max(1, MaxTieClusterOverfetch) + 1);
var cluster = (await _historianDataSource.ReadEventsAsync(
sourceName, startUtc, startUtc.AddTicks(resolution), overfetchCap, ct)
IReadOnlyList<HistoricalEvent> cluster = (await _historianDataSource.ReadEventsAsync(
sourceName, startUtc, startUtc.AddTicks(resolution), overfetchCap, reverse: false, ct)
.ConfigureAwait(false)).Events;
// The over-fetch reads the cluster FORWARD (ascending EventId), but a reverse read emitted its ties — and
// counted boundarySkip — in descending order. Slice in the read's own direction, as raw does.
if (reverse) cluster = cluster.Reverse().ToList();
if (cluster.Count > MaxTieClusterOverfetch)
{
_logger.LogError(
@@ -5274,7 +5302,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
HistoryPaging.SliceTieCluster(
cluster.Count, boundarySkip, numValuesPerNode, startUtc, endUtc,
out var sliceStart, out var sliceCount, out var nextStartUtc, out var nextSkip,
reverse: false, resolutionTicks: resolution);
reverse: reverse, resolutionTicks: resolution);
var slice = new List<HistoricalEvent>(sliceCount);
for (var i = sliceStart; i < sliceStart + sliceCount; i++) slice.Add(cluster[i]);
@@ -5283,17 +5311,20 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
if (nextStartUtc is { } resumeAt)
{
pendingCursor = new HistoryContinuationState(
sourceName ?? string.Empty, resumeAt, endUtc, nextSkip, numValuesPerNode, HistoryCursorKind.Events);
sourceName ?? string.Empty, resumeAt, endUtc, nextSkip, numValuesPerNode, HistoryCursorKind.Events,
Reverse: reverse);
}
}
else if (truncated && events.Count > 0)
{
// Cursor from the last event: resume at its time INCLUSIVE, skipping the trailing ties emitted.
// Cursor from the last event: resume at its time INCLUSIVE, skipping the trailing ties emitted. In a
// reverse read the last event is the OLDEST on the page, and the resumed (end, T] read starts there.
var boundary = events[^1].EventTimeUtc;
var skip = 0;
for (var i = events.Count - 1; i >= 0 && events[i].EventTimeUtc == boundary; i--) skip++;
pendingCursor = new HistoryContinuationState(
sourceName ?? string.Empty, boundary, endUtc, skip, numValuesPerNode, HistoryCursorKind.Events);
sourceName ?? string.Empty, boundary, endUtc, skip, numValuesPerNode, HistoryCursorKind.Events,
Reverse: reverse);
}
var historyEvent = ProjectEvents(events, selectClauses);
@@ -5314,6 +5345,15 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
errors[handle.Index] = StatusCodes.BadTimeout;
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadTimeout };
}
catch (NotSupportedException ex)
{
// The backend cannot serve this read's shape — today a reverse (newest-first) read on Gateway / Null
// (#675). A client request the backend refuses, not a backend fault: Warning, not Error.
_logger.LogWarning("OtOpcUaNodeManager: HistoryReadEvents for node {NodeId} is not supported by the historian backend: {Reason}",
handle.NodeId, ex.Message);
errors[handle.Index] = StatusCodes.BadHistoryOperationUnsupported;
results[handle.Index] = new SdkHistoryReadResult { StatusCode = StatusCodes.BadHistoryOperationUnsupported };
}
catch (Exception ex)
{
// One node's backend failure must not throw out of the batch — surface Bad for THIS node
@@ -5693,7 +5733,7 @@ public sealed class OtOpcUaNodeManager : CustomNodeManager2
private static int ClampToInt(uint value) => value > int.MaxValue ? int.MaxValue : (int)value;
/// <summary>Map a HistoryRead-Events <c>NumValuesPerNode</c> request cap onto the
/// <see cref="IHistorianDataSource.ReadEventsAsync"/> <c>maxEvents</c> argument, honouring the OPC UA
/// <see cref="IHistorianDataSource.ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/> <c>maxEvents</c> argument, honouring the OPC UA
/// Part 4/11 semantics that <c>NumValuesPerNode == 0</c> means "no limit — return ALL values".
/// We translate 0 to UNBOUNDED (<see cref="int.MaxValue"/>) — a very large positive cap — rather than
/// the backend's <c>maxEvents &lt;= 0</c> "use the default cap" sentinel, so a client asking for the