feat(historian): #675 reverse HistoryRead-Events paging + end-only backward reads

- IHistorianDataSource.ReadEventsAsync(..., bool reverse, ...) overload: reverse = Part 11 window (lo, hi]
  newest-first by (time, id) DESC. Default forwards forward reads and throws NotSupportedException for
  reverse (Null); Gateway refuses explicitly (ascending-only stream). Server answers
  BadHistoryOperationUnsupported instead of silently returning the earliest N ascending.
- SqlServer / QuestDB serve reverse natively (ORDER BY time DESC, id DESC + TOP/LIMIT).
- Node manager: reverse event reads page with continuation points; direction stored in the cursor
  (HistoryContinuationState.Reverse) so a resume continues in the cursor's direction; reverse tie clusters
  over-fetched forward and sliced from the newest end (SliceTieCluster(reverse:)); end-only reads with a
  cap page backward from EndTime (mirrors raw).
- Resumed event pages over-fetch by the boundary skip, so every non-final page is a full cap page.
- Tests: node-manager reverse paging (ties, clusters, coarse resolution, overfetch bound, resume
  direction both ways, end-only, unsupported backend); shared EventReadConformance on SqlServer +
  QuestDB; over-the-wire reverse paging; offline Gateway pin of the UTC DateTime.MaxValue bound through
  the published client's proto conversion (in-process gRPC capture) and FILETIME.
- docs/Historian.md event paging section.
This commit is contained in:
Joseph Doherty
2026-10-11 11:19:54 -04:00
parent 825393299e
commit 8d154c6a81
20 changed files with 752 additions and 60 deletions
@@ -45,9 +45,10 @@ public sealed class IHistorianDataSourceContractTests
[InlineData("ReadEventsAsync", typeof(Task<HistoricalEventsResult>))]
public void ReadMethods_ReturnExpectedTaskShape(string methodName, Type expectedReturnType)
{
var method = typeof(IHistorianDataSource).GetMethod(methodName);
method.ShouldNotBeNull();
method!.ReturnType.ShouldBe(expectedReturnType);
// ReadEventsAsync is overloaded (the #675 direction-aware overload) — every overload must keep the shape.
var methods = Overloads(methodName);
methods.ShouldNotBeEmpty();
foreach (var method in methods) method.ReturnType.ShouldBe(expectedReturnType);
}
/// <summary>Verifies that GetHealthSnapshot is synchronous.</summary>
@@ -146,11 +147,14 @@ public sealed class IHistorianDataSourceContractTests
[InlineData("ReadEventsAsync", "maxEvents", typeof(int))]
public void HistoryRead_MaxParameter_TypePinned(string methodName, string parameterName, Type expectedType)
{
var method = typeof(IHistorianDataSource).GetMethod(methodName);
method.ShouldNotBeNull();
var parameter = method!.GetParameters().FirstOrDefault(p => p.Name == parameterName);
parameter.ShouldNotBeNull($"Method {methodName} should expose a '{parameterName}' parameter.");
parameter!.ParameterType.ShouldBe(expectedType);
var methods = Overloads(methodName);
methods.ShouldNotBeEmpty();
foreach (var method in methods)
{
var parameter = method.GetParameters().FirstOrDefault(p => p.Name == parameterName);
parameter.ShouldNotBeNull($"Method {methodName} should expose a '{parameterName}' parameter.");
parameter!.ParameterType.ShouldBe(expectedType);
}
}
/// <summary>
@@ -163,13 +167,17 @@ public sealed class IHistorianDataSourceContractTests
[Fact]
public void ReadEventsAsync_sentinel_and_continuation_contract_types_pinned()
{
var method = typeof(IHistorianDataSource).GetMethod("ReadEventsAsync");
method.ShouldNotBeNull();
var methods = Overloads("ReadEventsAsync");
methods.Count.ShouldBe(2, "the five-argument read plus the #675 direction-aware overload");
methods.ShouldContain(m => m.GetParameters().Any(p => p.Name == "reverse" && p.ParameterType == typeof(bool)));
var maxEventsParam = method!.GetParameters().FirstOrDefault(p => p.Name == "maxEvents");
maxEventsParam.ShouldNotBeNull("maxEvents parameter must exist for the sentinel contract");
maxEventsParam!.ParameterType.ShouldBe(typeof(int),
"maxEvents is int (not uint) so callers can pass <=0 as a 'use backend default cap' sentinel");
foreach (var method in methods)
{
var maxEventsParam = method.GetParameters().FirstOrDefault(p => p.Name == "maxEvents");
maxEventsParam.ShouldNotBeNull("maxEvents parameter must exist for the sentinel contract");
maxEventsParam!.ParameterType.ShouldBe(typeof(int),
"maxEvents is int (not uint) so callers can pass <=0 as a 'use backend default cap' sentinel");
}
// HistoricalEventsResult must carry a nullable ContinuationPoint so implementations
// can signal 'more results exist' (non-null) vs 'all results returned' (null).
@@ -179,4 +187,7 @@ public sealed class IHistorianDataSourceContractTests
continuationProp!.PropertyType.ShouldBe(typeof(byte[]),
"ContinuationPoint is byte[]? — null means no more pages, non-null means caller must page");
}
private static List<System.Reflection.MethodInfo> Overloads(string name) =>
typeof(IHistorianDataSource).GetMethods().Where(m => m.Name == name).ToList();
}
@@ -67,6 +67,20 @@ public sealed class NullHistorianDataSourceTests
result.ContinuationPoint.ShouldBeNull();
}
/// <summary>
/// #675: the direction-aware overload's default — forward is the five-argument read, reverse is refused
/// (the server answers <c>BadHistoryOperationUnsupported</c>).
/// </summary>
[Fact]
public async Task Direction_aware_ReadEventsAsync_forwards_forward_and_refuses_reverse()
{
IHistorianDataSource source = Source;
(await source.ReadEventsAsync(null, DateTime.UtcNow.AddHours(-1), DateTime.UtcNow, 0, reverse: false, Ct))
.Events.ShouldBeEmpty();
await Should.ThrowAsync<NotSupportedException>(() =>
source.ReadEventsAsync(null, DateTime.UtcNow, DateTime.UtcNow.AddHours(-1), 0, reverse: true, Ct));
}
/// <summary>GetHealthSnapshot reports a fully-disabled (zeroed, disconnected, no-node) shape.</summary>
[Fact]
public void GetHealthSnapshot_is_disabled()
@@ -0,0 +1,106 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
/// <summary>
/// The direction-aware event read (Gitea #675): forward is <c>[lo, hi)</c> ascending by <c>(EventTimeUtc, EventId)</c>;
/// reverse is the Part 11 reverse window <c>(lo, hi]</c> DESCENDING by the same key, so a capped reverse read returns the
/// LATEST events and flags truncation. Paging a reverse read the way the server does — resume at the last returned
/// time inclusive and drop the boundary ties already emitted — must visit every event exactly once, newest-first,
/// through a same-timestamp tie cluster.
/// </summary>
public abstract class EventReadConformance : ConformanceBase
{
private static readonly CancellationToken Ct = CancellationToken.None;
/// <summary>e0@0, e1@1, then the ties t0..t4@2, then e5@3 and e6@4 (all source "s"), plus o0@2 from another source.</summary>
private async Task<IHistorianDataSource> SeedAsync()
{
await B.SeedEventsAsync(
(At(0), "e0", "s"), (At(1), "e1", "s"),
(At(2), "t0", "s"), (At(2), "t1", "s"), (At(2), "t2", "s"), (At(2), "t3", "s"), (At(2), "t4", "s"),
(At(3), "e5", "s"), (At(4), "e6", "s"),
(At(2), "o0", "other"));
return B.CreateDataSource();
}
private static readonly string[] NewestFirst = ["e6", "e5", "t4", "t3", "t2", "t1", "t0", "e1", "e0"];
[Fact]
public async Task Reverse_read_is_newest_first_by_time_then_EventId_descending()
{
using var src = await SeedAsync();
var r = await src.ReadEventsAsync("s", At(-1), At(10), 0, reverse: true, Ct);
r.Events.Select(e => e.EventId).ShouldBe(NewestFirst);
r.ContinuationPoint.ShouldBeNull();
}
[Fact]
public async Task Reverse_window_is_end_exclusive_start_inclusive_whichever_order_the_bounds_come_in()
{
using var src = await SeedAsync();
// (At(1), At(3)]: e1 at the lower bound is excluded, e5 at the upper bound included.
(await src.ReadEventsAsync("s", At(3), At(1), 0, reverse: true, Ct)).Events.Select(e => e.EventId)
.ShouldBe(["e5", "t4", "t3", "t2", "t1", "t0"]);
(await src.ReadEventsAsync("s", At(1), At(3), 0, reverse: true, Ct)).Events.Select(e => e.EventId)
.ShouldBe(["e5", "t4", "t3", "t2", "t1", "t0"]);
// Forward is unchanged: [At(1), At(3)) ascending.
(await src.ReadEventsAsync("s", At(1), At(3), 0, reverse: false, Ct)).Events.Select(e => e.EventId)
.ShouldBe(["e1", "t0", "t1", "t2", "t3", "t4"]);
}
[Fact]
public async Task A_capped_reverse_read_keeps_the_LATEST_events_and_flags_truncation()
{
using var src = await SeedAsync();
var r = await src.ReadEventsAsync("s", At(10), At(-1), 3, reverse: true, Ct);
r.Events.Select(e => e.EventId).ShouldBe(["e6", "e5", "t4"]);
r.ContinuationPoint.ShouldNotBeNull();
// An exact fit is complete.
(await src.ReadEventsAsync("s", At(10), At(-1), 9, reverse: true, Ct)).ContinuationPoint.ShouldBeNull();
}
[Fact]
public async Task Paging_a_reverse_read_through_a_tie_cluster_visits_every_event_once_newest_first()
{
using var src = await SeedAsync();
foreach (var cap in new[] { 1, 2, 3, 4 })
{
var all = new List<string>();
var lo = At(-1);
var from = At(10);
var skip = 0;
for (var page = 0; ; page++)
{
page.ShouldBeLessThan(40, "paging must terminate");
var r = await src.ReadEventsAsync("s", from, lo, cap + skip, reverse: true, Ct);
var events = r.Events.ToList();
var drop = 0;
while (drop < skip && drop < events.Count && events[drop].EventTimeUtc == from) drop++;
events = events.Skip(drop).ToList();
// The server asks for cap + skip rows (it re-reads the boundary ties it then drops), so a page is never
// emptied by the trim while more remains.
if (r.ContinuationPoint is not null) events.Count.ShouldBe(cap);
all.AddRange(events.Select(e => e.EventId));
if (r.ContinuationPoint is null) break;
var boundary = events[^1].EventTimeUtc;
var ties = events.AsEnumerable().Reverse().TakeWhile(e => e.EventTimeUtc == boundary).Count();
skip = boundary == from ? skip + ties : ties;
from = boundary;
}
all.SequenceEqual(NewestFirst).ShouldBeTrue($"cap {cap}: {string.Join(",", all)}");
}
}
[Fact]
public async Task Reverse_read_with_a_null_source_spans_every_source()
{
using var src = await SeedAsync();
var r = await src.ReadEventsAsync(null, At(2.5), At(1.5), 0, reverse: true, Ct);
r.Events.Select(e => e.EventId).ShouldBe(["t4", "t3", "t2", "t1", "t0", "o0"]);
}
}
@@ -41,6 +41,13 @@ public interface IConformanceBackend : IAsyncDisposable
/// <param name="samples">The rows to store.</param>
Task SeedAsync(string tag, params (DateTime T, HistorianValue V, uint Status)[] samples);
/// <summary>
/// Writes historical events directly in the backend's event table (never through the alarm writer): event id,
/// source and time as given (rounded to the storage resolution), severity 500, received time = event time.
/// </summary>
/// <param name="events">The events to store.</param>
Task SeedEventsAsync(params (DateTime T, string Id, string Source)[] events);
/// <summary>
/// Live (non-deleted) rows for a tag, ascending by time, read directly from storage. The status is returned
/// VERBATIM as stored — there is no read-side clamp (an Empty row stored non-Bad reads back non-Bad here).
@@ -0,0 +1,123 @@
using System.Net;
using Google.Protobuf.WellKnownTypes;
using Grpc.Core;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Server.Kestrel.Core;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
using Xunit;
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
/// <summary>
/// Gitea #675: the Gateway backend's event-read bounds and direction, offline.
/// <list type="bullet">
/// <item>A reverse (newest-first) read is refused before anything is sent — the gateway streams ascending only.</item>
/// <item>The open upper bound a start-only forward read uses (<c>DateTime.MaxValue</c>, UTC kind) survives the
/// PUBLISHED client package's proto conversion — captured on the wire by an in-process gRPC server — and the
/// Windows FILETIME encoding the gateway converts it to. The live AVEVA leg (what the historian does with a
/// year-9999 bound) stays open: it needs the VPN.</item>
/// </list>
/// </summary>
public sealed class GatewayEventBoundsTests
{
private static readonly DateTime UtcMax = new(DateTime.MaxValue.Ticks, DateTimeKind.Utc);
private static CancellationToken Ct => TestContext.Current.CancellationToken;
[Fact]
public async Task A_reverse_events_read_is_refused_and_sends_nothing()
{
var fake = new FakeHistorianGatewayClient();
IHistorianDataSource ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
await Should.ThrowAsync<NotSupportedException>(() =>
ds.ReadEventsAsync("Pump1", DateTime.UtcNow, DateTime.UtcNow.AddHours(-1), 10, reverse: true, Ct));
fake.ReadEventsCallCount.ShouldBe(0);
// Forward through the same overload is the five-argument read, unchanged.
await ds.ReadEventsAsync("Pump1", DateTime.UtcNow.AddHours(-1), DateTime.UtcNow, 10, reverse: false, Ct);
fake.ReadEventsCallCount.ShouldBe(1);
}
[Fact]
public void UtcMax_round_trips_through_the_proto_Timestamp_and_FILETIME_encodings()
{
var ts = Timestamp.FromDateTime(UtcMax);
ts.ToDateTime().ShouldBe(UtcMax);
ts.ToDateTime().Kind.ShouldBe(DateTimeKind.Utc);
// The gateway hands its bounds to the historian as FILETIME (100 ns since 1601) — in range up to year 30828.
var fileTime = ts.ToDateTime().ToFileTimeUtc();
DateTime.FromFileTimeUtc(fileTime).ShouldBe(UtcMax);
}
[Fact]
public async Task A_start_only_bound_of_UtcMax_reaches_the_wire_intact_through_the_published_client()
{
var capture = new CapturingReadService();
await using var server = await StartServerAsync(capture);
using var adapter = HistorianGatewayClientAdapter.Create(
new ServerHistorianOptions
{
Enabled = true,
Endpoint = $"http://127.0.0.1:{server.Port}",
ApiKey = "histgw_test_key",
UseTls = false,
},
NullLoggerFactory.Instance);
IHistorianDataSource ds = new GatewayHistorianDataSource(adapter, NullLogger<GatewayHistorianDataSource>.Instance);
var start = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
var r = await ds.ReadEventsAsync("Pump1", start, UtcMax, 5, Ct);
r.Events.ShouldBeEmpty();
capture.Request.ShouldNotBeNull();
capture.Request!.StartTime.ToDateTime().ShouldBe(start);
capture.Request.EndTime.ToDateTime().ShouldBe(UtcMax);
DateTime.FromFileTimeUtc(capture.Request.EndTime.ToDateTime().ToFileTimeUtc()).ShouldBe(UtcMax);
}
private sealed class CapturingReadService : HistorianRead.HistorianReadBase
{
public ReadEventsRequest? Request { get; private set; }
public override Task ReadEvents(
ReadEventsRequest request, IServerStreamWriter<HistorianEvent> responseStream, ServerCallContext context)
{
Request = request;
return Task.CompletedTask;
}
}
private sealed class Server(WebApplication app, int port) : IAsyncDisposable
{
public int Port { get; } = port;
public async ValueTask DisposeAsync()
{
await app.StopAsync();
await app.DisposeAsync();
}
}
private static async Task<Server> StartServerAsync(CapturingReadService service)
{
var builder = WebApplication.CreateSlimBuilder();
builder.Logging.ClearProviders();
builder.WebHost.ConfigureKestrel(k =>
k.Listen(IPAddress.Loopback, 0, o => o.Protocols = HttpProtocols.Http2));
builder.Services.AddGrpc();
builder.Services.AddSingleton(service);
var app = builder.Build();
app.MapGrpcService<CapturingReadService>();
await app.StartAsync();
var port = new Uri(app.Urls.First()).Port;
return new Server(app, port);
}
}
@@ -15,12 +15,18 @@
<PackageReference Include="Microsoft.NET.Test.Sdk"/>
<!-- ServiceCollection + BuildServiceProvider for the CreateDataSource factory test. -->
<PackageReference Include="Microsoft.Extensions.DependencyInjection"/>
<!-- In-process gRPC server capturing the published client's ReadEvents request (#675 bounds pin). -->
<PackageReference Include="Grpc.AspNetCore"/>
<PackageReference Include="xunit.runner.visualstudio">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>
<ItemGroup>
<FrameworkReference Include="Microsoft.AspNetCore.App"/>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\..\..\src\Drivers\ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway\ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.csproj"/>
</ItemGroup>
@@ -60,3 +60,10 @@ public sealed class QuestDbModifiedValueConformance(QuestDbFixture fx) : Modifie
/// <inheritdoc/>
protected override Task<IConformanceBackend> CreateBackendAsync() => QuestDbConformanceBackend.CreateAsync(fx);
}
[Collection(QuestDbCollection.Name)]
public sealed class QuestDbEventReadConformance(QuestDbFixture fx) : EventReadConformance
{
/// <inheritdoc/>
protected override Task<IConformanceBackend> CreateBackendAsync() => QuestDbConformanceBackend.CreateAsync(fx);
}
@@ -143,6 +143,27 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend
await Wal.WaitAsync(Conn.SampleTable, CancellationToken.None);
}
/// <inheritdoc/>
public async Task SeedEventsAsync(params (DateTime T, string Id, string Source)[] events)
{
await using (var c = await Conn.OpenAsync(CancellationToken.None))
{
foreach (var (t, id, source) in events)
{
await using var cmd = Conn.CreateCommand(c,
$"INSERT INTO {Conn.EventTable} (ts, event_id, source, severity, message, received_utc) VALUES ($1, $2, $3, $4, $5, $6)");
QuestDbConnection.BindTimestamp(cmd, t);
QuestDbConnection.BindText(cmd, id);
QuestDbConnection.BindText(cmd, source);
cmd.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = 500 });
QuestDbConnection.BindText(cmd, null);
QuestDbConnection.BindTimestamp(cmd, t);
await cmd.ExecuteNonQueryAsync();
}
}
await Wal.WaitAsync(Conn.EventTable, CancellationToken.None);
}
/// <summary>Runs a statement against this table set (test-only escape hatch for hand-made rows), then waits for WAL visibility of the sample table.</summary>
/// <param name="sql">The statement; parameters are bound positionally by <paramref name="bind"/>.</param>
/// <param name="bind">Binds the parameters.</param>
@@ -60,3 +60,10 @@ public sealed class SqlServerModifiedValueConformance(SqlHistorianFixture fx) :
/// <inheritdoc/>
protected override Task<IConformanceBackend> CreateBackendAsync() => SqlServerConformanceBackend.CreateAsync(fx);
}
[Collection(SqlHistorianCollection.Name)]
public sealed class SqlServerEventReadConformance(SqlHistorianFixture fx) : EventReadConformance
{
/// <inheritdoc/>
protected override Task<IConformanceBackend> CreateBackendAsync() => SqlServerConformanceBackend.CreateAsync(fx);
}
@@ -40,6 +40,22 @@ internal sealed class SqlServerConformanceBackend : IConformanceBackend
public async Task SeedAsync(string tag, params (DateTime T, HistorianValue V, uint Status)[] samples) =>
await _db.SeedAsync(tag, samples);
/// <inheritdoc/>
public async Task SeedEventsAsync(params (DateTime T, string Id, string Source)[] events)
{
await using var c = await _db.Conn.OpenAsync(CancellationToken.None);
foreach (var (t, id, source) in events)
{
await using var cmd = _db.Conn.CreateCommand(c,
$"INSERT {_db.Conn.Q("Event")} (EventId, SourceName, AlarmId, AlarmName, AlarmTypeName, Severity, EventKind, Message, " +
"UserName, Comment, EventTimeUtc, ReceivedTimeUtc) VALUES (@id, @src, 'a', 'n', 'T', 500, 'Activated', NULL, NULL, NULL, @t, @t)");
cmd.Parameters.AddWithValue("@id", id);
cmd.Parameters.AddWithValue("@src", source);
SqlHistorianConnection.AddDateTime2(cmd, "@t", t);
await cmd.ExecuteNonQueryAsync();
}
}
/// <inheritdoc/>
public async Task<IReadOnlyList<(DateTime T, HistorianValue V, uint Status)>> ReadBackAsync(string tag)
{
@@ -54,8 +54,48 @@ public sealed class EventHistoryPagingOverTheWireTests
finally { await rig.DisposeAsync(); }
}
[Fact]
public async Task A_capped_REVERSE_event_read_pages_newest_first_through_the_session_store_exactly_once()
{
// Gitea #675: StartTime later than EndTime. Before the fix the cap kept the EARLIEST 10, ascending, with no
// continuation point.
var ct = TestContext.Current.CancellationToken;
var rig = await StartAsync("EvtPagingWireRev", encrypted: false, ct);
try
{
var events = Enumerable.Range(0, 25)
.Select(i => new HistoricalEvent($"e{i:00}", "src", Epoch.AddSeconds(i / 2), Epoch, "m", 500))
.ToList(); // pairs share a timestamp, so page boundaries cut through ties
rig.Nm.HistorianDataSource = new SqlLikeEventSource(events);
rig.MaterialiseNative();
rig.Enforce(false, [Key()]);
using var session = await rig.ConnectAsync(anonymous: true, ct);
var node = new NodeId(RawDeviceFolder, (ushort)session.NamespaceUris.GetIndex(V3NodeIds.RawNamespaceUri));
var collected = new List<string>();
var sizes = new List<int>();
byte[]? cp = null;
do
{
var (ids, next) = await ReadPageAsync(session, node, cp, ct, reverse: true);
collected.AddRange(ids);
sizes.Add(ids.Count);
cp = next;
sizes.Count.ShouldBeLessThan(30, "paging must terminate");
}
while (cp is not null);
sizes.ShouldBe([10, 10, 5]);
// Newest-first: time descending, ties by EventId descending.
collected.ShouldBe(events.OrderByDescending(e => e.EventTimeUtc)
.ThenByDescending(e => e.EventId, StringComparer.Ordinal).Select(e => e.EventId));
}
finally { await rig.DisposeAsync(); }
}
private static async Task<(List<string> Ids, byte[]? Cp)> ReadPageAsync(
ISession session, NodeId node, byte[]? cp, CancellationToken ct)
ISession session, NodeId node, byte[]? cp, CancellationToken ct, bool reverse = false)
{
var filter = new EventFilter();
filter.SelectClauses.Add(new SimpleAttributeOperand(ObjectTypeIds.BaseEventType, new QualifiedName("EventId"))
@@ -64,8 +104,8 @@ public sealed class EventHistoryPagingOverTheWireTests
});
var details = new ReadEventDetails
{
StartTime = Epoch.AddDays(-1),
EndTime = Epoch.AddDays(1),
StartTime = reverse ? Epoch.AddDays(1) : Epoch.AddDays(-1),
EndTime = reverse ? Epoch.AddDays(-1) : Epoch.AddDays(1),
NumValuesPerNode = 10,
Filter = filter,
};
@@ -94,6 +134,18 @@ public sealed class EventHistoryPagingOverTheWireTests
return new HistoricalEventsResult(window.Take(cap).ToList(), window.Count > cap ? new byte[] { 1 } : null);
}
/// <summary>The #675 overload: reverse is <c>(lo, hi]</c> by (time, id) descending.</summary>
public async Task<HistoricalEventsResult> ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse, CancellationToken ct)
{
if (!reverse) return await ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, ct).ConfigureAwait(false);
await Task.Delay(1, ct).ConfigureAwait(false);
var (lo, hi) = startUtc <= endUtc ? (startUtc, endUtc) : (endUtc, startUtc);
var window = _all.Where(e => e.EventTimeUtc > lo && e.EventTimeUtc <= hi).Reverse().ToList();
var cap = maxEvents <= 0 ? 1000 : maxEvents;
return new HistoricalEventsResult(window.Take(cap).ToList(), window.Count > cap ? new byte[] { 1 } : null);
}
public Task<CoreHistoryReadResult> ReadRawAsync(string r, DateTime s, DateTime e, uint m, CancellationToken ct) =>
NullHistorianDataSource.Instance.ReadRawAsync(r, s, e, m, ct);
public Task<CoreHistoryReadResult> ReadProcessedAsync(
@@ -199,11 +199,168 @@ public sealed class NodeManagerHistoryReadEventsPagingTests : IDisposable
var mixed = ReadBatch(nm, 4, null, (node, good1[0].Cp), (other, Guid.NewGuid().ToByteArray()));
mixed[0].Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
mixed[0].Ids.ShouldBe(src.All.Skip(4).Take(3).Select(e => e.EventId));
// A resumed page is FULL (#675): it over-fetches by the boundary skip it then trims.
mixed[0].Ids.ShouldBe(src.All.Skip(4).Take(4).Select(e => e.EventId));
mixed[1].Error.StatusCode.Code.ShouldBe(StatusCodes.BadContinuationPointInvalid);
await host.DisposeAsync();
}
// --- Gitea #675: reverse (newest-first) event paging ------------------------------------------------------
[Fact]
public async Task Reverse_events_page_newest_first_exactly_once_with_exact_page_counts()
{
var (host, nm, node, src) = await SetupAsync(Distinct(250));
var pages = new List<int>();
var ids = DrainAsync(nm, node, max: 100, start: Late, end: Early, pageSizes: pages);
ids.ShouldBe(src.All.Reverse().Select(e => e.EventId));
pages.ShouldBe([100, 100, 50]);
await host.DisposeAsync();
}
[Fact]
public async Task Reverse_first_page_is_the_LATEST_events_with_a_continuation_point()
{
// The pre-#675 shape: the cap kept the EARLIEST N, ascending, Good, and no continuation point.
var (host, nm, node, _) = await SetupAsync(Distinct(10));
var p1 = ReadEvents(nm, node, max: 3, inboundCp: null, end: Early, start: Late);
p1.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
p1.Ids.ShouldBe(["e0009", "e0008", "e0007"]);
p1.Cp.ShouldNotBeNull("a capped reverse read has more behind it and must page");
await host.DisposeAsync();
}
[Fact]
public async Task Reverse_tie_at_page_boundary_is_neither_duplicated_nor_skipped()
{
var events = new List<HistoricalEvent>
{
Ev("e0", 0), Ev("e1", 1), Ev("e2", 2), Ev("e3", 2), Ev("e4", 2), Ev("e5", 3),
};
var (host, nm, node, _) = await SetupAsync(events);
// Newest-first, ties in descending EventId: cap 2 cuts the cluster at e4|e3.
DrainAsync(nm, node, max: 2, start: Late, end: Early).ShouldBe(["e5", "e4", "e3", "e2", "e1", "e0"]);
await host.DisposeAsync();
}
[Fact]
public async Task Reverse_tie_cluster_larger_than_the_cap_pages_within_the_cluster()
{
var events = new List<HistoricalEvent> { Ev("a0", 0) };
for (var i = 0; i < 7; i++) events.Add(Ev($"t{i}", 5));
events.Add(Ev("z0", 9));
var (host, nm, node, _) = await SetupAsync(events);
var pages = new List<int>();
DrainAsync(nm, node, max: 3, start: Late, end: Early, pageSizes: pages).ShouldBe(
["z0", "t6", "t5", "t4", "t3", "t2", "t1", "t0", "a0"]);
pages.Sum().ShouldBe(9);
await host.DisposeAsync();
}
[Fact]
public async Task Reverse_tie_cluster_pages_on_a_coarse_resolution_backend()
{
const long res = 10;
var events = new List<HistoricalEvent>();
DateTime At(int slot) => Epoch.AddTicks(slot * res);
events.Add(new("a0", "s", At(0), Epoch, "m", 500));
for (var i = 0; i < 7; i++) events.Add(new($"t{i}", "s", At(3), Epoch, "m", 500));
events.Add(new("z0", "s", At(8), Epoch, "m", 500));
var (host, nm, node, _) = await SetupAsync(events, res);
DrainAsync(nm, node, max: 3, start: Late, end: Early).ShouldBe(
["z0", "t6", "t5", "t4", "t3", "t2", "t1", "t0", "a0"]);
await host.DisposeAsync();
}
[Fact]
public async Task Reverse_tie_cluster_beyond_the_overfetch_bound_fails_loudly()
{
var events = new List<HistoricalEvent>();
for (var i = 0; i < 10; i++) events.Add(Ev($"t{i:00}", 5));
var (host, nm, node, _) = await SetupAsync(events);
nm.MaxTieClusterOverfetch = 4;
var p1 = ReadEvents(nm, node, max: 3, inboundCp: null, end: Early, start: Late);
p1.Cp.ShouldNotBeNull();
ReadEvents(nm, node, max: 3, inboundCp: p1.Cp, end: Early, start: Late).Error.StatusCode.Code
.ShouldBe(StatusCodes.BadHistoryOperationUnsupported);
await host.DisposeAsync();
}
[Fact]
public async Task A_reverse_cursor_resumes_reverse_whatever_the_resume_request_says()
{
// The direction lives in the cursor: a resume carrying FORWARD details still continues newest-first.
var (host, nm, node, _) = await SetupAsync(Distinct(10));
var p1 = ReadEvents(nm, node, max: 4, inboundCp: null, end: Early, start: Late);
p1.Ids.ShouldBe(["e0009", "e0008", "e0007", "e0006"]);
var p2 = ReadEvents(nm, node, max: 4, inboundCp: p1.Cp); // default details: Early -> Late (forward)
p2.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
p2.Ids.ShouldBe(["e0005", "e0004", "e0003", "e0002"]);
await host.DisposeAsync();
}
[Fact]
public async Task A_forward_cursor_resumes_forward_whatever_the_resume_request_says()
{
var (host, nm, node, _) = await SetupAsync(Distinct(10));
var p1 = ReadEvents(nm, node, max: 4, inboundCp: null);
p1.Ids.ShouldBe(["e0000", "e0001", "e0002", "e0003"]);
var p2 = ReadEvents(nm, node, max: 4, inboundCp: p1.Cp, end: Early, start: Late);
p2.Ids.ShouldBe(["e0004", "e0005", "e0006", "e0007"]);
await host.DisposeAsync();
}
[Fact]
public async Task Reverse_unbounded_read_returns_everything_newest_first_without_paging()
{
var (host, nm, node, src) = await SetupAsync(Distinct(30));
var r = ReadEvents(nm, node, max: 0, inboundCp: null, end: Early, start: Late);
r.Ids.ShouldBe(src.All.Reverse().Select(e => e.EventId));
r.Cp.ShouldBeNull();
await host.DisposeAsync();
}
[Fact]
public async Task End_only_events_read_pages_backward_from_EndTime()
{
// StartTime unset + EndTime + NumValuesPerNode > 0 => backward from EndTime (Part 11; mirrors raw). The reverse
// window is (.., EndTime], so an event exactly AT EndTime is the first one returned.
var (host, nm, node, _) = await SetupAsync(Distinct(25));
var ids = DrainAsync(nm, node, max: 10, start: DateTime.MinValue, end: Epoch.AddSeconds(20));
ids.ShouldBe(Enumerable.Range(0, 21).Reverse().Select(i => $"e{i:0000}"));
await host.DisposeAsync();
}
[Fact]
public async Task A_backend_without_reverse_support_answers_BadHistoryOperationUnsupported_not_the_earliest_events()
{
var (host, nm, node, src) = await SetupAsync(Distinct(10));
nm.HistorianDataSource = new ForwardOnlyEventSource(src);
ReadEvents(nm, node, max: 3, inboundCp: null, end: Early, start: Late).Error.StatusCode.Code
.ShouldBe(StatusCodes.BadHistoryOperationUnsupported);
// Forward reads on the same backend are unaffected.
DrainAsync(nm, node, max: 4).ShouldBe(src.All.Select(e => e.EventId));
await host.DisposeAsync();
}
private static readonly DateTime Early = Epoch.AddDays(-1);
private static readonly DateTime Late = Epoch.AddDays(1);
/// <summary>A backend that implements only the five-argument (forward) event read - Gateway / Null shaped.</summary>
private sealed class ForwardOnlyEventSource(IHistorianDataSource inner) : IHistorianDataSource
{
public Task<HistoricalEventsResult> ReadEventsAsync(string? n, DateTime s, DateTime e, int m, CancellationToken ct) =>
inner.ReadEventsAsync(n, s, e, m, ct);
public Task<HistorianRead> ReadRawAsync(string r, DateTime s, DateTime e, uint m, CancellationToken ct) =>
inner.ReadRawAsync(r, s, e, m, ct);
public Task<HistorianRead> ReadProcessedAsync(string r, DateTime s, DateTime e, TimeSpan i, HistoryAggregateType a, CancellationToken ct) =>
inner.ReadProcessedAsync(r, s, e, i, a, ct);
public Task<HistorianRead> ReadAtTimeAsync(string r, IReadOnlyList<DateTime> t, CancellationToken ct) =>
inner.ReadAtTimeAsync(r, t, ct);
public HistorianHealthSnapshot GetHealthSnapshot() => inner.GetHealthSnapshot();
public void Dispose() { }
}
private sealed class RawSeriesSource : IHistorianDataSource
{
public Task<HistorianRead> ReadRawAsync(string r, DateTime s, DateTime e, uint m, CancellationToken ct)
@@ -265,16 +422,18 @@ public sealed class NodeManagerHistoryReadEventsPagingTests : IDisposable
return (started.Host, nm, nm.TryGetFolder(equipmentId)!.NodeId, src);
}
private static List<string> DrainAsync(OtOpcUaNodeManager nm, NodeId node, uint max, DateTime? end = null)
private static List<string> DrainAsync(
OtOpcUaNodeManager nm, NodeId node, uint max, DateTime? end = null, DateTime? start = null, List<int>? pageSizes = null)
{
var all = new List<string>();
byte[]? cp = null;
var pages = 0;
do
{
var r = ReadEvents(nm, node, max, cp, end);
var r = ReadEvents(nm, node, max, cp, end, start);
r.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
all.AddRange(r.Ids);
pageSizes?.Add(r.Ids.Count);
cp = r.Cp;
(++pages).ShouldBeLessThan(60, "paging must terminate");
}
@@ -283,11 +442,15 @@ public sealed class NodeManagerHistoryReadEventsPagingTests : IDisposable
}
private static (List<string> Ids, ServiceResult Error, byte[]? Cp) ReadEvents(
OtOpcUaNodeManager nm, NodeId node, uint max, byte[]? inboundCp, DateTime? end = null) =>
ReadBatch(nm, max, end, (node, inboundCp))[0];
OtOpcUaNodeManager nm, NodeId node, uint max, byte[]? inboundCp, DateTime? end = null, DateTime? start = null) =>
ReadBatch(nm, max, end, start, (node, inboundCp))[0];
private static List<(List<string> Ids, ServiceResult Error, byte[]? Cp)> ReadBatch(
OtOpcUaNodeManager nm, uint max, DateTime? end, params (NodeId Node, byte[]? Cp)[] reads)
OtOpcUaNodeManager nm, uint max, DateTime? end, params (NodeId Node, byte[]? Cp)[] reads) =>
ReadBatch(nm, max, end, null, reads);
private static List<(List<string> Ids, ServiceResult Error, byte[]? Cp)> ReadBatch(
OtOpcUaNodeManager nm, uint max, DateTime? end, DateTime? start, params (NodeId Node, byte[]? Cp)[] reads)
{
var context = new OperationContext(
new RequestHeader(), secureChannelContext: null, RequestType.HistoryRead, identity: null);
@@ -298,7 +461,7 @@ public sealed class NodeManagerHistoryReadEventsPagingTests : IDisposable
});
var details = new ReadEventDetails
{
StartTime = Epoch.AddDays(-1),
StartTime = start ?? Epoch.AddDays(-1),
EndTime = end ?? Epoch.AddDays(1),
NumValuesPerNode = max,
Filter = filter,
@@ -361,6 +524,22 @@ public sealed class NodeManagerHistoryReadEventsPagingTests : IDisposable
window.Take(cap).ToList(), truncated ? new byte[] { 1 } : null));
}
/// <summary>Like the SQL backends' #675 overload: reverse => <c>(lo, hi]</c> newest-first by (time, id) descending.</summary>
public Task<HistoricalEventsResult> ReadEventsAsync(
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents, bool reverse, CancellationToken ct)
{
if (!reverse) return ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, ct);
Calls++;
var (lo, hi) = startUtc <= endUtc ? (startUtc, endUtc) : (endUtc, startUtc);
lo = Trunc(lo);
hi = Trunc(hi);
var window = All.Where(e => e.EventTimeUtc > lo && e.EventTimeUtc <= hi).Reverse().ToList();
var cap = maxEvents <= 0 ? 1000 : maxEvents;
var truncated = window.Count > cap;
return Task.FromResult(new HistoricalEventsResult(
window.Take(cap).ToList(), truncated ? new byte[] { 1 } : null));
}
public Task<HistorianRead> ReadRawAsync(string r, DateTime s, DateTime e, uint m, CancellationToken ct) =>
NullHistorianDataSource.Instance.ReadRawAsync(r, s, e, m, ct);
public Task<HistorianRead> ReadProcessedAsync(