dohertj2/lmxopcua
1
0
Fork 0
Code Issues 14 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(historian): #691.4 walk reverse events reads back on forward-only backends (Gateway)

Browse Source 
A reverse or end-only capped events read on a backend without
SupportsReverseEvents (Gateway, Null) was refused
BadHistoryOperationUnsupported (explicit reverse) or answered with the
OLDEST N ascending (end-only, #675). It is now served newest-first by
the same ReverseWalk as raw reads: page-sized ms-aligned segments,
never answered from a read the source flagged truncated, ties sorted by
EventId (the gateway orders by event time only - the fake shuffles
them per call), one extra event walked to signal 'more', uncapped = one
forward read reversed, and the width + proven-empty stretch carried in
the events cursor. The oversized-cluster over-fetch keeps exactly the
events at T in the same order. A tie cluster over the budget is
BadTooManyOperations. Docs note the gateway's EventReadMaxRows must be
unlimited (it truncates silently).
This commit is contained in:
Joseph Doherty
2026-10-12 16:48:05 -04:00
parent 186b7d5ea5
commit d277f34a96
9 changed files with 251 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayEventBoundsTests.cs
+1 -1
View File
@@ -35,7 +35,7 @@ public sealed class GatewayEventBoundsTests
{
var fake = new FakeHistorianGatewayClient();
IHistorianDataSource ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
ds.SupportsReverseEvents.ShouldBeFalse("the server must keep end-only reads forward on the default backend");
ds.SupportsReverseEvents.ShouldBeFalse("the server must walk reverse reads back from forward reads on the default backend (#691.4)");
await Should.ThrowAsync<NotSupportedException>(() =>
ds.ReadEventsAsync("Pump1", DateTime.UtcNow, DateTime.UtcNow.AddHours(-1), 10, reverse: true, Ct));
tests/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer.Tests/NodeManagerHistoryReadEventsPagingTests.cs
+127 -28
View File
@@ -333,12 +333,16 @@ public sealed class NodeManagerHistoryReadEventsPagingTests : IDisposable
}
[Fact]
public async Task A_backend_without_reverse_support_answers_BadHistoryOperationUnsupported_not_the_earliest_events()
public async Task A_backend_without_reverse_support_walks_a_reverse_read_back_never_answering_the_earliest_events()
{
// #691.4 (was refused BadHistoryOperationUnsupported since #675): served newest-first from forward reads.
var (host, nm, node, src) = await SetupAsync(Distinct(10));
nm.HistorianDataSource = new ForwardOnlyEventSource(src);
ReadEvents(nm, node, max: 3, inboundCp: null, end: Early, start: Late).Error.StatusCode.Code
.ShouldBe(StatusCodes.BadHistoryOperationUnsupported);
var fwd = new ForwardOnlyEventSource(src);
nm.HistorianDataSource = fwd;
var r = ReadEvents(nm, node, max: 3, inboundCp: null, end: Early, start: Late);
r.Error.StatusCode.Code.ShouldBe(StatusCodes.Good);
r.Ids.ShouldBe(["e0009", "e0008", "e0007"]);
r.Cp.ShouldNotBeNull();
// Forward reads on the same backend are unaffected.
DrainAsync(nm, node, max: 4).ShouldBe(src.All.Select(e => e.EventId));
await host.DisposeAsync();
@@ -349,30 +353,115 @@ public sealed class NodeManagerHistoryReadEventsPagingTests : IDisposable
// --- #675 review follow-ups: the reverse-events capability ------------------------------------------------
// --- #691.4: reverse events on a backend that reads forward only (the Gateway) are walked back ------------------
[Fact]
public async Task End_only_capped_read_on_a_backend_without_reverse_support_pages_forward_as_before()
public async Task End_only_capped_read_on_a_forward_only_backend_pages_backward_from_EndTime()
{
// Gateway (the default backend) cannot read newest-first: an end-only read keeps its pre-#675 forward
// [MinValue, EndTime) meaning there rather than turning into a refused reverse read.
var (host, nm, node, src) = await SetupAsync(Distinct(25));
var fwd = new NonReverseSource(src);
nm.HistorianDataSource = fwd;
// Part 11 (as raw, #686): the latest events at or before EndTime, newest first — on Gateway this used to keep its
// pre-#675 forward [MinValue, EndTime) meaning, i.e. the OLDEST events. The backend is never sent a reverse read.
var (host, nm, node, _) = await SetupAsync([]);
var gw = new GatewayShapedEventSource(Distinct(25));
nm.HistorianDataSource = gw;
var ids = DrainAsync(nm, node, max: 10, start: DateTime.MinValue, end: Epoch.AddSeconds(20));
ids.ShouldBe(Enumerable.Range(0, 20).Select(i => $"e{i:0000}"));
fwd.ReverseCalls.ShouldBe(0);
ids.ShouldBe(Enumerable.Range(0, 21).Reverse().Select(i => $"e{i:0000}"));
gw.ReverseCalls.ShouldBe(0);
gw.BadRequests.ShouldBe(0);
await host.DisposeAsync();
}
[Fact]
public async Task Explicit_reverse_read_on_a_backend_without_reverse_support_is_refused_without_calling_it()
public async Task Explicit_reverse_read_on_a_forward_only_backend_pages_the_reverse_window_newest_first()
{
var (host, nm, node, src) = await SetupAsync(Distinct(10));
var fwd = new NonReverseSource(src);
nm.HistorianDataSource = fwd;
ReadEvents(nm, node, max: 3, inboundCp: null, end: Early, start: Late).Error.StatusCode.Code
.ShouldBe(StatusCodes.BadHistoryOperationUnsupported);
fwd.ReverseCalls.ShouldBe(0, "the capability is checked BEFORE the source is called");
fwd.ForwardCalls.ShouldBe(0);
// (EndTime, StartTime]: StartTime inclusive, EndTime exclusive.
var (host, nm, node, _) = await SetupAsync([]);
var gw = new GatewayShapedEventSource(Distinct(30));
nm.HistorianDataSource = gw;
var ids = DrainAsync(nm, node, max: 4, start: Epoch.AddSeconds(25), end: Epoch.AddSeconds(5));
ids.ShouldBe(Enumerable.Range(6, 20).Reverse().Select(i => $"e{i:0000}"));
gw.ReverseCalls.ShouldBe(0);
await host.DisposeAsync();
}
[Fact]
public async Task Reverse_ties_on_a_forward_only_backend_page_in_a_stable_order_although_the_backend_shuffles_them()
{
// The gateway orders events by EventTimeUtc only, so ties come back in any order — this fake shuffles them on every
// call. The walk sorts ties by EventId (descending, newest-first) on every read, and the oversized-cluster over-fetch
// does the same, so a boundary-tie skip always lines up. Cap 3 cuts through a 7-way cluster.
var events = new List<HistoricalEvent> { Ev("a0", 0) };
for (var i = 0; i < 7; i++) events.Add(Ev($"t{i}", 5));
events.Add(Ev("z0", 9));
var (host, nm, node, _) = await SetupAsync([]);
nm.HistorianDataSource = new GatewayShapedEventSource(events, shuffleSeed: 7);
DrainAsync(nm, node, max: 3, start: DateTime.MinValue, end: Epoch.AddSeconds(30)).ShouldBe(
["z0", "t6", "t5", "t4", "t3", "t2", "t1", "t0", "a0"]);
await host.DisposeAsync();
}
[Fact]
public async Task Uncapped_reverse_read_on_a_forward_only_backend_is_one_forward_read()
{
var (host, nm, node, _) = await SetupAsync([]);
var gw = new GatewayShapedEventSource(Distinct(30));
nm.HistorianDataSource = gw;
var r = ReadEvents(nm, node, max: 0, inboundCp: null, end: Epoch.AddSeconds(5), start: Epoch.AddSeconds(25));
r.Ids.ShouldBe(Enumerable.Range(6, 20).Reverse().Select(i => $"e{i:0000}"));
r.Cp.ShouldBeNull();
gw.ForwardCalls.ShouldBe(1);
await host.DisposeAsync();
}
[Fact]
public async Task Reverse_paging_on_a_forward_only_backend_is_exact_on_random_data()
{
var (host, nm, node, _) = await SetupAsync([]);
for (var seed = 0; seed < 80; seed++)
{
var rng = new Random(seed);
var events = new List<HistoricalEvent>();
var t = Epoch.Ticks;
var n = rng.Next(1, 120);
while (events.Count < n)
{
t += rng.Next(4) switch
{
0 => rng.Next(1, 10),
1 => rng.Next(1, 5) * TimeSpan.TicksPerMillisecond,
2 => rng.Next(1, 90) * TimeSpan.TicksPerMinute,
_ => rng.Next(1, 2000) * TimeSpan.TicksPerMillisecond,
};
var ties = rng.Next(5) == 0 ? rng.Next(2, 8) : 1;
for (var k = 0; k < ties; k++)
{
var at = new DateTime(t, DateTimeKind.Utc);
events.Add(new HistoricalEvent($"{Guid.NewGuid():N}", "src", at, at, "m", 500));
}
}
nm.HistorianDataSource = new GatewayShapedEventSource(events, shuffleSeed: seed);
nm.MaxTieClusterOverfetch = rng.Next(2) == 0 ? 8 : 65536;
var max = (uint)rng.Next(1, 9);
var end = new DateTime(t + rng.Next(0, 3) * TimeSpan.TicksPerSecond, DateTimeKind.Utc);
var expected = events.OrderByDescending(e => e.EventTimeUtc).ThenByDescending(e => e.EventId, StringComparer.Ordinal)
.Select(e => e.EventId).ToList();
var all = new List<string>();
byte[]? cp = null;
var pages = 0;
do
{
var r = ReadEvents(nm, node, max, cp, end, DateTime.MinValue);
r.Error.StatusCode.Code.ShouldBe(StatusCodes.Good, $"seed {seed}");
all.AddRange(r.Ids);
cp = r.Cp;
(++pages).ShouldBeLessThan(2000);
}
while (cp is not null);
all.ShouldBe(expected, $"seed {seed}, N = {max}");
}
await host.DisposeAsync();
}
@@ -391,29 +480,39 @@ public sealed class NodeManagerHistoryReadEventsPagingTests : IDisposable
await host.DisposeAsync();
}
/// <summary>Declares the reverse overload (so a call would be visible) but reports no reverse capability.</summary>
private sealed class NonReverseSource(IHistorianDataSource inner) : IHistorianDataSource
/// <summary>
/// Shaped like the Gateway's SQL event read (#691.4): forward only (<see cref="SupportsReverseEvents"/> false — a
/// reverse call is counted and refused), <c>[start, end]</c> INCLUSIVE, ordered by event time only with ties in an
/// arbitrary order (shuffled on every call when a seed is given), capped with a truncation flag.
/// </summary>
private sealed class GatewayShapedEventSource(List<HistoricalEvent> events, int? shuffleSeed = null) : IHistorianDataSource
{
private readonly Random? _rng = shuffleSeed is { } s ? new Random(s) : null;
public int ReverseCalls { get; private set; }
public int ForwardCalls { get; private set; }
public int BadRequests { get; private set; }
public bool SupportsReverseEvents => false;
public Task<HistoricalEventsResult> ReadEventsAsync(string? n, DateTime s, DateTime e, int m, CancellationToken ct)
{
ForwardCalls++;
return inner.ReadEventsAsync(n, s, e, m, ct);
if (s > e || s < DateTime.UnixEpoch) BadRequests++;
var window = events.Where(x => x.EventTimeUtc >= s && x.EventTimeUtc <= e)
.OrderBy(x => x.EventTimeUtc).ThenBy(_ => _rng?.Next() ?? 0).ToList();
var cap = m <= 0 ? int.MaxValue : m;
return Task.FromResult(new HistoricalEventsResult(window.Take(cap).ToList(), window.Count > cap ? new byte[] { 1 } : null));
}
public Task<HistoricalEventsResult> ReadEventsAsync(string? n, DateTime s, DateTime e, int m, bool reverse, CancellationToken ct)
{
if (reverse) { ReverseCalls++; return inner.ReadEventsAsync(n, s, e, m, true, ct); }
if (reverse) { ReverseCalls++; throw new NotSupportedException("forward only"); }
return ReadEventsAsync(n, s, e, m, ct);
}
public Task<HistorianRead> ReadRawAsync(string r, DateTime s, DateTime e, uint m, CancellationToken ct) =>
inner.ReadRawAsync(r, s, e, m, ct);
NullHistorianDataSource.Instance.ReadRawAsync(r, s, e, m, ct);
public Task<HistorianRead> ReadProcessedAsync(string r, DateTime s, DateTime e, TimeSpan i, HistoryAggregateType a, CancellationToken ct) =>
inner.ReadProcessedAsync(r, s, e, i, a, ct);
NullHistorianDataSource.Instance.ReadProcessedAsync(r, s, e, i, a, ct);
public Task<HistorianRead> ReadAtTimeAsync(string r, IReadOnlyList<DateTime> t, CancellationToken ct) =>
inner.ReadAtTimeAsync(r, t, ct);
public HistorianHealthSnapshot GetHealthSnapshot() => inner.GetHealthSnapshot();
NullHistorianDataSource.Instance.ReadAtTimeAsync(r, t, ct);
public HistorianHealthSnapshot GetHealthSnapshot() => NullHistorianDataSource.Instance.GetHealthSnapshot();
public void Dispose() { }
}