Merge feat/675-reverse-event-paging: reverse event HistoryRead paging, end-only backward, Gateway MaxValue pin (#675)

2026-10-11 11:23:51 -04:00
parent e59ab69398 8d154c6a81
commit 8e6ac4b344
20 changed files with 752 additions and 60 deletions
@@ -0,0 +1,106 @@
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Conformance;
+
+/// <summary>
+///     The direction-aware event read (Gitea #675): forward is <c>[lo, hi)</c> ascending by <c>(EventTimeUtc, EventId)</c>;
+///     reverse is the Part 11 reverse window <c>(lo, hi]</c> DESCENDING by the same key, so a capped reverse read returns the
+///     LATEST events and flags truncation. Paging a reverse read the way the server does — resume at the last returned
+///     time inclusive and drop the boundary ties already emitted — must visit every event exactly once, newest-first,
+///     through a same-timestamp tie cluster.
+/// </summary>
+public abstract class EventReadConformance : ConformanceBase
+{
+    private static readonly CancellationToken Ct = CancellationToken.None;
+
+    /// <summary>e0@0, e1@1, then the ties t0..t4@2, then e5@3 and e6@4 (all source "s"), plus o0@2 from another source.</summary>
+    private async Task<IHistorianDataSource> SeedAsync()
+    {
+        await B.SeedEventsAsync(
+            (At(0), "e0", "s"), (At(1), "e1", "s"),
+            (At(2), "t0", "s"), (At(2), "t1", "s"), (At(2), "t2", "s"), (At(2), "t3", "s"), (At(2), "t4", "s"),
+            (At(3), "e5", "s"), (At(4), "e6", "s"),
+            (At(2), "o0", "other"));
+        return B.CreateDataSource();
+    }
+
+    private static readonly string[] NewestFirst = ["e6", "e5", "t4", "t3", "t2", "t1", "t0", "e1", "e0"];
+
+    [Fact]
+    public async Task Reverse_read_is_newest_first_by_time_then_EventId_descending()
+    {
+        using var src = await SeedAsync();
+        var r = await src.ReadEventsAsync("s", At(-1), At(10), 0, reverse: true, Ct);
+        r.Events.Select(e => e.EventId).ShouldBe(NewestFirst);
+        r.ContinuationPoint.ShouldBeNull();
+    }
+
+    [Fact]
+    public async Task Reverse_window_is_end_exclusive_start_inclusive_whichever_order_the_bounds_come_in()
+    {
+        using var src = await SeedAsync();
+        // (At(1), At(3)]: e1 at the lower bound is excluded, e5 at the upper bound included.
+        (await src.ReadEventsAsync("s", At(3), At(1), 0, reverse: true, Ct)).Events.Select(e => e.EventId)
+            .ShouldBe(["e5", "t4", "t3", "t2", "t1", "t0"]);
+        (await src.ReadEventsAsync("s", At(1), At(3), 0, reverse: true, Ct)).Events.Select(e => e.EventId)
+            .ShouldBe(["e5", "t4", "t3", "t2", "t1", "t0"]);
+        // Forward is unchanged: [At(1), At(3)) ascending.
+        (await src.ReadEventsAsync("s", At(1), At(3), 0, reverse: false, Ct)).Events.Select(e => e.EventId)
+            .ShouldBe(["e1", "t0", "t1", "t2", "t3", "t4"]);
+    }
+
+    [Fact]
+    public async Task A_capped_reverse_read_keeps_the_LATEST_events_and_flags_truncation()
+    {
+        using var src = await SeedAsync();
+        var r = await src.ReadEventsAsync("s", At(10), At(-1), 3, reverse: true, Ct);
+        r.Events.Select(e => e.EventId).ShouldBe(["e6", "e5", "t4"]);
+        r.ContinuationPoint.ShouldNotBeNull();
+        // An exact fit is complete.
+        (await src.ReadEventsAsync("s", At(10), At(-1), 9, reverse: true, Ct)).ContinuationPoint.ShouldBeNull();
+    }
+
+    [Fact]
+    public async Task Paging_a_reverse_read_through_a_tie_cluster_visits_every_event_once_newest_first()
+    {
+        using var src = await SeedAsync();
+        foreach (var cap in new[] { 1, 2, 3, 4 })
+        {
+            var all = new List<string>();
+            var lo = At(-1);
+            var from = At(10);
+            var skip = 0;
+            for (var page = 0; ; page++)
+            {
+                page.ShouldBeLessThan(40, "paging must terminate");
+                var r = await src.ReadEventsAsync("s", from, lo, cap + skip, reverse: true, Ct);
+                var events = r.Events.ToList();
+                var drop = 0;
+                while (drop < skip && drop < events.Count && events[drop].EventTimeUtc == from) drop++;
+                events = events.Skip(drop).ToList();
+                // The server asks for cap + skip rows (it re-reads the boundary ties it then drops), so a page is never
+                // emptied by the trim while more remains.
+                if (r.ContinuationPoint is not null) events.Count.ShouldBe(cap);
+
+                all.AddRange(events.Select(e => e.EventId));
+                if (r.ContinuationPoint is null) break;
+                var boundary = events[^1].EventTimeUtc;
+                var ties = events.AsEnumerable().Reverse().TakeWhile(e => e.EventTimeUtc == boundary).Count();
+                skip = boundary == from ? skip + ties : ties;
+                from = boundary;
+            }
+
+            all.SequenceEqual(NewestFirst).ShouldBeTrue($"cap {cap}: {string.Join(",", all)}");
+        }
+    }
+
+    [Fact]
+    public async Task Reverse_read_with_a_null_source_spans_every_source()
+    {
+        using var src = await SeedAsync();
+        var r = await src.ReadEventsAsync(null, At(2.5), At(1.5), 0, reverse: true, Ct);
+        r.Events.Select(e => e.EventId).ShouldBe(["t4", "t3", "t2", "t1", "t0", "o0"]);
+    }
+}
@@ -41,6 +41,13 @@ public interface IConformanceBackend : IAsyncDisposable
    /// <param name="samples">The rows to store.</param>
    Task SeedAsync(string tag, params (DateTime T, HistorianValue V, uint Status)[] samples);

+    /// <summary>
+    ///     Writes historical events directly in the backend's event table (never through the alarm writer): event id,
+    ///     source and time as given (rounded to the storage resolution), severity 500, received time = event time.
+    /// </summary>
+    /// <param name="events">The events to store.</param>
+    Task SeedEventsAsync(params (DateTime T, string Id, string Source)[] events);
+
    /// <summary>
    ///     Live (non-deleted) rows for a tag, ascending by time, read directly from storage. The status is returned
    ///     VERBATIM as stored — there is no read-side clamp (an Empty row stored non-Bad reads back non-Bad here).
@@ -0,0 +1,123 @@
+using System.Net;
+using Google.Protobuf.WellKnownTypes;
+using Grpc.Core;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Server.Kestrel.Core;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
+
+/// <summary>
+///     Gitea #675: the Gateway backend's event-read bounds and direction, offline.
+///     <list type="bullet">
+///       <item>A reverse (newest-first) read is refused before anything is sent — the gateway streams ascending only.</item>
+///       <item>The open upper bound a start-only forward read uses (<c>DateTime.MaxValue</c>, UTC kind) survives the
+///         PUBLISHED client package's proto conversion — captured on the wire by an in-process gRPC server — and the
+///         Windows FILETIME encoding the gateway converts it to. The live AVEVA leg (what the historian does with a
+///         year-9999 bound) stays open: it needs the VPN.</item>
+///     </list>
+/// </summary>
+public sealed class GatewayEventBoundsTests
+{
+    private static readonly DateTime UtcMax = new(DateTime.MaxValue.Ticks, DateTimeKind.Utc);
+    private static CancellationToken Ct => TestContext.Current.CancellationToken;
+
+    [Fact]
+    public async Task A_reverse_events_read_is_refused_and_sends_nothing()
+    {
+        var fake = new FakeHistorianGatewayClient();
+        IHistorianDataSource ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
+
+        await Should.ThrowAsync<NotSupportedException>(() =>
+            ds.ReadEventsAsync("Pump1", DateTime.UtcNow, DateTime.UtcNow.AddHours(-1), 10, reverse: true, Ct));
+        fake.ReadEventsCallCount.ShouldBe(0);
+
+        // Forward through the same overload is the five-argument read, unchanged.
+        await ds.ReadEventsAsync("Pump1", DateTime.UtcNow.AddHours(-1), DateTime.UtcNow, 10, reverse: false, Ct);
+        fake.ReadEventsCallCount.ShouldBe(1);
+    }
+
+    [Fact]
+    public void UtcMax_round_trips_through_the_proto_Timestamp_and_FILETIME_encodings()
+    {
+        var ts = Timestamp.FromDateTime(UtcMax);
+        ts.ToDateTime().ShouldBe(UtcMax);
+        ts.ToDateTime().Kind.ShouldBe(DateTimeKind.Utc);
+
+        // The gateway hands its bounds to the historian as FILETIME (100 ns since 1601) — in range up to year 30828.
+        var fileTime = ts.ToDateTime().ToFileTimeUtc();
+        DateTime.FromFileTimeUtc(fileTime).ShouldBe(UtcMax);
+    }
+
+    [Fact]
+    public async Task A_start_only_bound_of_UtcMax_reaches_the_wire_intact_through_the_published_client()
+    {
+        var capture = new CapturingReadService();
+        await using var server = await StartServerAsync(capture);
+        using var adapter = HistorianGatewayClientAdapter.Create(
+            new ServerHistorianOptions
+            {
+                Enabled = true,
+                Endpoint = $"http://127.0.0.1:{server.Port}",
+                ApiKey = "histgw_test_key",
+                UseTls = false,
+            },
+            NullLoggerFactory.Instance);
+        IHistorianDataSource ds = new GatewayHistorianDataSource(adapter, NullLogger<GatewayHistorianDataSource>.Instance);
+
+        var start = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
+        var r = await ds.ReadEventsAsync("Pump1", start, UtcMax, 5, Ct);
+
+        r.Events.ShouldBeEmpty();
+        capture.Request.ShouldNotBeNull();
+        capture.Request!.StartTime.ToDateTime().ShouldBe(start);
+        capture.Request.EndTime.ToDateTime().ShouldBe(UtcMax);
+        DateTime.FromFileTimeUtc(capture.Request.EndTime.ToDateTime().ToFileTimeUtc()).ShouldBe(UtcMax);
+    }
+
+    private sealed class CapturingReadService : HistorianRead.HistorianReadBase
+    {
+        public ReadEventsRequest? Request { get; private set; }
+
+        public override Task ReadEvents(
+            ReadEventsRequest request, IServerStreamWriter<HistorianEvent> responseStream, ServerCallContext context)
+        {
+            Request = request;
+            return Task.CompletedTask;
+        }
+    }
+
+    private sealed class Server(WebApplication app, int port) : IAsyncDisposable
+    {
+        public int Port { get; } = port;
+
+        public async ValueTask DisposeAsync()
+        {
+            await app.StopAsync();
+            await app.DisposeAsync();
+        }
+    }
+
+    private static async Task<Server> StartServerAsync(CapturingReadService service)
+    {
+        var builder = WebApplication.CreateSlimBuilder();
+        builder.Logging.ClearProviders();
+        builder.WebHost.ConfigureKestrel(k =>
+            k.Listen(IPAddress.Loopback, 0, o => o.Protocols = HttpProtocols.Http2));
+        builder.Services.AddGrpc();
+        builder.Services.AddSingleton(service);
+        var app = builder.Build();
+        app.MapGrpcService<CapturingReadService>();
+        await app.StartAsync();
+        var port = new Uri(app.Urls.First()).Port;
+        return new Server(app, port);
+    }
+}
@@ -15,12 +15,18 @@
        <PackageReference Include="Microsoft.NET.Test.Sdk"/>
        <!-- ServiceCollection + BuildServiceProvider for the CreateDataSource factory test. -->
        <PackageReference Include="Microsoft.Extensions.DependencyInjection"/>
+        <!-- In-process gRPC server capturing the published client's ReadEvents request (#675 bounds pin). -->
+        <PackageReference Include="Grpc.AspNetCore"/>
        <PackageReference Include="xunit.runner.visualstudio">
            <PrivateAssets>all</PrivateAssets>
            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
        </PackageReference>
    </ItemGroup>

+    <ItemGroup>
+        <FrameworkReference Include="Microsoft.AspNetCore.App"/>
+    </ItemGroup>
+
    <ItemGroup>
        <ProjectReference Include="..\..\..\src\Drivers\ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway\ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.csproj"/>
    </ItemGroup>
@@ -60,3 +60,10 @@ public sealed class QuestDbModifiedValueConformance(QuestDbFixture fx) : Modifie
    /// <inheritdoc/>
    protected override Task<IConformanceBackend> CreateBackendAsync() => QuestDbConformanceBackend.CreateAsync(fx);
 }
+
+[Collection(QuestDbCollection.Name)]
+public sealed class QuestDbEventReadConformance(QuestDbFixture fx) : EventReadConformance
+{
+    /// <inheritdoc/>
+    protected override Task<IConformanceBackend> CreateBackendAsync() => QuestDbConformanceBackend.CreateAsync(fx);
+}
@@ -143,6 +143,27 @@ internal sealed class QuestDbConformanceBackend : IConformanceBackend
        await Wal.WaitAsync(Conn.SampleTable, CancellationToken.None);
    }

+    /// <inheritdoc/>
+    public async Task SeedEventsAsync(params (DateTime T, string Id, string Source)[] events)
+    {
+        await using (var c = await Conn.OpenAsync(CancellationToken.None))
+        {
+            foreach (var (t, id, source) in events)
+            {
+                await using var cmd = Conn.CreateCommand(c,
+                    $"INSERT INTO {Conn.EventTable} (ts, event_id, source, severity, message, received_utc) VALUES ($1, $2, $3, $4, $5, $6)");
+                QuestDbConnection.BindTimestamp(cmd, t);
+                QuestDbConnection.BindText(cmd, id);
+                QuestDbConnection.BindText(cmd, source);
+                cmd.Parameters.Add(new NpgsqlParameter { NpgsqlDbType = NpgsqlDbType.Integer, Value = 500 });
+                QuestDbConnection.BindText(cmd, null);
+                QuestDbConnection.BindTimestamp(cmd, t);
+                await cmd.ExecuteNonQueryAsync();
+            }
+        }
+        await Wal.WaitAsync(Conn.EventTable, CancellationToken.None);
+    }
+
    /// <summary>Runs a statement against this table set (test-only escape hatch for hand-made rows), then waits for WAL visibility of the sample table.</summary>
    /// <param name="sql">The statement; parameters are bound positionally by <paramref name="bind"/>.</param>
    /// <param name="bind">Binds the parameters.</param>
@@ -60,3 +60,10 @@ public sealed class SqlServerModifiedValueConformance(SqlHistorianFixture fx) :
    /// <inheritdoc/>
    protected override Task<IConformanceBackend> CreateBackendAsync() => SqlServerConformanceBackend.CreateAsync(fx);
 }
+
+[Collection(SqlHistorianCollection.Name)]
+public sealed class SqlServerEventReadConformance(SqlHistorianFixture fx) : EventReadConformance
+{
+    /// <inheritdoc/>
+    protected override Task<IConformanceBackend> CreateBackendAsync() => SqlServerConformanceBackend.CreateAsync(fx);
+}
@@ -40,6 +40,22 @@ internal sealed class SqlServerConformanceBackend : IConformanceBackend
    public async Task SeedAsync(string tag, params (DateTime T, HistorianValue V, uint Status)[] samples) =>
        await _db.SeedAsync(tag, samples);

+    /// <inheritdoc/>
+    public async Task SeedEventsAsync(params (DateTime T, string Id, string Source)[] events)
+    {
+        await using var c = await _db.Conn.OpenAsync(CancellationToken.None);
+        foreach (var (t, id, source) in events)
+        {
+            await using var cmd = _db.Conn.CreateCommand(c,
+                $"INSERT {_db.Conn.Q("Event")} (EventId, SourceName, AlarmId, AlarmName, AlarmTypeName, Severity, EventKind, Message, " +
+                "UserName, Comment, EventTimeUtc, ReceivedTimeUtc) VALUES (@id, @src, 'a', 'n', 'T', 500, 'Activated', NULL, NULL, NULL, @t, @t)");
+            cmd.Parameters.AddWithValue("@id", id);
+            cmd.Parameters.AddWithValue("@src", source);
+            SqlHistorianConnection.AddDateTime2(cmd, "@t", t);
+            await cmd.ExecuteNonQueryAsync();
+        }
+    }
+
    /// <inheritdoc/>
    public async Task<IReadOnlyList<(DateTime T, HistorianValue V, uint Status)>> ReadBackAsync(string tag)
    {