fix(historian): respect gateway ceilings and history value contracts

2026-10-13 03:51:04 -04:00
parent 66a2b0c3e9
commit 97668d26d3
15 changed files with 474 additions and 29 deletions
@@ -128,6 +128,35 @@ public sealed class GatewayAtTimeTests
        return r.Samples;
    }

+    [Fact]
+    public async Task A_future_time_on_a_live_tag_finds_the_prior_sample_without_walking_the_empty_future()
+    {
+        var store = new AvevaStore(Enumerable.Range(0, 7201).Select(i => (Now.AddSeconds(i - 7200), (double)i, false)));
+        var fake = store.Client();
+        var result = (await AtTime(fake, Now.AddHours(1))).Single();
+        result.Value.ShouldBe(7200d);
+        result.StatusCode.ShouldBe(UncertainInterpolated);
+        fake.ReadRawCallCount.ShouldBeLessThanOrEqualTo(4);
+    }
+
+    [Fact]
+    public async Task A_future_time_still_uses_a_stored_future_prior_sample()
+    {
+        var fake = new AvevaStore([(Now.AddSeconds(-1), 1d, false), (Now.AddMinutes(30), 2d, false)]).Client();
+        var result = (await AtTime(fake, Now.AddHours(1))).Single();
+        result.Value.ShouldBe(2d);
+        result.StatusCode.ShouldBe(UncertainInterpolated);
+    }
+
+    [Fact]
+    public async Task A_full_future_probe_walks_to_the_latest_stored_future_sample()
+    {
+        var fake = new AvevaStore(Enumerable.Range(0, 7201).Select(i => (Now.AddSeconds(i), (double)i, false))).Client();
+        var result = (await AtTime(fake, Now.AddHours(3))).Single();
+        result.Value.ShouldBe(7200d);
+        result.StatusCode.ShouldBe(UncertainInterpolated);
+    }
+
    // ---- the four live findings ------------------------------------------------------------------------------------

    [Fact] // live: 19:00:03 (a stored NULL) ⇒ 107 Good
@@ -420,6 +420,54 @@ public sealed class GatewayCeilingPagingTests : IDisposable
        finally { await host.DisposeAsync(); }
    }

+    [Theory]
+    [InlineData(1709, 1366)] // #695 reverse-walk trace
+    [InlineData(1369, 1292)] // #695 bounds trace
+    public async Task A_warm_read_uses_the_last_room_below_the_ceiling_for_pre_window_ties(int ceiling, int cluster)
+    {
+        var start = At(1000);
+        var stored = Enumerable.Range(0, cluster).Select(i => (start.AddMilliseconds(-0.5), (double)i))
+            .Append((start, 9000d)).Append((start.AddSeconds(1), 9001d)).ToList();
+        var fake = Gateway(stored, ceiling);
+        using var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
+        await ds.ReadRawAsync("T", At(0), At(1), 5000, Ct); // learn without crowding
+        fake.RawCalls.Clear();
+        var result = await ds.ReadRawAsync("T", start, start.AddSeconds(2), 1, Ct);
+        result.Samples.Select(s => s.Value).ShouldBe([9000d]);
+        fake.RawCalls.ShouldAllBe(c => c.MaxValues <= ceiling);
+    }
+
+    [Fact]
+    public async Task A_warm_reverse_segment_can_shrink_again_when_the_pre_window_cluster_grows()
+    {
+        var start = At(1000);
+        var stored = Enumerable.Range(0, 1366).Select(i => (start.AddMilliseconds(-0.5), (double)i))
+            .Concat(Enumerable.Range(0, 500).Select(i => (start.AddMilliseconds(i), 9000d + i))).ToList();
+        var fake = Gateway(stored, 1709);
+        using var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
+        await ds.ReadRawAsync("T", At(0), At(1), 5000, Ct);
+        var page = await ReverseRawReader.ReadAsync(ds, "T", start.AddMilliseconds(499), start.AddTicks(-1),
+            249, 65536, At(2000), new ReverseWalkHint(499 * TimeSpan.TicksPerMillisecond), Ct);
+        page.Samples.Select(s => (double)s.Value!).ShouldBe(Enumerable.Range(9251, 249).Reverse().Select(i => (double)i));
+    }
+
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public async Task Gateway_transport_faults_surface_as_BadNoCommunication(bool deadline)
+    {
+        var fake = new FakeHistorianGatewayClient { ThrowOnRead = deadline
+            ? new ZB.MOM.WW.HistorianGateway.Client.HistorianGatewayException("deadline", new Grpc.Core.RpcException(new Grpc.Core.Status(Grpc.Core.StatusCode.DeadlineExceeded, "deadline")))
+            : new ZB.MOM.WW.HistorianGateway.Client.HistorianGatewayUnavailableException("unavailable") };
+        var (host, nm, nodeId) = await BootAsync(fake);
+        try
+        {
+            var r = ReadRaw(nm, nodeId, At(0), At(10), 10, null);
+            r.Error.StatusCode.Code.ShouldBe(StatusCodes.BadNoCommunication);
+        }
+        finally { await host.DisposeAsync(); }
+    }
+
    // ---- helpers ---------------------------------------------------------------------------------------------------------

    private async Task<(OpcUaApplicationHost Host, OtOpcUaNodeManager Nm, NodeId NodeId)> BootAsync(
@@ -11,6 +11,33 @@ public sealed class QuestDbHistorianProviderTests
        ConnectionString = "Host=localhost;Port=18812;Username=admin;Password=quest;Database=qdb;ServerCompatibilityMode=NoTypeLoading",
    });

+    [Theory]
+    [InlineData("Raw")]
+    [InlineData("AtTime")]
+    [InlineData("Processed")]
+    [InlineData("Events")]
+    [InlineData("Modified")]
+    public async Task A_connection_failure_is_exposed_as_a_transport_failure(string kind)
+    {
+        using var provider = new QuestDbHistorianProvider(new QuestDbHistorianOptions {
+            ConnectionString = "Host=127.0.0.1;Port=1;Username=admin;Password=test;Database=qdb;Timeout=1;Pooling=False;ServerCompatibilityMode=NoTypeLoading", AutoCreateSchema = false
+        });
+        using var source = provider.CreateDataSource(new NullSp());
+        var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
+        Func<Task> read = kind switch
+        {
+            "Raw" => async () => await source.ReadRawAsync("T", t, t.AddHours(1), 10, TestContext.Current.CancellationToken),
+            "AtTime" => async () => await source.ReadAtTimeAsync("T", [t], TestContext.Current.CancellationToken),
+            "Processed" => async () => await source.ReadProcessedAsync("T", t, t.AddHours(1), TimeSpan.FromSeconds(10), HistoryAggregateType.Average, TestContext.Current.CancellationToken),
+            "Events" => async () => await source.ReadEventsAsync("T", t, t.AddHours(1), 10, TestContext.Current.CancellationToken),
+            _ => async () => await source.ReadModifiedAsync("T", t, t.AddHours(1), 10, null, TestContext.Current.CancellationToken)
+        };
+        var error = await Should.ThrowAsync<System.IO.IOException>(read);
+        error.InnerException.ShouldNotBeNull();
+        // A cached bootstrap outage must remain retryable on subsequent reads as well.
+        (await Should.ThrowAsync<System.IO.IOException>(read)).InnerException.ShouldNotBeNull();
+    }
+
    [Fact]
    public void Name_and_all_six_kinds()
    {
@@ -11,6 +11,33 @@ public sealed class SqlServerHistorianProviderTests
        ConnectionString = "Server=localhost;User Id=sa;Password=x;TrustServerCertificate=True",
    });

+    [Theory]
+    [InlineData("Raw")]
+    [InlineData("AtTime")]
+    [InlineData("Processed")]
+    [InlineData("Events")]
+    [InlineData("Modified")]
+    public async Task A_connection_failure_is_exposed_as_a_transport_failure(string kind)
+    {
+        var provider = new SqlServerHistorianProvider(new SqlServerHistorianOptions {
+            ConnectionString = "Server=127.0.0.1,1;User Id=sa;Password=test;TrustServerCertificate=True;Connect Timeout=1;ConnectRetryCount=0;Pooling=False", AutoCreateSchema = false
+        });
+        using var source = provider.CreateDataSource(new NullSp());
+        var t = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
+        Func<Task> read = kind switch
+        {
+            "Raw" => async () => await source.ReadRawAsync("T", t, t.AddHours(1), 10, TestContext.Current.CancellationToken),
+            "AtTime" => async () => await source.ReadAtTimeAsync("T", [t], TestContext.Current.CancellationToken),
+            "Processed" => async () => await source.ReadProcessedAsync("T", t, t.AddHours(1), TimeSpan.FromSeconds(10), HistoryAggregateType.Average, TestContext.Current.CancellationToken),
+            "Events" => async () => await source.ReadEventsAsync("T", t, t.AddHours(1), 10, TestContext.Current.CancellationToken),
+            _ => async () => await source.ReadModifiedAsync("T", t, t.AddHours(1), 10, null, TestContext.Current.CancellationToken)
+        };
+        var error = await Should.ThrowAsync<System.IO.IOException>(read);
+        error.InnerException.ShouldNotBeNull();
+        // A cached bootstrap outage must remain retryable on subsequent reads as well.
+        (await Should.ThrowAsync<System.IO.IOException>(read)).InnerException.ShouldNotBeNull();
+    }
+
    [Fact]
    public void Name_and_all_six_kinds()
    {
@@ -295,12 +295,14 @@ public sealed class NodeManagerHistoryReadEventsTests : IDisposable
    /// HistoryRead call. The fake source MUST be invoked (proving we reached the bridge) and threw;
    /// the production catch now sets BOTH errors and results explicitly, so both are asserted here
    /// rather than relying on the SDK base pre-seeding results[i].</summary>
-    [Fact]
-    public async Task Events_backend_throw_yields_bad_status_and_does_not_escape()
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public async Task Events_backend_throw_yields_bad_status_and_does_not_escape(bool communication)
    {
        var (host, server) = await BootAsync();
        var nm = server.NodeManager!;
-        var fake = new RecordingHistorianDataSource { ThrowOnRead = true };
+        var fake = new RecordingHistorianDataSource { ThrowOnRead = !communication, ReadException = communication ? new System.IO.IOException("lost connection") : null };
        nm.HistorianDataSource = fake;

        const string equipmentId = "eq-boom";
@@ -321,10 +323,10 @@ public sealed class NodeManagerHistoryReadEventsTests : IDisposable

        // Authoritative per-node signal: errors[0] must be Bad.
        ServiceResult.IsBad(errors[0]).ShouldBeTrue();
-        errors[0].StatusCode.Code.ShouldBe(StatusCodes.BadHistoryOperationUnsupported);
+        errors[0].StatusCode.Code.ShouldBe(communication ? StatusCodes.BadNoCommunication : StatusCodes.BadHistoryOperationUnsupported);
        // The production catch now sets results[0] explicitly — assert it directly (not relying on base seeding).
        results[0].ShouldNotBeNull();
-        results[0].StatusCode.Code.ShouldBe(StatusCodes.BadHistoryOperationUnsupported);
+        results[0].StatusCode.Code.ShouldBe(communication ? StatusCodes.BadNoCommunication : StatusCodes.BadHistoryOperationUnsupported);
        // The source WAS entered (proving the override reached the bridge and the throw was swallowed).
        fake.EventsEntered.ShouldBeTrue();
        // ThrowOnRead fires before EventsCalled/LastSourceName are set — that's the throw path.
@@ -380,6 +382,7 @@ public sealed class NodeManagerHistoryReadEventsTests : IDisposable
    private sealed class RecordingHistorianDataSource : IHistorianDataSource
    {
        public bool ThrowOnRead { get; init; }
+        public Exception? ReadException { get; init; }
        public HistoricalEventsResult EventsResult { get; set; } =
            new(Array.Empty<HistoricalEvent>(), null);

@@ -411,6 +414,7 @@ public sealed class NodeManagerHistoryReadEventsTests : IDisposable
            CancellationToken cancellationToken)
        {
            EventsEntered = true;
+            if (ReadException is not null) throw ReadException;
            if (ThrowOnRead) throw new InvalidOperationException("backend boom for events");
            EventsCalled = true;
            LastSourceName = sourceName;
@@ -25,6 +25,34 @@ public sealed class NodeManagerHistoryReadModifiedTests : IDisposable
    private static readonly DateTime Epoch = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
    private readonly string _pkiRoot = Path.Combine(Path.GetTempPath(), $"otopcua-modified-{Guid.NewGuid():N}");

+    [Fact]
+    public async Task Modified_values_use_the_scalar_node_type_and_preserve_the_audit_information()
+    {
+        var (host, nm, node, _) = await SetupAsync([Mod(1, 1, 2d, HistoryModificationType.Replace, "alice", Epoch)], "Float");
+        try
+        {
+            var r = Read(nm, node, Epoch, Epoch.AddHours(1), 10, null);
+            r.Data!.DataValues.Single().Value.ShouldBeOfType<float>().ShouldBe(2f);
+            r.Data.ModificationInfos.Single().UserName.ShouldBe("alice");
+            r.Data.ModificationInfos.Single().UpdateType.ShouldBe(HistoryUpdateType.Replace);
+        }
+        finally { await host.DisposeAsync(); }
+    }
+
+    [Fact]
+    public async Task A_modified_read_of_an_unreachable_backend_answers_BadNoCommunication()
+    {
+        var (host, nm, node, src) = await SetupAsync([]);
+        try
+        {
+            src.Throw = new System.IO.IOException("connection lost");
+            var r = Read(nm, node, Epoch, Epoch.AddHours(1), 10, null);
+            r.Error.StatusCode.Code.ShouldBe(StatusCodes.BadNoCommunication);
+            r.Result.StatusCode.Code.ShouldBe(StatusCodes.BadNoCommunication);
+        }
+        finally { await host.DisposeAsync(); }
+    }
+
    [Fact]
    public async Task A_modified_read_dispatches_to_the_trail_and_projects_modification_infos()
    {
@@ -289,7 +317,7 @@ public sealed class NodeManagerHistoryReadModifiedTests : IDisposable
    }

    private async Task<(OpcUaApplicationHost Host, OtOpcUaNodeManager Nm, NodeId Node, SqlLikeTrail Src)> SetupAsync(
-        List<HistoryModifiedValue> trail)
+        List<HistoryModifiedValue> trail, string dataType = "Double")
    {
        var started = await OpcUaHostStart.StartAsync(
            port => new OpcUaApplicationHost(
@@ -307,7 +335,7 @@ public sealed class NodeManagerHistoryReadModifiedTests : IDisposable
        nm.HistoryContinuationStore = new InMemoryHistoryContinuationStore();
        var src = new SqlLikeTrail(trail);
        nm.HistorianDataSource = src;
-        nm.EnsureVariable("eq-1/mod", null, "Mod", "Double", false, historianTagname: Tag, realm: AddressSpaceRealm.Uns);
+        nm.EnsureVariable("eq-1/mod", null, "Mod", dataType, false, historianTagname: Tag, realm: AddressSpaceRealm.Uns);
        return (started.Host, nm, nm.TryGetVariable("eq-1/mod")!.NodeId, src);
    }

@@ -786,6 +786,132 @@ public sealed class NodeManagerHistoryReadTests : IDisposable
        return (results, errors);
    }

+    [Theory]
+    [InlineData("Raw")]
+    [InlineData("AtTime")]
+    [InlineData("Processed")]
+    public async Task An_unreachable_backend_answers_a_communication_status(string kind)
+    {
+        var (host, server) = await BootAsync();
+        try
+        {
+            var nm = server.NodeManager!;
+            nm.HistorianDataSource = new RecordingHistorianDataSource { ReadException = new System.IO.IOException("connection lost") };
+            nm.EnsureVariable("eq/comm", null, "Comm", "Double", false, historianTagname: "T", realm: AddressSpaceRealm.Uns);
+            var t = DateTime.UtcNow.AddMinutes(-1);
+            HistoryReadDetails details = kind switch
+            {
+                "Raw" => new ReadRawModifiedDetails { IsReadModified = false, ReturnBounds = false, StartTime = t, EndTime = t.AddSeconds(10) },
+                "AtTime" => new ReadAtTimeDetails { ReqTimes = new DateTimeCollection { t } },
+                _ => new ReadProcessedDetails { StartTime = t, EndTime = t.AddSeconds(10), ProcessingInterval = 1000,
+                    AggregateType = new NodeIdCollection { ObjectIds.AggregateFunction_Average } }
+            };
+            var (results, errors) = InvokeHistoryRead(server, nm, details, nm.TryGetVariable("eq/comm")!.NodeId);
+            errors[0].StatusCode.Code.ShouldBe(StatusCodes.BadNoCommunication);
+            results[0].StatusCode.Code.ShouldBe(StatusCodes.BadNoCommunication);
+        }
+        finally { await host.DisposeAsync(); }
+    }
+
+    [Theory]
+    [InlineData("Int32", "Raw")]
+    [InlineData("Boolean", "Raw")]
+    [InlineData("Float", "Raw")]
+    [InlineData("Int32", "AtTime")]
+    [InlineData("Boolean", "AtTime")]
+    [InlineData("Float", "AtTime")]
+    public async Task History_values_use_the_nodes_scalar_type(string dataType, string kind)
+    {
+        var (host, server) = await BootAsync();
+        try
+        {
+            var nm = server.NodeManager!;
+            var t = DateTime.UtcNow.AddMinutes(-1);
+            var value = new HistorianRead([new DataValueSnapshot(1d, StatusCodes.Good, t, t)], null);
+            nm.HistorianDataSource = new RecordingHistorianDataSource { RawResult = value, AtTimeResult = value };
+            nm.EnsureVariable("eq/typed", null, "Typed", dataType, false, historianTagname: "T", realm: AddressSpaceRealm.Uns);
+            HistoryReadDetails details = kind == "Raw"
+                ? new ReadRawModifiedDetails { IsReadModified = false, ReturnBounds = false, StartTime = t, EndTime = t.AddSeconds(10) }
+                : new ReadAtTimeDetails { ReqTimes = new DateTimeCollection { t } };
+            var (results, errors) = InvokeHistoryRead(server, nm, details, nm.TryGetVariable("eq/typed")!.NodeId);
+            errors[0].StatusCode.Code.ShouldBe(StatusCodes.Good);
+            var actual = ((HistoryData)ExtensionObject.ToEncodeable(results[0].HistoryData)).DataValues.Single();
+            if (dataType == "Int32") actual.Value.ShouldBeOfType<int>().ShouldBe(1);
+            if (dataType == "Boolean") actual.Value.ShouldBeOfType<bool>().ShouldBeTrue();
+            if (dataType == "Float") actual.Value.ShouldBeOfType<float>().ShouldBe(1f);
+            actual.SourceTimestamp.ShouldBe(t);
+            actual.StatusCode.Code.ShouldBe(StatusCodes.Good);
+        }
+        finally { await host.DisposeAsync(); }
+    }
+
+    [Theory]
+    [InlineData("Int16", 40000d)]
+    [InlineData("DateTime", 1d)]
+    public async Task An_unrepresentable_historical_value_is_null_with_BadTypeMismatch(string type, double input)
+    {
+        var (host, server) = await BootAsync();
+        try
+        {
+            var nm = server.NodeManager!;
+            var t = DateTime.UtcNow.AddMinutes(-1);
+            nm.EnsureVariable("eq/mismatch", null, "Mismatch", type, false, historianTagname: "T", realm: AddressSpaceRealm.Uns);
+            nm.HistorianDataSource = new RecordingHistorianDataSource { RawResult = new HistorianRead([new(input, StatusCodes.Good, t, t)], null) };
+            var (results, _) = InvokeHistoryRead(server, nm, new ReadRawModifiedDetails { IsReadModified = false, ReturnBounds = false,
+                StartTime = t, EndTime = t.AddSeconds(10) }, nm.TryGetVariable("eq/mismatch")!.NodeId);
+            var value = ((HistoryData)ExtensionObject.ToEncodeable(results[0].HistoryData)).DataValues.Single();
+            value.Value.ShouldBeNull();
+            value.StatusCode.Code.ShouldBe(StatusCodes.BadTypeMismatch);
+            value.SourceTimestamp.ShouldBe(t);
+            value.ServerTimestamp.ShouldBe(t);
+        }
+        finally { await host.DisposeAsync(); }
+    }
+
+    [Fact]
+    public async Task Array_and_null_history_values_preserve_quality_and_timestamps()
+    {
+        var (host, server) = await BootAsync();
+        try
+        {
+            var nm = server.NodeManager!;
+            var t = DateTime.UtcNow.AddMinutes(-1);
+            var serverTime = t.AddSeconds(1);
+            nm.EnsureVariable("eq/array", null, "Array", "Int32", false, historianTagname: "T", realm: AddressSpaceRealm.Raw, isArray: true, arrayLength: 2);
+            nm.HistorianDataSource = new RecordingHistorianDataSource { RawResult = new HistorianRead([
+                new(new int[] { 1, 2 }, StatusCodes.Uncertain, t, serverTime),
+                new(null, StatusCodes.BadNoCommunication, t.AddSeconds(1), serverTime)], null) };
+            var (results, _) = InvokeHistoryRead(server, nm, new ReadRawModifiedDetails { IsReadModified = false, ReturnBounds = false,
+                StartTime = t, EndTime = t.AddSeconds(10) }, nm.TryGetVariable("eq/array", AddressSpaceRealm.Raw)!.NodeId);
+            var values = ((HistoryData)ExtensionObject.ToEncodeable(results[0].HistoryData)).DataValues;
+            values[0].Value.ShouldBeOfType<int[]>().ShouldBe([1, 2]);
+            values[0].StatusCode.Code.ShouldBe(StatusCodes.Uncertain);
+            values[0].SourceTimestamp.ShouldBe(t);
+            values[0].ServerTimestamp.ShouldBe(serverTime);
+            values[1].Value.ShouldBeNull();
+            values[1].StatusCode.Code.ShouldBe(StatusCodes.BadNoCommunication);
+        }
+        finally { await host.DisposeAsync(); }
+    }
+
+    [Fact]
+    public async Task Processed_Count_keeps_its_aggregate_type_on_a_Boolean_node()
+    {
+        var (host, server) = await BootAsync();
+        try
+        {
+            var nm = server.NodeManager!;
+            var t = DateTime.UtcNow.AddMinutes(-1);
+            nm.EnsureVariable("eq/count", null, "Count", "Boolean", false, historianTagname: "T", realm: AddressSpaceRealm.Uns);
+            nm.HistorianDataSource = new RecordingHistorianDataSource { ProcessedResult = new HistorianRead([new(3d, StatusCodes.Good, t, t)], null) };
+            var (results, _) = InvokeHistoryRead(server, nm, new ReadProcessedDetails { StartTime = t, EndTime = t.AddSeconds(10),
+                ProcessingInterval = 1000, AggregateType = new NodeIdCollection { ObjectIds.AggregateFunction_Count } }, nm.TryGetVariable("eq/count")!.NodeId);
+            var value = ((HistoryData)ExtensionObject.ToEncodeable(results[0].HistoryData)).DataValues.Single();
+            value.Value.ShouldBeOfType<double>().ShouldBe(3d);
+        }
+        finally { await host.DisposeAsync(); }
+    }
+
    private static HistorianRead Empty() => new(Array.Empty<DataValueSnapshot>(), null);

    /// <summary>A recording fake historian source — captures the last call's kind + arguments and returns
@@ -793,6 +919,7 @@ public sealed class NodeManagerHistoryReadTests : IDisposable
    private sealed class RecordingHistorianDataSource : IHistorianDataSource
    {
        public bool ThrowOnRead { get; init; }
+        public Exception? ReadException { get; init; }
        public bool AggregateNotSupported { get; init; }
        public bool LimitExceeded { get; init; }
        public IReadOnlySet<HistoryAggregateType>? Supported { get; init; } = HistoryAggregateTypes.All;
@@ -814,6 +941,7 @@ public sealed class NodeManagerHistoryReadTests : IDisposable
            string fullReference, DateTime startUtc, DateTime endUtc, uint maxValuesPerNode,
            CancellationToken cancellationToken)
        {
+            if (ReadException is not null) throw ReadException;
            if (ThrowOnRead) throw new InvalidOperationException("backend boom");
            LastCall = "Raw";
            LastTagname = fullReference;
@@ -827,6 +955,7 @@ public sealed class NodeManagerHistoryReadTests : IDisposable
            string fullReference, DateTime startUtc, DateTime endUtc, TimeSpan interval,
            HistoryAggregateType aggregate, CancellationToken cancellationToken)
        {
+            if (ReadException is not null) throw ReadException;
            if (ThrowOnRead) throw new InvalidOperationException("backend boom");
            if (AggregateNotSupported) throw new HistoryAggregateNotSupportedException(aggregate);
            if (LimitExceeded) throw new HistoryReadLimitExceededException(4, "too many raw rows");
@@ -842,6 +971,7 @@ public sealed class NodeManagerHistoryReadTests : IDisposable
        public Task<HistorianRead> ReadAtTimeAsync(
            string fullReference, IReadOnlyList<DateTime> timestampsUtc, CancellationToken cancellationToken)
        {
+            if (ReadException is not null) throw ReadException;
            if (ThrowOnRead) throw new InvalidOperationException("backend boom");
            LastCall = "AtTime";
            LastTagname = fullReference;