fix(historian): fail fast for missing tags and reduce gateway read cost

2026-10-13 04:31:52 -04:00
parent 97668d26d3
commit 5897764024
15 changed files with 439 additions and 18 deletions
@@ -12,6 +12,17 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
 /// </summary>
 public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
 {
+    public bool TagExistsResult = true;
+    public int TagExistsCallCount;
+    public Exception? TagExistsThrows;
+    public Task<bool> TagExistsAsync(string tag, CancellationToken ct)
+    {
+        ct.ThrowIfCancellationRequested();
+        TagExistsCallCount++;
+        if (TagExistsThrows is { } error) throw error;
+        return Task.FromResult(TagExistsResult);
+    }
+
    // ---- ReadRaw -------------------------------------------------------------------------------
    public IReadOnlyList<HistorianSample> RawSamples = Array.Empty<HistorianSample>();
    public string? LastReadRawTag;
@@ -385,6 +385,21 @@ public sealed class GatewayCeilingPagingTests : IDisposable
        finally { await host.DisposeAsync(); }
    }

+    [Fact]
+    public async Task A_stopped_dense_tag_under_a_strict_ceiling_is_found_with_bounded_discovery_cost()
+    {
+        var stored = PerSecond(7200);
+        var fake = Gateway(stored, Ceiling);
+        var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance);
+        await ds.ReadRawAsync("T", At(0), At(1), 5000, Ct);
+        fake.RawCalls.Clear();
+        var page = await ReverseRawReader.ReadAsync(ds, "T", At(7200).AddDays(1), DateTime.UnixEpoch,
+            3, 65536, At(7200).AddDays(1), 0, Ct);
+        page.Samples.Select(s => s.Value).ShouldBe(new object?[] { 7199d, 7198d, 7197d });
+        fake.RawCalls.Count.ShouldBeLessThanOrEqualTo(12);
+        EveryRequestFits(fake);
+    }
+
    // ---- F3: rows crowding the look-back before a page start ---------------------------------------------------------------

    [Theory]
@@ -0,0 +1,126 @@
+using Google.Protobuf.WellKnownTypes;
+using Grpc.Core;
+using Microsoft.Extensions.Logging.Abstractions;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
+using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
+
+public sealed class GatewayMissingTagTests
+{
+    private static CancellationToken Ct => TestContext.Current.CancellationToken;
+    private static readonly DateTime Now = new(2026, 10, 13, 8, 0, 0, DateTimeKind.Utc);
+    private sealed class Clock : TimeProvider
+    {
+        public DateTime Current = Now;
+        public override DateTimeOffset GetUtcNow() => new(Current);
+    }
+    private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient fake, Clock? clock = null, bool native = false) =>
+        new(fake, NullLogger<GatewayHistorianDataSource>.Instance, timeProvider: clock, atTimeFromRaw: !native);
+
+    [Theory]
+    [InlineData("Raw")]
+    [InlineData("AtTime")]
+    [InlineData("NativeAtTime")]
+    [InlineData("Count")]
+    [InlineData("TimeAverage")]
+    public async Task A_missing_tag_returns_no_data_without_a_history_RPC(string kind)
+    {
+        var fake = new FakeHistorianGatewayClient { TagExistsResult = false };
+        var ds = Source(fake, native: kind == "NativeAtTime");
+        var result = kind switch
+        {
+            "Raw" => await ds.ReadRawAsync("Missing", Now.AddMinutes(-1), Now, 10, Ct),
+            "AtTime" or "NativeAtTime" => await ds.ReadAtTimeAsync("Missing", [Now], Ct),
+            _ => await ds.ReadProcessedAsync("Missing", Now.AddMinutes(-1), Now, TimeSpan.FromMinutes(1),
+                kind == "Count" ? HistoryAggregateType.Count : HistoryAggregateType.TimeAverage, Ct),
+        };
+        if (kind == "Raw") result.Samples.ShouldBeEmpty();
+        else
+        {
+            result.Samples.Count.ShouldBe(1);
+            result.Samples[0].Value.ShouldBeNull();
+            (result.Samples[0].StatusCode & 0xFFFF0000u).ShouldBe(0x809B0000u);
+            result.Samples[0].SourceTimestampUtc.ShouldBe(kind.Contains("AtTime") ? Now : Now.AddMinutes(-1));
+        }
+        fake.TagExistsCallCount.ShouldBe(1);
+        (fake.ReadRawCallCount + fake.ReadAggregateCallCount + fake.ReadAtTimeCallCount).ShouldBe(0);
+        ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
+    }
+
+    [Fact]
+    public async Task Negative_cache_expires_so_a_newly_provisioned_tag_can_be_read()
+    {
+        var fake = new FakeHistorianGatewayClient { TagExistsResult = false };
+        var clock = new Clock(); var ds = Source(fake, clock);
+        await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
+        fake.TagExistsResult = true;
+        fake.RawSamples = [new HistorianSample { Timestamp = Timestamp.FromDateTime(Now.AddSeconds(-1)), NumericValue = 42, OpcQuality = 192 }];
+        (await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct)).Samples.ShouldBeEmpty();
+        fake.TagExistsCallCount.ShouldBe(1);
+        clock.Current = Now.AddSeconds(5);
+        (await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct)).Samples.Single().Value.ShouldBe(42d);
+        fake.TagExistsCallCount.ShouldBe(2);
+    }
+
+    [Fact]
+    public async Task Positive_cache_avoids_checks_per_page_and_expires_after_a_minute()
+    {
+        var fake = new FakeHistorianGatewayClient(); var clock = new Clock(); var ds = Source(fake, clock);
+        await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
+        await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
+        fake.TagExistsCallCount.ShouldBe(1);
+        clock.Current = Now.AddSeconds(60);
+        await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
+        fake.TagExistsCallCount.ShouldBe(2);
+    }
+
+    [Fact]
+    public async Task An_explicit_nested_NotFound_is_cached_as_missing()
+    {
+        var fake = new FakeHistorianGatewayClient { TagExistsThrows = new InvalidOperationException("wrapper",
+            new RpcException(new Status(StatusCode.NotFound, "absent"))) };
+        var ds = Source(fake);
+        (await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct)).Samples.ShouldBeEmpty();
+        (await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct)).Samples.ShouldBeEmpty();
+        fake.TagExistsCallCount.ShouldBe(1);
+        fake.ReadRawCallCount.ShouldBe(0);
+    }
+
+    [Fact]
+    public async Task The_presence_cache_is_bounded_and_a_displaced_entry_is_checked_again()
+    {
+        var fake = new FakeHistorianGatewayClient { TagExistsResult = false }; var ds = Source(fake);
+        for (var i = 0; i <= GatewayTagPresenceCache.Capacity; i++)
+            await ds.ReadRawAsync($"T{i}", Now.AddMinutes(-1), Now, 10, Ct);
+        await ds.ReadRawAsync("T0", Now.AddMinutes(-1), Now, 10, Ct);
+        fake.TagExistsCallCount.ShouldBe(GatewayTagPresenceCache.Capacity + 2);
+    }
+
+    [Fact]
+    public async Task Caller_cancellation_is_not_cached_as_missing()
+    {
+        var fake = new FakeHistorianGatewayClient { TagExistsThrows = new OperationCanceledException() }; var ds = Source(fake);
+        await Should.ThrowAsync<OperationCanceledException>(() => ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct));
+        fake.TagExistsThrows = null;
+        await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
+        fake.TagExistsCallCount.ShouldBe(2);
+    }
+
+    [Theory]
+    [InlineData(StatusCode.Unavailable)]
+    [InlineData(StatusCode.PermissionDenied)]
+    [InlineData(StatusCode.Internal)]
+    public async Task A_metadata_fault_is_not_cached_as_a_missing_tag(StatusCode code)
+    {
+        var fake = new FakeHistorianGatewayClient { TagExistsThrows = new RpcException(new Status(code, "failure")) };
+        var ds = Source(fake);
+        await Should.ThrowAsync<Exception>(() => ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct));
+        fake.TagExistsThrows = null;
+        await ds.ReadRawAsync("T", Now.AddMinutes(-1), Now, 10, Ct);
+        fake.TagExistsCallCount.ShouldBe(2);
+        fake.ReadRawCallCount.ShouldBe(1);
+    }
+}
@@ -57,6 +57,30 @@ public sealed class GatewayStrictCeilingTests
        public override DateTimeOffset GetUtcNow() => new(now);
    }

+    private sealed class MovingClock : TimeProvider
+    {
+        public DateTime Current = Now;
+        public override DateTimeOffset GetUtcNow() => new(Current);
+    }
+
+    [Fact]
+    public async Task Aggregate_ceiling_re_probes_wait_thirty_seconds_then_notice_a_raised_limit()
+    {
+        var fake = Store(100, ceiling: 50);
+        var clock = new MovingClock();
+        var ds = new GatewayHistorianDataSource(fake, NullLogger<GatewayHistorianDataSource>.Instance, timeProvider: clock);
+        Task<HistoryReadResult> Read() => ds.ReadProcessedAsync("T", At(0), At(100), TimeSpan.FromSeconds(100), HistoryAggregateType.Count, Ct);
+        await Should.ThrowAsync<HistoryReadLimitExceededException>(Read);
+        await Should.ThrowAsync<HistoryReadLimitExceededException>(Read);
+        Refusals(fake).ShouldBe(1);
+        fake.MaxValuesCeiling = 200;
+        await Should.ThrowAsync<HistoryReadLimitExceededException>(Read);
+        Refusals(fake).ShouldBe(1);
+        clock.Current = Now.AddSeconds(30);
+        (await Read()).Samples.Single().Value.ShouldBe(100);
+        ds.MaxRawValuesPerRead.ShouldBe(200 - RawAggregateReader.PreWindowAllowance);
+    }
+
    // ---- raw, uncapped ---------------------------------------------------------------------------------------------------

    [Fact]
@@ -0,0 +1,90 @@
+using System.Net;
+using Grpc.Core;
+using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Server.Kestrel.Core;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+using Shouldly;
+using Xunit;
+using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
+using ZB.MOM.WW.OtOpcUa.Runtime.Historian;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
+
+public sealed class GatewayTagCatalogWireTests
+{
+    private static CancellationToken Ct => TestContext.Current.CancellationToken;
+
+    [Theory]
+    [InlineData("Pump1", true)]
+    [InlineData("Missing", false)]
+    [InlineData("Pump*", false)]
+    public async Task Presence_is_proved_by_exact_names_without_using_the_broken_metadata_parser(string tag, bool expected)
+    {
+        var service = new TagsService();
+        await using var server = await StartAsync(service);
+        using var adapter = Create(server.Port);
+        (await adapter.TagExistsAsync(tag, Ct)).ShouldBe(expected);
+        service.Filter.ShouldBe(tag);
+        service.MetadataCalls.ShouldBe(0);
+        service.Completed.ShouldBe(1);
+    }
+
+    [Fact]
+    public async Task A_catalog_stream_fault_after_a_matching_name_is_not_cached_as_presence()
+    {
+        var service = new TagsService { FailAfterRow = true };
+        await using var server = await StartAsync(service);
+        using var adapter = Create(server.Port);
+        var source = new GatewayHistorianDataSource(adapter, NullLogger<GatewayHistorianDataSource>.Instance);
+        var now = DateTime.UtcNow;
+        await Should.ThrowAsync<Exception>(() => source.ReadRawAsync("Pump1", now.AddMinutes(-1), now, 10, Ct));
+        service.FailAfterRow = false; service.Names = [];
+        (await source.ReadRawAsync("Pump1", now.AddMinutes(-1), now, 10, Ct)).Samples.ShouldBeEmpty();
+        service.BrowseCalls.ShouldBe(2);
+    }
+
+    private static HistorianGatewayClientAdapter Create(int port) => HistorianGatewayClientAdapter.Create(
+        new ServerHistorianOptions { Endpoint = $"http://127.0.0.1:{port}", ApiKey = "histgw_test_key", UseTls = false },
+        NullLoggerFactory.Instance);
+
+    private sealed class TagsService : HistorianTags.HistorianTagsBase
+    {
+        public int MetadataCalls, BrowseCalls, Completed;
+        public string? Filter;
+        public bool FailAfterRow;
+        public string[] Names = ["Pump1", "Pump2"];
+        public override Task<GetTagMetadataReply> GetTagMetadata(GetTagMetadataRequest request, ServerCallContext context)
+        {
+            MetadataCalls++;
+            throw new RpcException(new Status(StatusCode.Internal, "GetTagInfo response ended unexpectedly."));
+        }
+        public override async Task BrowseTagNames(BrowseTagNamesRequest request, IServerStreamWriter<TagName> stream, ServerCallContext context)
+        {
+            BrowseCalls++; Filter = request.Filter;
+            foreach (var name in Names)
+            {
+                await stream.WriteAsync(new TagName { Name = name }, context.CancellationToken);
+                if (FailAfterRow) throw new RpcException(new Status(StatusCode.Internal, "catalog failure"));
+            }
+            Completed++;
+        }
+    }
+
+    private sealed class Server(WebApplication app, int port) : IAsyncDisposable
+    {
+        public int Port => port;
+        public async ValueTask DisposeAsync() { await app.StopAsync(); await app.DisposeAsync(); }
+    }
+    private static async Task<Server> StartAsync(TagsService service)
+    {
+        var builder = WebApplication.CreateSlimBuilder();
+        builder.Logging.ClearProviders();
+        builder.WebHost.ConfigureKestrel(k => k.Listen(IPAddress.Loopback, 0, o => o.Protocols = HttpProtocols.Http2));
+        builder.Services.AddGrpc(); builder.Services.AddSingleton(service);
+        var app = builder.Build(); app.MapGrpcService<TagsService>(); await app.StartAsync();
+        return new Server(app, new Uri(app.Urls.First()).Port);
+    }
+}