fix(historian): fail fast for missing tags and reduce gateway read cost

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayHistorianDataSource.cs

@@ -68,7 +68,8 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
     internal int MaxPreWindowRows => _maxPreWindowRows;
 
     /// <summary>A strict gateway's row ceiling, learned from its first refusal (#691.3).</summary>
-    private readonly GatewayRowCeiling _ceiling = new();
+    private readonly GatewayRowCeiling _ceiling;
+    private readonly GatewayTagPresenceCache _tags;
 
     private readonly object _healthLock = new();
     private DateTime? _lastSuccessUtc;
@@ -107,6 +108,8 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
         _logger = logger;
         _maxRawRows = maxRawRows;
         _time = timeProvider ?? TimeProvider.System;
+        _ceiling = new GatewayRowCeiling(_time);
+        _tags = new GatewayTagPresenceCache(_client, _time);
     }
 
     /// <inheritdoc />
@@ -126,6 +129,11 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
 
         try
         {
+            if (!await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
+            {
+                RecordOutcome(success: true, error: null);
+                return new HistoryReadResult([], ContinuationPoint: null);
+            }
             var samples = await ReadRawWindowAsync(fullReference, startUtc, endUtc, maxValuesPerNode, cancellationToken)
                 .ConfigureAwait(false);
             var snapshots = SampleMapper.ToSnapshots(samples);
@@ -351,6 +359,11 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
         if (plan.BucketCount == 0) return new HistoryReadResult([], ContinuationPoint: null);
         try
         {
+            if (!await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
+            {
+                RecordOutcome(success: true, error: null);
+                return new HistoryReadResult(plan.Assemble(new Dictionary<long, DataValueSnapshot>(), t => AggregateEvaluator.NoData(t, _time.GetUtcNow().UtcDateTime)), ContinuationPoint: null);
+            }
             var byBucket = new Dictionary<long, DataValueSnapshot>();
             foreach (var request in NativeAggregatePlan.Requests(plan))
             {
@@ -390,6 +403,11 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
         try
         {
             var now = _time.GetUtcNow().UtcDateTime;
+            if (plan.BucketCount > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
+            {
+                RecordOutcome(success: true, error: null);
+                return new HistoryReadResult(plan.Assemble(new Dictionary<long, DataValueSnapshot>(), t => AggregateEvaluator.NoData(t, now)), ContinuationPoint: null);
+            }
             var values = await RawAggregateReader
                 .ReadAsync(_client, _ceiling, fullReference, aggregate, plan, _maxRawRows, now, cancellationToken)
                 .ConfigureAwait(false);
@@ -460,6 +478,11 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
         try
         {
             var now = _time.GetUtcNow().UtcDateTime;
+            if (timestampsUtc.Count > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
+            {
+                RecordOutcome(success: true, error: null);
+                return MissingAtTime(timestampsUtc, now);
+            }
             var values = await GatewayAtTimeReader
                 .ReadAsync(_client, _ceiling, fullReference, timestampsUtc, now, cancellationToken, valueType)
                 .ConfigureAwait(false);
@@ -488,6 +511,12 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
     {
         try
         {
+            if (timestampsUtc.Count > 0 && !await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
+            {
+                RecordOutcome(success: true, error: null);
+                return MissingAtTime(timestampsUtc, _time.GetUtcNow().UtcDateTime);
+            }
+
             var samples = await _client
                 .ReadAtTimeAsync(fullReference, timestampsUtc, cancellationToken)
                 .ConfigureAwait(false);
@@ -669,6 +698,10 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
         return result;
     }
 
+    /// <summary>No-data values for a missing tag, stamped with the client's requested times.</summary>
+    private static HistoryReadResult MissingAtTime(IReadOnlyList<DateTime> times, DateTime now) =>
+        new(times.Select(t => AtTimeEvaluator.Evaluate(t, null, null, now, now)).ToArray(), ContinuationPoint: null);
+
     /// <summary>Classifies actual gRPC/package transport failures without treating refusals as outages (#697).</summary>
     private static bool IsTransportFailure(Exception ex) => ex switch
     {

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayReadLimits.cs

@@ -76,13 +76,15 @@ internal static class GatewayReadLimits
 ///         the read is re-issued at <c>N</c>; refused naming <c>C</c> again ⇒ the <c>C</c>-bounded result stands (no repeat).
 ///     </para>
 ///     <para>
-///         <b>Why no TTL.</b> A raise only matters to a read that needs more than <c>C</c>, and such a read re-probes anyway
-///         (rule 3), so a timer would only add refusals for reads that fit. The cost of the rule is one refused re-probe per
-///         genuinely over-ceiling read — a read that fails <c>BadTooManyOperations</c> whatever is done.
+///         <b>Re-probe timing (#696).</b> Raw and AtTime reads re-probe incomplete streams immediately. Per-sample
+///         aggregate reads opt into a 30-second cooldown after a refusal; a full stream during it remains a request
+///         limit, never a truncated answer. After the cooldown the next incomplete read detects a raised ceiling.
 ///     </para>
 /// </remarks>
-internal sealed class GatewayRowCeiling
+internal sealed class GatewayRowCeiling(TimeProvider? timeProvider = null)
 {
+    private readonly TimeProvider _time = timeProvider ?? TimeProvider.System;
+    private long _lastRefusalTicks;
     private int _learned; // 0 = unknown / not strict
 
     /// <summary>The learned ceiling (0 = none).</summary>
@@ -93,8 +95,9 @@ internal sealed class GatewayRowCeiling
     /// <param name="budget">The <c>max_values</c> the read wants.</param>
     /// <param name="pass">One drained read with the given budget.</param>
     /// <param name="needsMore">True when a pass with the given budget delivered it all without proving completeness.</param>
+    /// <param name="deferRecentReprobe">Aggregate-only opt-in: reuse a refusal for 30 seconds before re-probing an incomplete read.</param>
     /// <returns>The result and the budget it was read with.</returns>
-    public async Task<(T Result, int Budget)> ReadAsync<T>(int budget, Func<int, Task<T>> pass, Func<T, int, bool> needsMore)
+    public async Task<(T Result, int Budget)> ReadAsync<T>(int budget, Func<int, Task<T>> pass, Func<T, int, bool> needsMore, bool deferRecentReprobe = false)
     {
         var learned = Learned;
         if (learned <= 0 || learned >= budget) return await AttemptAsync(budget, pass).ConfigureAwait(false);
@@ -102,6 +105,10 @@ internal sealed class GatewayRowCeiling
         var bounded = await AttemptAsync(learned, pass).ConfigureAwait(false);
         if (bounded.Budget < learned || !needsMore(bounded.Result, bounded.Budget)) return bounded;
 
+        // #696: aggregate reads may reuse a recent refusal for 30 seconds. Raw paging keeps immediate re-probes.
+        if (deferRecentReprobe && _time.GetUtcNow().UtcTicks - Volatile.Read(ref _lastRefusalTicks) < TimeSpan.FromSeconds(30).Ticks)
+            return bounded;
+
         // Rule 3: full at a possibly stale ceiling — re-probe once at the full budget.
         int fresh;
         try
@@ -114,10 +121,16 @@ internal sealed class GatewayRowCeiling
         {
             // fall through (outside the catch)
         }
-        Volatile.Write(ref _learned, fresh);
+        Learn(fresh);
         return fresh > learned ? (await pass(fresh).ConfigureAwait(false), fresh) : bounded;
     }
 
+    private void Learn(int ceiling)
+    {
+        Volatile.Write(ref _lastRefusalTicks, _time.GetUtcNow().UtcTicks);
+        Volatile.Write(ref _learned, ceiling);
+    }
+
     private async Task<(T Result, int Budget)> AttemptAsync<T>(int budget, Func<int, Task<T>> pass)
     {
         int ceiling;
@@ -129,7 +142,7 @@ internal sealed class GatewayRowCeiling
         {
             // fall through to the retry (outside the catch, so a second refusal is not nested in this one)
         }
-        Volatile.Write(ref _learned, ceiling);
+        Learn(ceiling);
         return (await pass(ceiling).ConfigureAwait(false), ceiling);
     }
 }

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/GatewayTagPresenceCache.cs

@@ -0,0 +1,36 @@
+using Grpc.Core;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
+
+/// <summary>Bounded metadata cache (#653). Missing tags expire quickly; failures never become missing tags.</summary>
+internal sealed class GatewayTagPresenceCache(IHistorianGatewayClient client, TimeProvider time)
+{
+    internal const int Capacity = 4096;
+    private readonly object _sync = new();
+    private readonly Dictionary<string, (bool Exists, DateTimeOffset Expires)> _entries = new(StringComparer.Ordinal);
+
+    public async Task<bool> ExistsAsync(string tag, CancellationToken ct)
+    {
+        ct.ThrowIfCancellationRequested();
+        lock (_sync)
+            if (_entries.TryGetValue(tag, out var entry) && time.GetUtcNow() < entry.Expires) return entry.Exists;
+
+        bool exists;
+        try { exists = await client.TagExistsAsync(tag, ct).ConfigureAwait(false); }
+        catch (Exception ex) when (IsNotFound(ex)) { exists = false; }
+        ct.ThrowIfCancellationRequested();
+        lock (_sync)
+        {
+            if (!_entries.ContainsKey(tag) && _entries.Count >= Capacity) _entries.Remove(_entries.Keys.First());
+            _entries[tag] = (exists, time.GetUtcNow().AddSeconds(exists ? 60 : 5));
+        }
+        return exists;
+    }
+
+    private static bool IsNotFound(Exception ex)
+    {
+        for (Exception? e = ex; e is not null; e = e.InnerException)
+            if (e is RpcException { StatusCode: StatusCode.NotFound }) return true;
+        return false;
+    }
+}

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/HistorianGatewayClientAdapter.cs

@@ -81,6 +81,17 @@ public sealed class HistorianGatewayClientAdapter : IHistorianGatewayClient, IDi
         return new HistorianGatewayClientAdapter(HistorianGatewayClient.Create(clientOptions));
     }
 
+    /// <inheritdoc />
+    public async Task<bool> TagExistsAsync(string tag, CancellationToken ct)
+    {
+        // #653: older gateway SDKs misparse a missing GetTagInfo response as Internal. A catalog query proves
+        // absence without guessing from an error. Even a glob-like tag is checked by exact name, never by row count.
+        var found = false;
+        await foreach (var name in _inner.BrowseTagNamesAsync(tag, ct).ConfigureAwait(false))
+            found |= string.Equals(name, tag, StringComparison.OrdinalIgnoreCase);
+        return found; // drain first: a trailing stream fault must propagate and must not become a cache entry
+    }
+
     /// <inheritdoc />
     public IAsyncEnumerable<HistorianSample> ReadRawAsync(
         string tag, DateTime startUtc, DateTime endUtc, int maxValues, CancellationToken ct) =>

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/IHistorianGatewayClient.cs

@@ -11,6 +11,12 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
 /// </summary>
 public interface IHistorianGatewayClient : IAsyncDisposable
 {
+    /// <summary>Checks the tag catalog; false means explicitly absent, while failures propagate.</summary>
+    /// <param name="tag">Historian tag name.</param>
+    /// <param name="ct">Cancellation token.</param>
+    /// <returns>Whether the historian contains the tag.</returns>
+    Task<bool> TagExistsAsync(string tag, CancellationToken ct);
+
     /// <summary>Streams raw historian samples for a tag over a time window.</summary>
     /// <param name="tag">The historian tag name to read.</param>
     /// <param name="startUtc">Inclusive start of the requested time window (UTC).</param>

src/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway/RawAggregateReader.cs

@@ -188,7 +188,7 @@ internal static class RawAggregateReader
         var requested = MaxValuesFor(maxWindowRows);
         var (pass, maxValues) = await ceiling.ReadAsync(requested,
             budget => ReadWindowAsync(client, tag, aggregate, plan, requestStart, budget, maxWindowRows, ct),
-            (r, budget) => !r.OverCap && r.Received >= budget).ConfigureAwait(false);
+            (r, budget) => !r.OverCap && r.Received >= budget, deferRecentReprobe: true).ConfigureAwait(false);
         var (buckets, earliest, latest, received, overCap) = pass;
         if (overCap) throw LimitExceeded(tag, maxWindowRows);
         if (received >= maxValues) throw BudgetExhausted(tag, maxWindowRows, requested, maxValues);

src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ReverseRawReader.cs

@@ -126,7 +126,7 @@ internal static class ReverseRawReader
                         (budget < Math.Max(1, maxTieCluster)
                             ? "the backend's row ceiling (HistorianGateway Historian:ReadLimits:MaxValuesCeiling)."
                             : "ServerHistorian:MaxTieClusterOverfetch.")),
-                    ct)
+                    ct, useDiscoveryBudgetForGaps: limit > 0)
                 .ConfigureAwait(false);
             return new ReverseRawPage(rows, next.WidthTicks) { Hint = next };
         }

src/Server/ZB.MOM.WW.OtOpcUa.OpcUaServer/ReverseWalk.cs

@@ -123,13 +123,14 @@ internal static class ReverseWalk
     /// <param name="hint">What the previous page of the same read carried (default = none).</param>
     /// <param name="tooManyAtOneTimestamp">Builds the exception for a timestamp holding more than M rows.</param>
     /// <param name="ct">Cancellation.</param>
+    /// <param name="useDiscoveryBudgetForGaps">Use ceiling-sized discovery and retain truncated remainder rows (#696, strict raw backends only).</param>
     /// <returns>At most <paramref name="want"/> rows, newest first, and the hint for the next page.</returns>
     public static async Task<(List<T> Rows, ReverseWalkHint Next)> WalkAsync<T>(
         Func<DateTime, DateTime, int, CancellationToken, Task<ForwardSegmentRead<T>>> read,
         Func<T, long> ticksOf,
         IComparer<T>? tieOrder,
         DateTime hi, DateTime lo, uint want, int tieBudget, long resolutionTicks, DateTime nowUtc,
-        ReverseWalkHint hint, Func<DateTime, Exception> tooManyAtOneTimestamp, CancellationToken ct)
+        ReverseWalkHint hint, Func<DateTime, Exception> tooManyAtOneTimestamp, CancellationToken ct, bool useDiscoveryBudgetForGaps = false)
     {
         var resolution = Math.Max(1L, resolutionTicks);
         var granularity = Math.Max(Alignment, resolution);
@@ -206,7 +207,8 @@ internal static class ReverseWalk
             var segLo = segHi - stopTicks <= w ? stopTicks : segHi - w;
             var start = Utc(AlignDown(segLo, granularity));
             var end = Utc(AlignUp(segHi + 1, granularity));
-            var segBudget = discovering && !probe ? discoveryBudget : pageBudget;
+            var segBudget = (discovering && !probe) || (useDiscoveryBudgetForGaps && (probe || known > 0))
+                ? discoveryBudget : pageBudget;
             var cap = segBudget + 2;
             IReadOnlyList<T> rows;
             var narrowed = false;
@@ -220,10 +222,14 @@ internal static class ReverseWalk
 
                 if (probe)
                 {
-                    // Deep history below: the probe saw only its oldest rows. Carry on walking from the same segHi.
+                    // #696: strict-ceiling discovery retains the rows/density instead of discarding the remainder probe.
                     probed = true;
-                    narrowed = true;
-                    break;
+                    if (!useDiscoveryBudgetForGaps)
+                    {
+                        narrowed = true;
+                        break;
+                    }
+                    probe = false;
                 }
 
                 var leading = 0;
@@ -284,7 +290,7 @@ internal static class ReverseWalk
                     // Rows exist at lastInner; what lies above it is unknown. Aim at about half the page budget at the
                     // density the truncated rows showed (their span, not the segment's — it may open on an empty stretch).
                     var aimed = inner > 1 && lastInner > firstInner
-                        ? (long)Math.Min((double)(lastInner - firstInner) * (pageBudget / 2.0) / (inner - 1), maxWidth)
+                        ? (long)Math.Min((double)(lastInner - firstInner) * ((useDiscoveryBudgetForGaps ? segBudget : pageBudget) / 2.0) / (inner - 1), maxWidth)
                         : resolution;
                     known = lastInner;
                     knownWidth = Math.Max(resolution, aimed);