fix(historian): #691.3 strict-ceiling gateway: re-ask once at the named ceiling, never answer a truncated read

2026-10-12 16:13:14 -04:00
parent 39552e15b9
commit dbffb6bdb4
7 changed files with 391 additions and 56 deletions
@@ -144,8 +144,12 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
    ///         gateway reject <c>max_values &lt;= 0</c>, #686). A cap asks for <c>cap + </c>
    ///         <see cref="RawAggregateReader.PreWindowAllowance"/> rows — NOT clamped at the gateway's default ceiling (#688
    ///         clamps only the per-sample aggregate budget): AVEVA always emits the boundary row, so a clamped budget would make
-    ///         every page size within the allowance of the ceiling come back short. A strict gateway refuses an over-ceiling
-    ///         budget instead (⇒ <see cref="HistoryReadLimitExceededException"/>, <c>BadTooManyOperations</c>).
+    ///         every page size within the allowance of the ceiling come back short. A strict gateway
+    ///         (<c>AllowStreamingBeyondCeiling=false</c>) refuses an over-ceiling budget and names its ceiling; the read is
+    ///         then re-issued ONCE with the ceiling as budget (#691.3, <see cref="GatewayReadLimits"/>) — so an uncapped read
+    ///         of a small window is served — and a ceiling-bounded read that is not provably complete (it delivered the
+    ///         whole ceiling without reaching the window end, or a page short of its cap) fails
+    ///         <see cref="HistoryReadLimitExceededException"/> (<c>BadTooManyOperations</c>), never a truncated answer.
    ///     </para>
    ///     <para>
    ///         <b>Completeness.</b> A full page must hold exactly the cap of stored samples — a short page ends the client's
@@ -166,22 +170,42 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
        // The gateway seam caps with an int; OPC UA hands us a uint, so clamp to int range.
        var cap = (int)Math.Min(maxValuesPerNode, int.MaxValue);
        if (cap == 0)
-            return (await ReadRawPassAsync(fullReference, startUtc, endUtc, cap, int.MaxValue, cancellationToken)
-                .ConfigureAwait(false)).Samples;
+        {
+            var all = await PassAsync(int.MaxValue).ConfigureAwait(false);
+            if (all.Result.Complete) return all.Result.Samples;
+            // Only reachable on a strict gateway (#691.3): the ceiling re-read delivered the whole ceiling without reaching
+            // the window end — the window may hold more, and "all values" is never answered from a truncated read.
+            throw new HistoryReadLimitExceededException(cap,
+                $"The raw read of '{fullReference}' holds more than the HistorianGateway's row ceiling ({all.Budget}, its " +
+                "Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); set NumValuesPerNode to page, " +
+                "or narrow the window.");
+        }

-        var first = await ReadRawPassAsync(fullReference, startUtc, endUtc, cap,
-            (int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance), cancellationToken).ConfigureAwait(false);
-        if (first.Complete) return first.Samples;
+        var asked = (int)Math.Min(int.MaxValue, (long)cap + RawAggregateReader.PreWindowAllowance);
+        var first = await PassAsync(asked).ConfigureAwait(false);
+        if (first.Result.Complete) return first.Result.Samples;
+
+        // A strict gateway's ceiling already bounded that read (#691.3): a larger budget would only be refused again.
+        if (first.Budget < asked)
+            throw new HistoryReadLimitExceededException(cap,
+                $"The raw read of '{fullReference}' could not be read completely within the HistorianGateway's row ceiling " +
+                $"({first.Budget}, its Historian:ReadLimits:MaxValuesCeiling with AllowStreamingBeyondCeiling=false); lower " +
+                "NumValuesPerNode or narrow the window.");

        // The margin crowded the window out: one bounded re-read with room for the pre-window rows already seen.
-        var second = await ReadRawPassAsync(fullReference, startUtc, endUtc, cap,
-            (int)Math.Min(int.MaxValue, (long)cap + first.PreWindowRows + RawAggregateReader.PreWindowAllowance),
-            cancellationToken).ConfigureAwait(false);
-        if (second.Complete) return second.Samples;
+        var second = await PassAsync(
+            (int)Math.Min(int.MaxValue, (long)cap + first.Result.PreWindowRows + RawAggregateReader.PreWindowAllowance))
+            .ConfigureAwait(false);
+        if (second.Result.Complete) return second.Result.Samples;

        throw new HistoryReadLimitExceededException(cap,
-            $"The raw read of '{fullReference}' could not be read completely: more than {second.PreWindowRows} rows lie in " +
+            $"The raw read of '{fullReference}' could not be read completely: more than {second.Result.PreWindowRows} rows lie in " +
            $"the {RawAggregateReader.BoundaryMargin.TotalMilliseconds} ms before the window and exhausted the row budget.");
+
+        // One drained pass; a strict gateway's refusal of the budget is retried once at the ceiling it names (#691.3).
+        Task<((List<HistorianSample> Samples, int PreWindowRows, bool Complete) Result, int Budget)> PassAsync(int budget) =>
+            GatewayReadLimits.ReadWithCeilingRetryAsync(budget,
+                b => ReadRawPassAsync(fullReference, startUtc, endUtc, cap, b, cancellationToken));
    }

    /// <summary>One forward raw read of a window, drained, with its pre-window row count and whether it is complete.</summary>
@@ -206,7 +230,9 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
            else if (sample.Timestamp is not null) reachedEnd = true;
        }

-        var complete = received < budget || samples.Count >= cap || reachedEnd || cap == 0;
+        // cap == 0 ("all values") is complete only when the stream proves it: short of the budget (always, unless a strict
+        // gateway's ceiling became the budget — #691.3) or past the window end.
+        var complete = received < budget || reachedEnd || (cap > 0 && samples.Count >= cap);
        return (samples, preWindow, complete);
    }

@@ -315,14 +341,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
    ///     True when <paramref name="ex"/> (or an inner exception — the client package wraps the <see cref="RpcException"/>)
    ///     is the gateway's <c>InvalidArgument</c> "max_values exceeds the server ceiling" rejection.
    /// </summary>
-    internal static bool IsMaxValuesCeilingRejection(Exception ex)
-    {
-        for (var e = ex; e is not null; e = e.InnerException)
-            if (e is RpcException { StatusCode: StatusCode.InvalidArgument } rpc
-                && rpc.Status.Detail.Contains("max_values exceeds the server ceiling", StringComparison.Ordinal))
-                return true;
-        return false;
-    }
+    internal static bool IsMaxValuesCeilingRejection(Exception ex) => GatewayReadLimits.IsCeilingRejection(ex);

    /// <inheritdoc />
    public async Task<HistoryReadResult> ReadAtTimeAsync(
@@ -0,0 +1,84 @@
+using System.Globalization;
+using System.Text.RegularExpressions;
+using Grpc.Core;
+
+namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
+
+/// <summary>
+///     The HistorianGateway's raw-read row ceiling (<c>Historian:ReadLimits:MaxValuesCeiling</c>, default 1,000,000) as seen
+///     from this side of the wire. With <c>AllowStreamingBeyondCeiling=false</c> (a "strict" gateway) the gateway refuses
+///     any <c>ReadRaw</c> whose <c>max_values</c> exceeds the ceiling with <c>InvalidArgument</c> — BEFORE it streams a row
+///     — and names the ceiling in the refusal (<c>"max_values exceeds the server ceiling of {n}; …"</c>). The gateway has
+///     no RPC that reports the ceiling, so the refusal is how it is learned (#691.3).
+/// </summary>
+/// <remarks>
+///     A read whose budget the gateway refuses is re-issued ONCE with the ceiling as its budget
+///     (<see cref="ReadWithCeilingRetryAsync{T}"/>); the caller then treats a stream that delivered the whole ceiling as
+///     possibly truncated (never answered from). Nothing is cached: a re-configured gateway is honoured on the next read,
+///     and a refusal costs one round-trip with no historian work behind it (it is validated before a session is acquired).
+/// </remarks>
+internal static class GatewayReadLimits
+{
+    private const string CeilingRejectionText = "max_values exceeds the server ceiling";
+
+    private static readonly Regex CeilingNumber = new(@"max_values exceeds the server ceiling of (\d+)",
+        RegexOptions.CultureInvariant | RegexOptions.Compiled);
+
+    /// <summary>
+    ///     True when <paramref name="ex"/> (or an inner exception — the client package wraps the <see cref="RpcException"/>)
+    ///     is the gateway's <c>InvalidArgument</c> "max_values exceeds the server ceiling" rejection.
+    /// </summary>
+    /// <param name="ex">The exception.</param>
+    /// <returns>Whether it is the ceiling rejection.</returns>
+    public static bool IsCeilingRejection(Exception ex) => Rejection(ex) is not null;
+
+    /// <summary>
+    ///     The ceiling a ceiling rejection names, or — when <paramref name="ex"/> is a ceiling rejection that does not
+    ///     carry a parseable number — the gateway's default (<see cref="RawAggregateReader.GatewayDefaultMaxValuesCeiling"/>).
+    /// </summary>
+    /// <param name="ex">The exception.</param>
+    /// <param name="ceiling">The ceiling (&gt; 0) when this returns true.</param>
+    /// <returns>Whether <paramref name="ex"/> is a ceiling rejection.</returns>
+    public static bool TryGetCeiling(Exception ex, out int ceiling)
+    {
+        ceiling = 0;
+        if (Rejection(ex) is not { } rpc) return false;
+        var m = CeilingNumber.Match(rpc.Status.Detail);
+        ceiling = m.Success && int.TryParse(m.Groups[1].Value, NumberStyles.None, CultureInfo.InvariantCulture, out var n) && n > 0
+            ? n
+            : RawAggregateReader.GatewayDefaultMaxValuesCeiling;
+        return true;
+    }
+
+    /// <summary>
+    ///     Runs <paramref name="pass"/> with <paramref name="budget"/>; if a strict gateway refuses that budget, runs it once
+    ///     more with the ceiling it named — only when that ceiling is smaller than the refused budget (otherwise the refusal
+    ///     is rethrown). The refusal arrives before any row, so the first pass has consumed nothing.
+    /// </summary>
+    /// <typeparam name="T">The pass result.</typeparam>
+    /// <param name="budget">The <c>max_values</c> to ask for first.</param>
+    /// <param name="pass">One drained read with the given budget.</param>
+    /// <returns>The pass result and the budget it actually used.</returns>
+    public static async Task<(T Result, int Budget)> ReadWithCeilingRetryAsync<T>(int budget, Func<int, Task<T>> pass)
+    {
+        int ceiling;
+        try
+        {
+            return (await pass(budget).ConfigureAwait(false), budget);
+        }
+        catch (Exception ex) when (TryGetCeiling(ex, out ceiling) && ceiling < budget)
+        {
+            // fall through to the retry (outside the catch, so a second refusal is not nested in this one)
+        }
+        return (await pass(ceiling).ConfigureAwait(false), ceiling);
+    }
+
+    private static RpcException? Rejection(Exception ex)
+    {
+        for (var e = ex; e is not null; e = e.InnerException)
+            if (e is RpcException { StatusCode: StatusCode.InvalidArgument } rpc
+                && rpc.Status.Detail.Contains(CeilingRejectionText, StringComparison.Ordinal))
+                return rpc;
+        return null;
+    }
+}
@@ -49,7 +49,10 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
 ///         requested row budget — has been drained. The gateway is asked for at most <c>cap + 2 + </c>
 ///         <see cref="PreWindowAllowance"/> rows, clamped at <see cref="GatewayDefaultMaxValuesCeiling"/> while the cap is at
 ///         or under it (#688, <see cref="ClampToGatewayCeiling"/>); a stream that delivers all of them cannot be proven
-///         complete and fails the same way rather than answering from a truncated window.
+///         complete and fails the same way rather than answering from a truncated window. A strict gateway whose ceiling is
+///         below that budget (a ceiling under 1,000,000, or an unbounded cap) refuses it naming its ceiling; the window is
+///         then re-read once with the ceiling as budget (#691.3, <see cref="GatewayReadLimits"/>), and the same
+///         "delivered the whole budget ⇒ cannot be proven complete" rule applies.
 ///     </para>
 ///     <para>
 ///         <b>Values and status.</b> A sample's value is its numeric value (Double), else its string value, else Empty; its
@@ -178,8 +181,43 @@ internal static class RawAggregateReader
        if (plan.BucketCount == 0) return [];
        var needsRange = AggregateEvaluator.NeedsDataRange(aggregate, plan);
        var requestStart = RequestStartFor(plan.StartUtc);
-        var maxValues = MaxValuesFor(maxWindowRows);

+        // A strict gateway that refuses the budget is re-asked once at the ceiling it names (#691.3; e.g. a ceiling below
+        // the 1,000,000 #688 clamps at, or an unbounded cap) — and a stream that fills that ceiling is never evaluated.
+        var (pass, maxValues) = await GatewayReadLimits.ReadWithCeilingRetryAsync(MaxValuesFor(maxWindowRows),
+            budget => ReadWindowAsync(client, tag, aggregate, plan, requestStart, budget, maxWindowRows, ct)).ConfigureAwait(false);
+        var (buckets, earliest, latest, received, overCap) = pass;
+        if (overCap || received >= maxValues)
+            throw LimitExceeded(tag, maxWindowRows);
+
+        if (needsRange)
+        {
+            // Earlier data: no non-Bad pre-window row ⇒ ask explicitly (normally only when the tag's history really starts
+            // inside the window).
+            var earlierSeen = earliest is { } e && (e < plan.StartUtc || (plan.Reversed && e == plan.StartUtc));
+            if (!earlierSeen && plan.StartUtc.AddTicks(-1) > ProbeFloor)
+                if (await FirstSampleAsync(client, tag, ProbeFloor, plan.StartUtc.AddTicks(-1), ct).ConfigureAwait(false) is { } before)
+                    earliest = before;
+
+            // Later data: the main read covers a sample exactly at the window end (forward: that is already later data);
+            // otherwise a small separate read from the end to now + horizon.
+            var laterSeen = latest is { } l && (plan.Reversed ? l > plan.EndUtc : l >= plan.EndUtc);
+            var horizon = DateTime.MaxValue - now > LookAheadHorizon
+                ? DateTime.SpecifyKind(now + LookAheadHorizon, DateTimeKind.Utc)
+                : DateTime.MaxValue;
+            if (!laterSeen && horizon > plan.EndUtc)
+                if (await FirstSampleAsync(client, tag, plan.EndUtc, horizon, ct).ConfigureAwait(false) is { } after)
+                    latest = after;
+        }
+
+        return AggregateEvaluator.EvaluateAll(aggregate, plan, buckets.Rows(), earliest, latest, now);
+    }
+
+    /// <summary>The window read of <see cref="ReadAsync"/> with one row budget, drained.</summary>
+    private static async Task<(RawBucketAccumulator Buckets, DateTime? Earliest, DateTime? Latest, long Received, bool OverCap)>
+        ReadWindowAsync(IHistorianGatewayClient client, string tag, HistoryAggregateType aggregate, BucketPlan plan,
+            DateTime requestStart, int maxValues, int maxWindowRows, CancellationToken ct)
+    {
        var buckets = new RawBucketAccumulator(aggregate == HistoryAggregateType.MaximumActualTime);
        DateTime? earliest = null, latest = null;
        long windowRows = 0, received = 0;
@@ -214,30 +252,7 @@ internal static class RawAggregateReader
                if (t >= plan.StartUtc && (latest is null || t > latest)) latest = t;
            }
        }
-        if (overCap || (maxWindowRows > 0 && received >= maxValues))
-            throw LimitExceeded(tag, maxWindowRows);
-
-        if (needsRange)
-        {
-            // Earlier data: no non-Bad pre-window row ⇒ ask explicitly (normally only when the tag's history really starts
-            // inside the window).
-            var earlierSeen = earliest is { } e && (e < plan.StartUtc || (plan.Reversed && e == plan.StartUtc));
-            if (!earlierSeen && plan.StartUtc.AddTicks(-1) > ProbeFloor)
-                if (await FirstSampleAsync(client, tag, ProbeFloor, plan.StartUtc.AddTicks(-1), ct).ConfigureAwait(false) is { } before)
-                    earliest = before;
-
-            // Later data: the main read covers a sample exactly at the window end (forward: that is already later data);
-            // otherwise a small separate read from the end to now + horizon.
-            var laterSeen = latest is { } l && (plan.Reversed ? l > plan.EndUtc : l >= plan.EndUtc);
-            var horizon = DateTime.MaxValue - now > LookAheadHorizon
-                ? DateTime.SpecifyKind(now + LookAheadHorizon, DateTimeKind.Utc)
-                : DateTime.MaxValue;
-            if (!laterSeen && horizon > plan.EndUtc)
-                if (await FirstSampleAsync(client, tag, plan.EndUtc, horizon, ct).ConfigureAwait(false) is { } after)
-                    latest = after;
-        }
-
-        return AggregateEvaluator.EvaluateAll(aggregate, plan, buckets.Rows(), earliest, latest, now);
+        return (buckets, earliest, latest, received, overCap);
    }

    /// <summary>