fix(historian): drain gateway event streams past the cap; cancellation is not a fault (lmxopcua#746)

A capped Gateway HistoryReadEvents read broke out of the gRPC stream as soon as the page was full. Abandoning a gateway stream cancels the RPC, which the gateway treats as a connection fault: it evicts its pooled historian session, so the next read reconnects and concurrent reads on that session fail. The read now keeps consuming the stream past the cap without collecting it, bounded by ServerHistorian:Gateway:EventDrainBudget (default 10000 rows; 0 = the old abandon-at-cap). Rows of a foreign source never count toward the budget before the "more events" signal is seen, so an abandoned stream never answers a page as complete. A paged-vs-unpaged fuzz (random tie series, foreign rows, caps 1..500, budgets 0..10000) pins the page contents and the drain; it found that abandon-before-signal case. ReadEventsAsync and the native AtTime arm now rethrow a caller cancellation instead of recording it as a backend failure. Rebased from the unpushed remed/lane-H2 branch (originally authored 2026-10-22 in session_01Ub9hGmYLQvn9XETzw7CMsT) onto current master. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-10-22 16:26:57 -04:00
parent c86c7cbf35
commit 24609ececa
6 changed files with 431 additions and 11 deletions
@@ -5,6 +5,7 @@ using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
 using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
 using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
 using ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common;
+using ZB.MOM.WW.OtOpcUa.Runtime.Historian;

 namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;

@@ -73,6 +74,23 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
    /// </summary>
    private readonly int _maxRawValuesPerRead;

+    /// <summary>The default of <c>eventDrainBudget</c>: <c>ServerHistorian:Gateway:EventDrainBudget</c>'s default.</summary>
+    public const int DefaultEventDrainBudget = GatewayHistorianSettings.DefaultEventDrainBudget;
+
+    /// <summary>
+    ///     Stream rows an events read drains past its cap before it abandons the gateway stream; see
+    ///     <see cref="ReadEventsAsync(string?, DateTime, DateTime, int, CancellationToken)"/>.
+    /// </summary>
+    private readonly int _eventDrainBudget;
+
+    private long _eventStreamsAbandoned;
+
+    /// <summary>The bound events drain budget — exposed so the factory wiring is testable.</summary>
+    internal int EventDrainBudget => _eventDrainBudget;
+
+    /// <summary>Events reads that abandoned their gateway stream past the drain budget (lifetime) — test/diagnostic.</summary>
+    internal long EventStreamsAbandoned => Interlocked.Read(ref _eventStreamsAbandoned);
+
    /// <summary>A strict gateway's row ceiling, learned from its first refusal.</summary>
    private readonly GatewayRowCeiling _ceiling;
    private readonly GatewayTagPresenceCache _tags;
@@ -109,10 +127,16 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
    ///     for cap + 1 rows and is refused when the window holds more, never truncated. &lt;= 0 = unbounded (the pre-#736
    ///     behaviour: an uncapped read is sent as <see cref="int.MaxValue"/>).
    /// </param>
+    /// <param name="eventDrainBudget">
+    ///     Stream rows a capped events read consumes (uncollected) past its cap before it abandons the gateway stream
+    ///     (<c>ServerHistorian:Gateway:EventDrainBudget</c>). Negative counts as 0 = abandon at the cap.
+    /// </param>
    public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger,
        int maxRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, TimeProvider? timeProvider = null,
-        bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows, int maxRawValuesPerRead = 0)
+        bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows, int maxRawValuesPerRead = 0,
+        int eventDrainBudget = DefaultEventDrainBudget)
    {
+        _eventDrainBudget = Math.Max(0, eventDrainBudget);
        _maxPreWindowRows = Math.Max(1, maxPreWindowRows);
        _maxRawValuesPerRead = Math.Max(0, maxRawValuesPerRead);
        _atTimeFromRaw = atTimeFromRaw;
@@ -556,6 +580,10 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
            RecordOutcome(success: true, error: null);
            return new HistoryReadResult(aligned, ContinuationPoint: null);
        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            throw; // caller-requested, not a backend fault
+        }
        catch (Exception ex)
        {
            RecordReadFailure(ex);
@@ -609,6 +637,13 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
            : ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken);

    /// <inheritdoc />
+    // The gateway's ReadEvents has no max-events field (the 0.3.0 wire contract), so the cap is applied here. The stream
+    // is DRAINED past the cap — counted, never collected — rather than broken out of: abandoning a gateway stream cancels
+    // the RPC, which the gateway classifies as a connection fault and answers by evicting its pooled historian session
+    // (the next read of any kind reconnects; concurrent reads on that session fail). The drain is bounded by the
+    // EventDrainBudget (ServerHistorian:Gateway:EventDrainBudget): past cap + budget events the stream is abandoned after
+    // all (a window far larger than the page — streaming all of it for every page would cost O(window²/N)), logged at
+    // Debug and counted in EventStreamsAbandoned. 0 = abandon at the cap (the previous behaviour).
    public async Task<HistoricalEventsResult> ReadEventsAsync(
        string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
        CancellationToken cancellationToken)
@@ -619,11 +654,34 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
            // Bounded initial capacity: an uncapped read arrives as int.MaxValue ("Array dimensions exceeded").
            var collected = new List<HistoricalEvent>(hasCap ? Math.Min(maxEvents, 1024) : 0);
            var truncated = false;
+            long pastCap = 0; // stream rows consumed after the cap filled (matching or not)
+            var abandoned = false;

            await foreach (var wireEvent in _client
                .ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken)
                .ConfigureAwait(false))
            {
+                if (hasCap && collected.Count == maxEvents)
+                {
+                    // The page is full: drain the rest of the stream without collecting it, within the budget.
+                    // One more MATCHING event is the "more events exist" signal; a foreign source's row is not, and
+                    // until the signal is seen no row counts toward the budget — abandoning before it would answer
+                    // "complete" for a page that may not be (and a spurious continuation point on a stretch of foreign
+                    // rows would make the server crawl it one resolution step per page).
+                    if (!truncated)
+                    {
+                        if (sourceName is not null && !string.Equals(wireEvent.SourceName, sourceName, StringComparison.Ordinal))
+                            continue;
+                        truncated = true;
+                    }
+                    if (++pastCap > _eventDrainBudget)
+                    {
+                        abandoned = true;
+                        break;
+                    }
+                    continue;
+                }
+
                var mapped = EventMapper.ToHistoricalEvent(wireEvent);

                // Defensive client-side source filter: the gateway's SQL ReadEvents source filter
@@ -633,17 +691,17 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
                    continue;
                }

-                // One more matching event arriving once the cap is full means the result is
-                // truncated — stop draining and flag it.
-                if (hasCap && collected.Count == maxEvents)
-                {
-                    truncated = true;
-                    break;
-                }
-
                collected.Add(mapped);
            }

+            if (abandoned)
+            {
+                Interlocked.Increment(ref _eventStreamsAbandoned);
+                _logger.LogDebug(
+                    "HistorianGateway events read abandoned its stream {Budget} rows past the page cap (EventDrainBudget); " +
+                    "the gateway may evict its historian session.", _eventDrainBudget);
+            }
+
            RecordOutcome(success: true, error: null);
            // A non-null, opaque token signals truncation to the caller.
            // The gateway has no resumable cursor, so the token's contents carry no paging state —
@@ -651,6 +709,10 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
            // from being shared/mutated.
            return new HistoricalEventsResult(collected, truncated ? new byte[] { 0x01 } : null);
        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            throw; // the caller (client cancel / HistoryReadDeadline) asked for this — not a backend fault
+        }
        catch (Exception ex)
        {
            RecordReadFailure(ex);
@@ -46,7 +46,9 @@ public static class GatewayHistorian
            // A crowding re-read never makes room for more pre-window rows than the tie-cluster limit.
            maxPreWindowRows: options.MaxTieClusterOverfetch,
            // The server raw cap (R08:1.1): the page limit the server reads at, and the bound on every raw read.
-            maxRawValuesPerRead: options.EffectiveMaxRawValuesPerRead);
+            maxRawValuesPerRead: options.EffectiveMaxRawValuesPerRead,
+            // How far a capped events read drains its stream past the cap before abandoning it.
+            eventDrainBudget: options.Gateway.EventDrainBudget);
    }

    /// <summary>