feat(historian-gateway): native TimeAverage/Total on by default, SendEvents batches, server-side max_events (#694, #746)
#694 item 3 — historiangw#27 is fixed and now live-verified through OtOpcUa: - ServerHistorian:Gateway:EnableNativeTimeWeighted defaults to true (false is the kill switch for a gateway older than 9f711c1). TimeAverage / Total are served and advertised again (12 aggregates). - The gateway sends the interval as max(1, round(ms)) and AVEVA clips its last interval at the request end (live). NativeAggregatePlan therefore sends a multi-interval grid only for whole-ms intervals; a fractional interval is read one interval per request (at most 1,000, else BadTooManyOperations), and every one-interval request goes out rounded UP to whole ms so AVEVA answers one clipped row. Live, a 3.0004 s clipped interval sent at its own width lost its last 0.4 ms (Total 123 vs 123.0172). #746 — HistorianGateway Contracts/Client 1.0.0 surface: - Alarm history: GatewayAlarmHistorianWriter sends SendEvents batches of <=200. A failed per-event result is re-sent alone via SendEvent (classified as before); a refused batch (InvalidArgument etc.) goes event by event so a poison event dead-letters alone; a systemic fault retries the rest without sending more; Unimplemented (older gateway) falls back to SendEvent for good. IHistorianGatewayClient.SendEventsAsync defaults to Unimplemented. - Events reads: a capped page sends cap + 1 as ReadEventsRequest.max_events, so the gateway completes the stream instead of the client cancelling it. An ambiguous capped answer (foreign rows filtered out of the server's cut) or a refused cap is re-read uncapped with the existing EventDrainBudget drain. Tests: GatewayServerCappedEventsTests, GatewayAlarmWriterBatchTests, GatewayNativeWholeMillisecondGridTests (gateway-faithful rounding fake), gate tests flipped to the new default; every new test mutation-checked (each mutation of the production change fails at least one). Live harness AvevaGatewayHistorianFollowupsLiveGateTests (read-only) for #694/#700/#746. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
+144
-64
@@ -39,10 +39,9 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// up-front <c>BadAggregateNotSupported</c> read it too.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The two native members are served only with <c>ServerHistorian:Gateway:EnableNativeTimeWeighted</c>: until the
|
||||
/// gateway's interval fix (historiangw#27) is live-verified AVEVA answers them with Good-status wrong values, so by
|
||||
/// default an instance serves (and advertises) <see cref="RawEvaluatedAggregates"/> only — see
|
||||
/// <see cref="SupportedAggregates"/>.
|
||||
/// The two native members are served only with <c>ServerHistorian:Gateway:EnableNativeTimeWeighted</c> (default
|
||||
/// <c>true</c> since lmxopcua#694, once the gateway's interval fix historiangw#27 was live-verified); with it off an
|
||||
/// instance serves (and advertises) <see cref="RawEvaluatedAggregates"/> only — see <see cref="SupportedAggregates"/>.
|
||||
/// </remarks>
|
||||
public static IReadOnlySet<HistoryAggregateType> ServedAggregates { get; } =
|
||||
Enum.GetValues<HistoryAggregateType>()
|
||||
@@ -50,8 +49,8 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
.ToFrozenSet();
|
||||
|
||||
/// <summary>
|
||||
/// The per-sample aggregates this backend evaluates from raw reads — what an instance serves without
|
||||
/// <c>ServerHistorian:Gateway:EnableNativeTimeWeighted</c> (the default).
|
||||
/// The per-sample aggregates this backend evaluates from raw reads — what an instance serves with
|
||||
/// <c>ServerHistorian:Gateway:EnableNativeTimeWeighted=false</c> (the kill switch).
|
||||
/// </summary>
|
||||
public static IReadOnlySet<HistoryAggregateType> RawEvaluatedAggregates { get; } =
|
||||
Enum.GetValues<HistoryAggregateType>().Where(RawAggregateReader.Serves).ToFrozenSet();
|
||||
@@ -163,8 +162,8 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// </param>
|
||||
/// <param name="enableNativeTimeWeighted">
|
||||
/// Serve TimeAverage / Total through AVEVA's native retrieval modes (<c>ServerHistorian:Gateway:EnableNativeTimeWeighted</c>,
|
||||
/// default <c>false</c>): until historiangw#27 is live-verified they answer Good-status wrong values, so by default they
|
||||
/// are refused <see cref="HistoryAggregateNotSupportedException"/> (<c>BadAggregateNotSupported</c>) and not advertised.
|
||||
/// default <c>true</c> since lmxopcua#694). <c>false</c> refuses both <see cref="HistoryAggregateNotSupportedException"/>
|
||||
/// (<c>BadAggregateNotSupported</c>) and stops advertising them — for a gateway older than the historiangw#27 fix.
|
||||
/// </param>
|
||||
/// <param name="probeInterval">
|
||||
/// Cadence of the <c>ProbeAsync</c> health loop (<c>ServerHistorian:Gateway:ProbeInterval</c>); the first probe runs
|
||||
@@ -178,7 +177,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger,
|
||||
int maxRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, TimeProvider? timeProvider = null,
|
||||
bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows, int maxRawValuesPerRead = 0,
|
||||
int eventDrainBudget = DefaultEventDrainBudget, bool enableNativeTimeWeighted = false,
|
||||
int eventDrainBudget = DefaultEventDrainBudget, bool enableNativeTimeWeighted = true,
|
||||
TimeSpan? probeInterval = null)
|
||||
{
|
||||
_enableNativeTimeWeighted = enableNativeTimeWeighted;
|
||||
@@ -494,7 +493,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
|
||||
// Mapped OUTSIDE the failure-recording try: an aggregate the gateway has no mode for is a client
|
||||
// request the server answers BadAggregateNotSupported, not a backend fault to count. The native pair is refused
|
||||
// the same way unless opted in (Good-status wrong values until historiangw#27).
|
||||
// the same way when EnableNativeTimeWeighted is off (the kill switch for a gateway older than historiangw#27).
|
||||
var mode = AggregateModeMapper.ToRetrievalMode(aggregate);
|
||||
if (!_enableNativeTimeWeighted) throw new HistoryAggregateNotSupportedException(aggregate);
|
||||
// Planned OUTSIDE the failure-recording try: too many intervals is a client request limit, not a backend fault.
|
||||
@@ -510,6 +509,8 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (plan.BucketCount == 0) return new HistoryReadResult([], ContinuationPoint: null);
|
||||
// Planned OUTSIDE the failure-recording try: a non-whole-ms interval over too many intervals is a client request limit.
|
||||
var requests = NativeAggregatePlan.Requests(plan);
|
||||
try
|
||||
{
|
||||
if (!await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
@@ -518,7 +519,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
return new HistoryReadResult(plan.Assemble(new Dictionary<long, DataValueSnapshot>(), t => AggregateEvaluator.NoData(t, _time.GetUtcNow().UtcDateTime)), ContinuationPoint: null);
|
||||
}
|
||||
var byBucket = new Dictionary<long, DataValueSnapshot>();
|
||||
foreach (var request in NativeAggregatePlan.Requests(plan))
|
||||
foreach (var request in requests)
|
||||
{
|
||||
var buckets = new List<HistorianAggregateSample>();
|
||||
await foreach (var bucket in _client
|
||||
@@ -738,72 +739,61 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
"The HistorianGateway backend reads events oldest-first only; a reverse (newest-first) events read is not supported.")
|
||||
: ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// The largest page cap sent as the server-side <c>max_events</c> (as cap + 1): the gateway's default
|
||||
/// <c>Historian:ReadLimits:MaxEventsCeiling</c> is 1,000,000 and a larger <c>max_events</c> is <c>InvalidArgument</c>.
|
||||
/// A larger cap (an uncapped read arrives as <see cref="int.MaxValue"/>) is read without a server cap.
|
||||
/// </summary>
|
||||
internal const int MaxServerCappedEvents = 999_999;
|
||||
|
||||
/// <inheritdoc />
|
||||
// The gateway's ReadEvents has no max-events field (the 0.3.0 wire contract), so the cap is applied here. The stream
|
||||
// is DRAINED past the cap — counted, never collected — rather than broken out of: abandoning a gateway stream cancels
|
||||
// the RPC, which the gateway classifies as a connection fault and answers by evicting its pooled historian session
|
||||
// (the next read of any kind reconnects; concurrent reads on that session fail). The drain is bounded by the
|
||||
// EventDrainBudget (ServerHistorian:Gateway:EventDrainBudget): past cap + budget events the stream is abandoned after
|
||||
// all (a window far larger than the page — streaming all of it for every page would cost O(window²/N)), logged at
|
||||
// Debug and counted in EventStreamsAbandoned. 0 = abandon at the cap (the previous behaviour).
|
||||
// A capped read asks the gateway for cap + 1 events through ReadEventsRequest.max_events (HistorianGateway
|
||||
// Contracts/Client 0.4.0+, lmxopcua#746): the gateway stops there and COMPLETES the stream normally — nothing is cancelled,
|
||||
// so it never evicts its pooled historian session for this read — and the (cap + 1)-th matching event is the "more
|
||||
// events exist" signal. When that answer is ambiguous — the server stopped at cap + 1 rows but the defensive client-side
|
||||
// source filter dropped some of them, so whether more matching events exist is unknown — or a gateway refuses the cap
|
||||
// (InvalidArgument: a MaxEventsCeiling below cap + 1), the read is repeated without a server cap by the drain below.
|
||||
//
|
||||
// Uncapped (no server cap) the stream is DRAINED past the page cap — counted, never collected — rather than broken out
|
||||
// of: abandoning a gateway stream cancels the RPC, which the gateway classifies as a connection fault and answers by
|
||||
// evicting its pooled historian session. The drain is bounded by the EventDrainBudget
|
||||
// (ServerHistorian:Gateway:EventDrainBudget): past cap + budget events the stream is abandoned after all, logged at Debug
|
||||
// and counted in EventStreamsAbandoned. 0 = abandon at the cap.
|
||||
public async Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var hasCap = maxEvents > 0;
|
||||
// Bounded initial capacity: an uncapped read arrives as int.MaxValue ("Array dimensions exceeded").
|
||||
var collected = new List<HistoricalEvent>(hasCap ? Math.Min(maxEvents, 1024) : 0);
|
||||
var truncated = false;
|
||||
long pastCap = 0; // stream rows consumed after the cap filled (matching or not)
|
||||
var abandoned = false;
|
||||
|
||||
await foreach (var wireEvent in _client
|
||||
.ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
var serverCap = maxEvents is > 0 and <= MaxServerCappedEvents ? maxEvents + 1 : 0;
|
||||
EventDrain drain;
|
||||
if (serverCap > 0)
|
||||
{
|
||||
if (hasCap && collected.Count == maxEvents)
|
||||
try
|
||||
{
|
||||
// The page is full: drain the rest of the stream without collecting it, within the budget.
|
||||
// One more MATCHING event is the "more events exist" signal; a foreign source's row is not, and
|
||||
// until the signal is seen no row counts toward the budget — abandoning before it would answer
|
||||
// "complete" for a page that may not be (and a spurious continuation point on a stretch of foreign
|
||||
// rows would make the server crawl it one resolution step per page).
|
||||
if (!truncated)
|
||||
{
|
||||
if (!EventSourceMatch.Matches(sourceName, wireEvent.SourceName))
|
||||
continue;
|
||||
truncated = true;
|
||||
}
|
||||
if (++pastCap > _eventDrainBudget)
|
||||
{
|
||||
abandoned = true;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
drain = await DrainEventsAsync(sourceName, startUtc, endUtc, maxEvents, serverCap, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var mapped = EventMapper.ToHistoricalEvent(wireEvent);
|
||||
|
||||
// Defensive client-side source filter: the gateway's SQL ReadEvents source filter
|
||||
// may not be present, so drop any event whose source does not match the request. The request's own
|
||||
// events and its attributes' (<source>.<attribute>, the name the gateway reports an AVEVA attribute
|
||||
// alarm under) match; a sibling that merely shares the prefix does not (#882, EventSourceMatch).
|
||||
if (!EventSourceMatch.Matches(sourceName, mapped.SourceName))
|
||||
catch (Exception ex) when (!cancellationToken.IsCancellationRequested && IsInvalidArgument(ex))
|
||||
{
|
||||
continue;
|
||||
_logger.LogDebug("HistorianGateway refused the events read's max_events {Cap}; re-reading it without a server cap.",
|
||||
serverCap);
|
||||
drain = await DrainEventsAsync(sourceName, startUtc, endUtc, maxEvents, 0, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
// Exactly cap + 1 rows and no signal: the server cut the stream, but foreign rows were filtered out of it.
|
||||
// (More rows than asked = a gateway that ignores max_events streamed everything: the drain is the answer.)
|
||||
if (drain.Received == serverCap && !drain.Truncated)
|
||||
{
|
||||
Interlocked.Increment(ref _eventServerCapReReads);
|
||||
drain = await DrainEventsAsync(sourceName, startUtc, endUtc, maxEvents, 0, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
collected.Add(mapped);
|
||||
}
|
||||
|
||||
if (abandoned)
|
||||
else
|
||||
{
|
||||
Interlocked.Increment(ref _eventStreamsAbandoned);
|
||||
_logger.LogDebug(
|
||||
"HistorianGateway events read abandoned its stream {Budget} rows past the page cap (EventDrainBudget); " +
|
||||
"the gateway may evict its historian session.", _eventDrainBudget);
|
||||
drain = await DrainEventsAsync(sourceName, startUtc, endUtc, maxEvents, 0, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
RecordOutcome(success: true, error: null);
|
||||
@@ -811,7 +801,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
// The gateway has no resumable cursor, so the token's contents carry no paging state —
|
||||
// its presence alone is the "more events exist" signal. A fresh array per call keeps it
|
||||
// from being shared/mutated.
|
||||
return new HistoricalEventsResult(collected, truncated ? new byte[] { 0x01 } : null);
|
||||
return new HistoricalEventsResult(drain.Collected, drain.Truncated ? new byte[] { 0x01 } : null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
@@ -825,6 +815,96 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
}
|
||||
}
|
||||
|
||||
private long _eventServerCapReReads;
|
||||
|
||||
/// <summary>
|
||||
/// Server-capped events reads repeated without the cap because the capped answer was ambiguous (foreign rows filtered
|
||||
/// out of a stream the server cut at cap + 1) — test/diagnostic.
|
||||
/// </summary>
|
||||
internal long EventServerCapReReads => Interlocked.Read(ref _eventServerCapReReads);
|
||||
|
||||
/// <summary>One events stream's outcome: the collected page, the "more exist" signal and the stream rows received.</summary>
|
||||
private readonly record struct EventDrain(List<HistoricalEvent> Collected, bool Truncated, long Received);
|
||||
|
||||
/// <summary>
|
||||
/// Reads one events stream: collects up to <paramref name="maxEvents"/> matching events, then (cap reached) looks for
|
||||
/// one more matching event as the "more exist" signal and drains the rest within the EventDrainBudget.
|
||||
/// <paramref name="serverCap"/> > 0 is sent as <c>max_events</c>; 0 = no server cap.
|
||||
/// </summary>
|
||||
private async Task<EventDrain> DrainEventsAsync(string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
|
||||
int serverCap, CancellationToken cancellationToken)
|
||||
{
|
||||
var hasCap = maxEvents > 0;
|
||||
// Bounded initial capacity: an uncapped read arrives as int.MaxValue ("Array dimensions exceeded").
|
||||
var collected = new List<HistoricalEvent>(hasCap ? Math.Min(maxEvents, 1024) : 0);
|
||||
var truncated = false;
|
||||
long pastCap = 0; // stream rows consumed after the cap filled (matching or not)
|
||||
long received = 0;
|
||||
var abandoned = false;
|
||||
|
||||
await foreach (var wireEvent in _client
|
||||
.ReadEventsAsync(sourceName, startUtc, endUtc, serverCap, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
received++;
|
||||
if (hasCap && collected.Count == maxEvents)
|
||||
{
|
||||
// The page is full: drain the rest of the stream without collecting it, within the budget.
|
||||
// One more MATCHING event is the "more events exist" signal; a foreign source's row is not, and
|
||||
// until the signal is seen no row counts toward the budget — abandoning before it would answer
|
||||
// "complete" for a page that may not be (and a spurious continuation point on a stretch of foreign
|
||||
// rows would make the server crawl it one resolution step per page).
|
||||
if (!truncated)
|
||||
{
|
||||
if (!EventSourceMatch.Matches(sourceName, wireEvent.SourceName))
|
||||
continue;
|
||||
truncated = true;
|
||||
}
|
||||
// Server-capped, the stream is already bounded at cap + 1 (by the gateway, or by the package client against an
|
||||
// older gateway): read it to its end so the call completes normally instead of being cancelled. Only a stream
|
||||
// running past the max_events it was sent is drained within the budget.
|
||||
if (serverCap > 0 && received <= serverCap) continue;
|
||||
if (++pastCap > _eventDrainBudget)
|
||||
{
|
||||
abandoned = true;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
var mapped = EventMapper.ToHistoricalEvent(wireEvent);
|
||||
|
||||
// Defensive client-side source filter: the gateway's SQL ReadEvents source filter
|
||||
// may not be present, so drop any event whose source does not match the request. The request's own
|
||||
// events and its attributes' (<source>.<attribute>, the name the gateway reports an AVEVA attribute
|
||||
// alarm under) match; a sibling that merely shares the prefix does not (#882, EventSourceMatch).
|
||||
if (!EventSourceMatch.Matches(sourceName, mapped.SourceName))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
collected.Add(mapped);
|
||||
}
|
||||
|
||||
if (abandoned)
|
||||
{
|
||||
Interlocked.Increment(ref _eventStreamsAbandoned);
|
||||
_logger.LogDebug(
|
||||
"HistorianGateway events read abandoned its stream {Budget} rows past the page cap (EventDrainBudget); " +
|
||||
"the gateway may evict its historian session.", _eventDrainBudget);
|
||||
}
|
||||
|
||||
return new EventDrain(collected, truncated, received);
|
||||
}
|
||||
|
||||
/// <summary>Whether a gateway failure is an <c>InvalidArgument</c> refusal (raw, or wrapped by the package client).</summary>
|
||||
private static bool IsInvalidArgument(Exception ex)
|
||||
{
|
||||
for (Exception? e = ex; e is not null; e = e.InnerException)
|
||||
if (e is RpcException { StatusCode: Grpc.Core.StatusCode.InvalidArgument }) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public HistorianHealthSnapshot GetHealthSnapshot()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user