feat(historian-gateway): native TimeAverage/Total on by default, SendEvents batches, server-side max_events (#694, #746)
#694 item 3 — historiangw#27 is fixed and now live-verified through OtOpcUa: - ServerHistorian:Gateway:EnableNativeTimeWeighted defaults to true (false is the kill switch for a gateway older than 9f711c1). TimeAverage / Total are served and advertised again (12 aggregates). - The gateway sends the interval as max(1, round(ms)) and AVEVA clips its last interval at the request end (live). NativeAggregatePlan therefore sends a multi-interval grid only for whole-ms intervals; a fractional interval is read one interval per request (at most 1,000, else BadTooManyOperations), and every one-interval request goes out rounded UP to whole ms so AVEVA answers one clipped row. Live, a 3.0004 s clipped interval sent at its own width lost its last 0.4 ms (Total 123 vs 123.0172). #746 — HistorianGateway Contracts/Client 1.0.0 surface: - Alarm history: GatewayAlarmHistorianWriter sends SendEvents batches of <=200. A failed per-event result is re-sent alone via SendEvent (classified as before); a refused batch (InvalidArgument etc.) goes event by event so a poison event dead-letters alone; a systemic fault retries the rest without sending more; Unimplemented (older gateway) falls back to SendEvent for good. IHistorianGatewayClient.SendEventsAsync defaults to Unimplemented. - Events reads: a capped page sends cap + 1 as ReadEventsRequest.max_events, so the gateway completes the stream instead of the client cancelling it. An ambiguous capped answer (foreign rows filtered out of the server's cut) or a refused cap is re-read uncapped with the existing EventDrainBudget drain. Tests: GatewayServerCappedEventsTests, GatewayAlarmWriterBatchTests, GatewayNativeWholeMillisecondGridTests (gateway-faithful rounding fake), gate tests flipped to the new default; every new test mutation-checked (each mutation of the production change fails at least one). Live harness AvevaGatewayHistorianFollowupsLiveGateTests (read-only) for #694/#700/#746. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
+158
-24
@@ -8,20 +8,35 @@ using ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Mapping;
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="IAlarmHistorianWriter"/> backed by the HistorianGateway <c>SendEvent</c> path. The
|
||||
/// drain worker behind <c>LocalDbStoreAndForwardSink</c> calls
|
||||
/// <see cref="WriteBatchAsync"/> and uses the returned per-event
|
||||
/// <see cref="HistorianWriteOutcome"/> to decide retry vs. dead-letter, so this writer maps every
|
||||
/// gateway result — success ack, the published client's typed exception hierarchy, raw
|
||||
/// <see cref="RpcException"/> (defensive), and any unexpected error — onto exactly one outcome per
|
||||
/// event and <b>never throws</b>.
|
||||
/// <see cref="IAlarmHistorianWriter"/> backed by the HistorianGateway <c>SendEvents</c> batch RPC (Contracts/Client
|
||||
/// 0.4.0+), with the single-event <c>SendEvent</c> path as its fallback. The drain worker behind
|
||||
/// <c>LocalDbStoreAndForwardSink</c> calls <see cref="WriteBatchAsync"/> and uses the returned per-event
|
||||
/// <see cref="HistorianWriteOutcome"/> to decide retry vs. dead-letter, so this writer maps every gateway result — a
|
||||
/// batch result, a success ack, the published client's typed exception hierarchy, raw <see cref="RpcException"/>
|
||||
/// (defensive), and any unexpected error — onto exactly one outcome per event and <b>never throws</b>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Each event is sent individually so one poison event cannot fail the whole batch: a permanent
|
||||
/// failure on event N is dead-lettered while its siblings continue. The sends are pipelined — at most
|
||||
/// <see cref="MaxSendsInFlight"/> in flight — because the gateway has no batch <c>SendEvent</c> RPC yet.
|
||||
/// Outcomes are returned in input order, one per event; an empty batch yields an empty list with no gateway call.
|
||||
/// <b>Batches (lmxopcua#746).</b> A drain batch is sent as <c>SendEvents</c> calls of at most
|
||||
/// <see cref="MaxEventsPerBatch"/> events, one round trip each. Per event result: committed / queued ⇒
|
||||
/// <see cref="HistorianWriteOutcome.Ack"/>; failed ⇒ that event is re-sent ALONE through <c>SendEvent</c> so it is
|
||||
/// classified exactly as before (a historian rejection dead-letters, a transient fault retries) — a failed result
|
||||
/// was not committed, so the re-send cannot duplicate it. A whole-call failure:
|
||||
/// </para>
|
||||
/// <list type="bullet">
|
||||
/// <item>transient / systemic (<c>Unavailable</c>, auth, back-pressure, deadline, cancellation) ⇒ that call's and
|
||||
/// every later call's events are <see cref="HistorianWriteOutcome.RetryPlease"/>, nothing more is sent (every
|
||||
/// later call would hit it too). Events before the fault may already be committed, so the retry can duplicate
|
||||
/// them — <c>SendEvent</c> is at-least-once too;</item>
|
||||
/// <item><c>Unimplemented</c> (a gateway older than the RPC) ⇒ the writer stops trying batches for its lifetime and
|
||||
/// sends every event individually;</item>
|
||||
/// <item>any other refusal (<c>InvalidArgument</c> — a poison event or an over-limit batch, nothing sent — and the
|
||||
/// other permanent codes) ⇒ that call's events are sent individually, so the poison event dead-letters alone.</item>
|
||||
/// </list>
|
||||
/// <para>
|
||||
/// <b>Single sends</b> are pipelined — at most <see cref="MaxSendsInFlight"/> in flight — and each gets its own
|
||||
/// outcome. Outcomes are returned in input order, one per event; an empty batch yields an empty list with no gateway
|
||||
/// call.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Outcome mapping.</b> Success (or store-forward-queued) ack ⇒ <see cref="HistorianWriteOutcome.Ack"/>.
|
||||
@@ -42,8 +57,11 @@ public sealed class GatewayAlarmHistorianWriter : IAlarmHistorianWriter, IDispos
|
||||
private readonly IHistorianGatewayClient _client;
|
||||
private readonly ILogger<GatewayAlarmHistorianWriter> _logger;
|
||||
|
||||
/// <summary>Set once the gateway answers <c>SendEvents</c> with <c>Unimplemented</c>: single sends from then on.</summary>
|
||||
private volatile bool _batchUnsupported;
|
||||
|
||||
/// <summary>Creates the writer over a gateway client seam.</summary>
|
||||
/// <param name="client">The gateway client used for the <c>SendEvent</c> write path.</param>
|
||||
/// <param name="client">The gateway client used for the <c>SendEvents</c> / <c>SendEvent</c> write paths.</param>
|
||||
/// <param name="logger">Logger for per-event outcome diagnostics (never logs event content).</param>
|
||||
public GatewayAlarmHistorianWriter(IHistorianGatewayClient client, ILogger<GatewayAlarmHistorianWriter> logger)
|
||||
{
|
||||
@@ -51,6 +69,15 @@ public sealed class GatewayAlarmHistorianWriter : IAlarmHistorianWriter, IDispos
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
/// <summary>The most events one <c>SendEvents</c> call carries (the gateway's own default limit is 10,000 / 16 MiB).</summary>
|
||||
public const int MaxEventsPerBatch = 200;
|
||||
|
||||
/// <summary>The most single <c>SendEvent</c> RPCs one batch has in flight at once (the fallback path).</summary>
|
||||
public const int MaxSendsInFlight = 8;
|
||||
|
||||
/// <summary>Whether the writer has fallen back to single sends for good (the gateway has no <c>SendEvents</c>) — diagnostic.</summary>
|
||||
internal bool BatchUnsupported => _batchUnsupported;
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<HistorianWriteOutcome>> WriteBatchAsync(
|
||||
IReadOnlyList<AlarmHistorianEvent> batch, CancellationToken cancellationToken)
|
||||
@@ -62,21 +89,123 @@ public sealed class GatewayAlarmHistorianWriter : IAlarmHistorianWriter, IDispos
|
||||
return Array.Empty<HistorianWriteOutcome>();
|
||||
}
|
||||
|
||||
var outcomes = new HistorianWriteOutcome[batch.Count];
|
||||
var outcomes = new HistorianWriteOutcome?[batch.Count];
|
||||
var singles = new List<int>();
|
||||
|
||||
// The gateway has no batch SendEvent RPC, so the sends are PIPELINED: up to MaxSendsInFlight at once, each still
|
||||
// its own RPC with its own outcome (so one poison event cannot fail the batch), outcomes kept in input order. A
|
||||
// sequential drain cost one round trip per event — an alarm storm over a WAN took minutes.
|
||||
using var window = new SemaphoreSlim(MaxSendsInFlight);
|
||||
var sends = new Task[batch.Count];
|
||||
for (var i = 0; i < batch.Count; i++)
|
||||
for (var offset = 0; offset < batch.Count; offset += MaxEventsPerBatch)
|
||||
{
|
||||
var index = i;
|
||||
sends[i] = SendInWindowAsync(index);
|
||||
var count = Math.Min(MaxEventsPerBatch, batch.Count - offset);
|
||||
if (_batchUnsupported)
|
||||
{
|
||||
for (var i = 0; i < count; i++) singles.Add(offset + i);
|
||||
continue;
|
||||
}
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Shutdown mid-drain: an event never sent stays queued for next startup — never dead-lettered.
|
||||
for (var i = offset; i < batch.Count; i++) outcomes[i] = HistorianWriteOutcome.RetryPlease;
|
||||
break;
|
||||
}
|
||||
|
||||
if (!await SendChunkAsync(batch, offset, count, outcomes, singles, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// A systemic fault: every later call would hit it too — retry them all, send nothing more.
|
||||
for (var i = offset + count; i < batch.Count; i++) outcomes[i] = HistorianWriteOutcome.RetryPlease;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (singles.Count > 0) await SendSinglesAsync(batch, singles, outcomes, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var result = new HistorianWriteOutcome[batch.Count];
|
||||
for (var i = 0; i < result.Length; i++) result[i] = outcomes[i] ?? HistorianWriteOutcome.RetryPlease;
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sends events [<paramref name="offset"/>, +<paramref name="count"/>) as one <c>SendEvents</c> call, recording each
|
||||
/// event's outcome or queueing it for a single send. Returns <c>false</c> on a systemic fault (the chunk's events are
|
||||
/// then <see cref="HistorianWriteOutcome.RetryPlease"/> and the caller sends nothing more).
|
||||
/// </summary>
|
||||
private async Task<bool> SendChunkAsync(IReadOnlyList<AlarmHistorianEvent> batch, int offset, int count,
|
||||
HistorianWriteOutcome?[] outcomes, List<int> singles, CancellationToken cancellationToken)
|
||||
{
|
||||
var events = new List<HistorianEvent>(count);
|
||||
var indexes = new List<int>(count);
|
||||
for (var i = offset; i < offset + count; i++)
|
||||
{
|
||||
try
|
||||
{
|
||||
events.Add(AlarmEventMapper.ToHistorianEvent(batch[i]));
|
||||
indexes.Add(i);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
// An event that cannot even be mapped is poison: classified exactly as a single send would classify it.
|
||||
outcomes[i] = Classify(exception);
|
||||
_logger.LogWarning("Alarm event could not be mapped for SendEvents ({Exception}); dead-lettering it.",
|
||||
exception.GetType().Name);
|
||||
}
|
||||
}
|
||||
if (events.Count == 0) return true;
|
||||
|
||||
SendEventsReply reply;
|
||||
try
|
||||
{
|
||||
reply = await _client.SendEventsAsync(events, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogDebug("Alarm SendEvents cancelled at shutdown; will retry.");
|
||||
foreach (var i in indexes) outcomes[i] = HistorianWriteOutcome.RetryPlease;
|
||||
return false;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
if (StatusOf(exception) == StatusCode.Unimplemented)
|
||||
{
|
||||
_batchUnsupported = true;
|
||||
_logger.LogInformation(
|
||||
"HistorianGateway has no SendEvents RPC (older than Contracts 0.4.0); alarm-history writes fall back to one SendEvent per event.");
|
||||
singles.AddRange(indexes);
|
||||
return true;
|
||||
}
|
||||
if (Classify(exception) == HistorianWriteOutcome.RetryPlease)
|
||||
{
|
||||
_logger.LogDebug("Alarm SendEvents transiently failed ({Exception}); the batch will retry.", exception.GetType().Name);
|
||||
foreach (var i in indexes) outcomes[i] = HistorianWriteOutcome.RetryPlease;
|
||||
return false;
|
||||
}
|
||||
// Refused up front (a poison event, an over-limit batch, …): nothing was sent — send them one by one so a poison
|
||||
// event dead-letters alone and its siblings still land.
|
||||
_logger.LogDebug("Alarm SendEvents refused the batch ({Exception}); sending its events individually.", exception.GetType().Name);
|
||||
singles.AddRange(indexes);
|
||||
return true;
|
||||
}
|
||||
|
||||
for (var k = 0; k < indexes.Count; k++)
|
||||
{
|
||||
if (k < reply.Results.Count && reply.Results[k] is { } r && (r.Success || r.Queued))
|
||||
outcomes[indexes[k]] = HistorianWriteOutcome.Ack;
|
||||
else
|
||||
singles.Add(indexes[k]); // failed (not committed) or missing from the reply: classify it by a single send
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>Sends the given events one <c>SendEvent</c> each, at most <see cref="MaxSendsInFlight"/> in flight.</summary>
|
||||
private async Task SendSinglesAsync(IReadOnlyList<AlarmHistorianEvent> batch, List<int> indexes,
|
||||
HistorianWriteOutcome?[] outcomes, CancellationToken cancellationToken)
|
||||
{
|
||||
using var window = new SemaphoreSlim(MaxSendsInFlight);
|
||||
var sends = new Task[indexes.Count];
|
||||
for (var k = 0; k < indexes.Count; k++)
|
||||
{
|
||||
var index = indexes[k];
|
||||
sends[k] = SendInWindowAsync(index);
|
||||
}
|
||||
// Every task completes (none throws), so the window is never disposed under a waiter.
|
||||
await Task.WhenAll(sends).ConfigureAwait(false);
|
||||
return outcomes;
|
||||
|
||||
async Task SendInWindowAsync(int index)
|
||||
{
|
||||
@@ -104,8 +233,13 @@ public sealed class GatewayAlarmHistorianWriter : IAlarmHistorianWriter, IDispos
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The most <c>SendEvent</c> RPCs one batch has in flight at once (bounded pipelining until a batch RPC exists).</summary>
|
||||
public const int MaxSendsInFlight = 8;
|
||||
/// <summary>The gRPC status of a gateway failure (raw, or wrapped by the package client); null when it has none.</summary>
|
||||
private static StatusCode? StatusOf(Exception exception)
|
||||
{
|
||||
for (Exception? e = exception; e is not null; e = e.InnerException)
|
||||
if (e is RpcException rpc) return rpc.StatusCode;
|
||||
return null;
|
||||
}
|
||||
|
||||
private async Task<HistorianWriteOutcome> SendOneAsync(AlarmHistorianEvent evt, CancellationToken cancellationToken)
|
||||
{
|
||||
|
||||
+144
-64
@@ -39,10 +39,9 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// up-front <c>BadAggregateNotSupported</c> read it too.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The two native members are served only with <c>ServerHistorian:Gateway:EnableNativeTimeWeighted</c>: until the
|
||||
/// gateway's interval fix (historiangw#27) is live-verified AVEVA answers them with Good-status wrong values, so by
|
||||
/// default an instance serves (and advertises) <see cref="RawEvaluatedAggregates"/> only — see
|
||||
/// <see cref="SupportedAggregates"/>.
|
||||
/// The two native members are served only with <c>ServerHistorian:Gateway:EnableNativeTimeWeighted</c> (default
|
||||
/// <c>true</c> since lmxopcua#694, once the gateway's interval fix historiangw#27 was live-verified); with it off an
|
||||
/// instance serves (and advertises) <see cref="RawEvaluatedAggregates"/> only — see <see cref="SupportedAggregates"/>.
|
||||
/// </remarks>
|
||||
public static IReadOnlySet<HistoryAggregateType> ServedAggregates { get; } =
|
||||
Enum.GetValues<HistoryAggregateType>()
|
||||
@@ -50,8 +49,8 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
.ToFrozenSet();
|
||||
|
||||
/// <summary>
|
||||
/// The per-sample aggregates this backend evaluates from raw reads — what an instance serves without
|
||||
/// <c>ServerHistorian:Gateway:EnableNativeTimeWeighted</c> (the default).
|
||||
/// The per-sample aggregates this backend evaluates from raw reads — what an instance serves with
|
||||
/// <c>ServerHistorian:Gateway:EnableNativeTimeWeighted=false</c> (the kill switch).
|
||||
/// </summary>
|
||||
public static IReadOnlySet<HistoryAggregateType> RawEvaluatedAggregates { get; } =
|
||||
Enum.GetValues<HistoryAggregateType>().Where(RawAggregateReader.Serves).ToFrozenSet();
|
||||
@@ -163,8 +162,8 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
/// </param>
|
||||
/// <param name="enableNativeTimeWeighted">
|
||||
/// Serve TimeAverage / Total through AVEVA's native retrieval modes (<c>ServerHistorian:Gateway:EnableNativeTimeWeighted</c>,
|
||||
/// default <c>false</c>): until historiangw#27 is live-verified they answer Good-status wrong values, so by default they
|
||||
/// are refused <see cref="HistoryAggregateNotSupportedException"/> (<c>BadAggregateNotSupported</c>) and not advertised.
|
||||
/// default <c>true</c> since lmxopcua#694). <c>false</c> refuses both <see cref="HistoryAggregateNotSupportedException"/>
|
||||
/// (<c>BadAggregateNotSupported</c>) and stops advertising them — for a gateway older than the historiangw#27 fix.
|
||||
/// </param>
|
||||
/// <param name="probeInterval">
|
||||
/// Cadence of the <c>ProbeAsync</c> health loop (<c>ServerHistorian:Gateway:ProbeInterval</c>); the first probe runs
|
||||
@@ -178,7 +177,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
public GatewayHistorianDataSource(IHistorianGatewayClient client, ILogger<GatewayHistorianDataSource> logger,
|
||||
int maxRawRows = TimeWeightedAggregator.DefaultMaxWindowRows, TimeProvider? timeProvider = null,
|
||||
bool atTimeFromRaw = true, int maxPreWindowRows = DefaultMaxPreWindowRows, int maxRawValuesPerRead = 0,
|
||||
int eventDrainBudget = DefaultEventDrainBudget, bool enableNativeTimeWeighted = false,
|
||||
int eventDrainBudget = DefaultEventDrainBudget, bool enableNativeTimeWeighted = true,
|
||||
TimeSpan? probeInterval = null)
|
||||
{
|
||||
_enableNativeTimeWeighted = enableNativeTimeWeighted;
|
||||
@@ -494,7 +493,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
|
||||
// Mapped OUTSIDE the failure-recording try: an aggregate the gateway has no mode for is a client
|
||||
// request the server answers BadAggregateNotSupported, not a backend fault to count. The native pair is refused
|
||||
// the same way unless opted in (Good-status wrong values until historiangw#27).
|
||||
// the same way when EnableNativeTimeWeighted is off (the kill switch for a gateway older than historiangw#27).
|
||||
var mode = AggregateModeMapper.ToRetrievalMode(aggregate);
|
||||
if (!_enableNativeTimeWeighted) throw new HistoryAggregateNotSupportedException(aggregate);
|
||||
// Planned OUTSIDE the failure-recording try: too many intervals is a client request limit, not a backend fault.
|
||||
@@ -510,6 +509,8 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (plan.BucketCount == 0) return new HistoryReadResult([], ContinuationPoint: null);
|
||||
// Planned OUTSIDE the failure-recording try: a non-whole-ms interval over too many intervals is a client request limit.
|
||||
var requests = NativeAggregatePlan.Requests(plan);
|
||||
try
|
||||
{
|
||||
if (!await _tags.ExistsAsync(fullReference, cancellationToken).ConfigureAwait(false))
|
||||
@@ -518,7 +519,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
return new HistoryReadResult(plan.Assemble(new Dictionary<long, DataValueSnapshot>(), t => AggregateEvaluator.NoData(t, _time.GetUtcNow().UtcDateTime)), ContinuationPoint: null);
|
||||
}
|
||||
var byBucket = new Dictionary<long, DataValueSnapshot>();
|
||||
foreach (var request in NativeAggregatePlan.Requests(plan))
|
||||
foreach (var request in requests)
|
||||
{
|
||||
var buckets = new List<HistorianAggregateSample>();
|
||||
await foreach (var bucket in _client
|
||||
@@ -738,72 +739,61 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
"The HistorianGateway backend reads events oldest-first only; a reverse (newest-first) events read is not supported.")
|
||||
: ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken);
|
||||
|
||||
/// <summary>
|
||||
/// The largest page cap sent as the server-side <c>max_events</c> (as cap + 1): the gateway's default
|
||||
/// <c>Historian:ReadLimits:MaxEventsCeiling</c> is 1,000,000 and a larger <c>max_events</c> is <c>InvalidArgument</c>.
|
||||
/// A larger cap (an uncapped read arrives as <see cref="int.MaxValue"/>) is read without a server cap.
|
||||
/// </summary>
|
||||
internal const int MaxServerCappedEvents = 999_999;
|
||||
|
||||
/// <inheritdoc />
|
||||
// The gateway's ReadEvents has no max-events field (the 0.3.0 wire contract), so the cap is applied here. The stream
|
||||
// is DRAINED past the cap — counted, never collected — rather than broken out of: abandoning a gateway stream cancels
|
||||
// the RPC, which the gateway classifies as a connection fault and answers by evicting its pooled historian session
|
||||
// (the next read of any kind reconnects; concurrent reads on that session fail). The drain is bounded by the
|
||||
// EventDrainBudget (ServerHistorian:Gateway:EventDrainBudget): past cap + budget events the stream is abandoned after
|
||||
// all (a window far larger than the page — streaming all of it for every page would cost O(window²/N)), logged at
|
||||
// Debug and counted in EventStreamsAbandoned. 0 = abandon at the cap (the previous behaviour).
|
||||
// A capped read asks the gateway for cap + 1 events through ReadEventsRequest.max_events (HistorianGateway
|
||||
// Contracts/Client 0.4.0+, lmxopcua#746): the gateway stops there and COMPLETES the stream normally — nothing is cancelled,
|
||||
// so it never evicts its pooled historian session for this read — and the (cap + 1)-th matching event is the "more
|
||||
// events exist" signal. When that answer is ambiguous — the server stopped at cap + 1 rows but the defensive client-side
|
||||
// source filter dropped some of them, so whether more matching events exist is unknown — or a gateway refuses the cap
|
||||
// (InvalidArgument: a MaxEventsCeiling below cap + 1), the read is repeated without a server cap by the drain below.
|
||||
//
|
||||
// Uncapped (no server cap) the stream is DRAINED past the page cap — counted, never collected — rather than broken out
|
||||
// of: abandoning a gateway stream cancels the RPC, which the gateway classifies as a connection fault and answers by
|
||||
// evicting its pooled historian session. The drain is bounded by the EventDrainBudget
|
||||
// (ServerHistorian:Gateway:EventDrainBudget): past cap + budget events the stream is abandoned after all, logged at Debug
|
||||
// and counted in EventStreamsAbandoned. 0 = abandon at the cap.
|
||||
public async Task<HistoricalEventsResult> ReadEventsAsync(
|
||||
string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
var hasCap = maxEvents > 0;
|
||||
// Bounded initial capacity: an uncapped read arrives as int.MaxValue ("Array dimensions exceeded").
|
||||
var collected = new List<HistoricalEvent>(hasCap ? Math.Min(maxEvents, 1024) : 0);
|
||||
var truncated = false;
|
||||
long pastCap = 0; // stream rows consumed after the cap filled (matching or not)
|
||||
var abandoned = false;
|
||||
|
||||
await foreach (var wireEvent in _client
|
||||
.ReadEventsAsync(sourceName, startUtc, endUtc, maxEvents, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
var serverCap = maxEvents is > 0 and <= MaxServerCappedEvents ? maxEvents + 1 : 0;
|
||||
EventDrain drain;
|
||||
if (serverCap > 0)
|
||||
{
|
||||
if (hasCap && collected.Count == maxEvents)
|
||||
try
|
||||
{
|
||||
// The page is full: drain the rest of the stream without collecting it, within the budget.
|
||||
// One more MATCHING event is the "more events exist" signal; a foreign source's row is not, and
|
||||
// until the signal is seen no row counts toward the budget — abandoning before it would answer
|
||||
// "complete" for a page that may not be (and a spurious continuation point on a stretch of foreign
|
||||
// rows would make the server crawl it one resolution step per page).
|
||||
if (!truncated)
|
||||
{
|
||||
if (!EventSourceMatch.Matches(sourceName, wireEvent.SourceName))
|
||||
continue;
|
||||
truncated = true;
|
||||
}
|
||||
if (++pastCap > _eventDrainBudget)
|
||||
{
|
||||
abandoned = true;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
drain = await DrainEventsAsync(sourceName, startUtc, endUtc, maxEvents, serverCap, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var mapped = EventMapper.ToHistoricalEvent(wireEvent);
|
||||
|
||||
// Defensive client-side source filter: the gateway's SQL ReadEvents source filter
|
||||
// may not be present, so drop any event whose source does not match the request. The request's own
|
||||
// events and its attributes' (<source>.<attribute>, the name the gateway reports an AVEVA attribute
|
||||
// alarm under) match; a sibling that merely shares the prefix does not (#882, EventSourceMatch).
|
||||
if (!EventSourceMatch.Matches(sourceName, mapped.SourceName))
|
||||
catch (Exception ex) when (!cancellationToken.IsCancellationRequested && IsInvalidArgument(ex))
|
||||
{
|
||||
continue;
|
||||
_logger.LogDebug("HistorianGateway refused the events read's max_events {Cap}; re-reading it without a server cap.",
|
||||
serverCap);
|
||||
drain = await DrainEventsAsync(sourceName, startUtc, endUtc, maxEvents, 0, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
// Exactly cap + 1 rows and no signal: the server cut the stream, but foreign rows were filtered out of it.
|
||||
// (More rows than asked = a gateway that ignores max_events streamed everything: the drain is the answer.)
|
||||
if (drain.Received == serverCap && !drain.Truncated)
|
||||
{
|
||||
Interlocked.Increment(ref _eventServerCapReReads);
|
||||
drain = await DrainEventsAsync(sourceName, startUtc, endUtc, maxEvents, 0, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
collected.Add(mapped);
|
||||
}
|
||||
|
||||
if (abandoned)
|
||||
else
|
||||
{
|
||||
Interlocked.Increment(ref _eventStreamsAbandoned);
|
||||
_logger.LogDebug(
|
||||
"HistorianGateway events read abandoned its stream {Budget} rows past the page cap (EventDrainBudget); " +
|
||||
"the gateway may evict its historian session.", _eventDrainBudget);
|
||||
drain = await DrainEventsAsync(sourceName, startUtc, endUtc, maxEvents, 0, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
RecordOutcome(success: true, error: null);
|
||||
@@ -811,7 +801,7 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
// The gateway has no resumable cursor, so the token's contents carry no paging state —
|
||||
// its presence alone is the "more events exist" signal. A fresh array per call keeps it
|
||||
// from being shared/mutated.
|
||||
return new HistoricalEventsResult(collected, truncated ? new byte[] { 0x01 } : null);
|
||||
return new HistoricalEventsResult(drain.Collected, drain.Truncated ? new byte[] { 0x01 } : null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
@@ -825,6 +815,96 @@ public sealed class GatewayHistorianDataSource : IHistorianDataSource, IAsyncDis
|
||||
}
|
||||
}
|
||||
|
||||
private long _eventServerCapReReads;
|
||||
|
||||
/// <summary>
|
||||
/// Server-capped events reads repeated without the cap because the capped answer was ambiguous (foreign rows filtered
|
||||
/// out of a stream the server cut at cap + 1) — test/diagnostic.
|
||||
/// </summary>
|
||||
internal long EventServerCapReReads => Interlocked.Read(ref _eventServerCapReReads);
|
||||
|
||||
/// <summary>One events stream's outcome: the collected page, the "more exist" signal and the stream rows received.</summary>
|
||||
private readonly record struct EventDrain(List<HistoricalEvent> Collected, bool Truncated, long Received);
|
||||
|
||||
/// <summary>
|
||||
/// Reads one events stream: collects up to <paramref name="maxEvents"/> matching events, then (cap reached) looks for
|
||||
/// one more matching event as the "more exist" signal and drains the rest within the EventDrainBudget.
|
||||
/// <paramref name="serverCap"/> > 0 is sent as <c>max_events</c>; 0 = no server cap.
|
||||
/// </summary>
|
||||
private async Task<EventDrain> DrainEventsAsync(string? sourceName, DateTime startUtc, DateTime endUtc, int maxEvents,
|
||||
int serverCap, CancellationToken cancellationToken)
|
||||
{
|
||||
var hasCap = maxEvents > 0;
|
||||
// Bounded initial capacity: an uncapped read arrives as int.MaxValue ("Array dimensions exceeded").
|
||||
var collected = new List<HistoricalEvent>(hasCap ? Math.Min(maxEvents, 1024) : 0);
|
||||
var truncated = false;
|
||||
long pastCap = 0; // stream rows consumed after the cap filled (matching or not)
|
||||
long received = 0;
|
||||
var abandoned = false;
|
||||
|
||||
await foreach (var wireEvent in _client
|
||||
.ReadEventsAsync(sourceName, startUtc, endUtc, serverCap, cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
received++;
|
||||
if (hasCap && collected.Count == maxEvents)
|
||||
{
|
||||
// The page is full: drain the rest of the stream without collecting it, within the budget.
|
||||
// One more MATCHING event is the "more events exist" signal; a foreign source's row is not, and
|
||||
// until the signal is seen no row counts toward the budget — abandoning before it would answer
|
||||
// "complete" for a page that may not be (and a spurious continuation point on a stretch of foreign
|
||||
// rows would make the server crawl it one resolution step per page).
|
||||
if (!truncated)
|
||||
{
|
||||
if (!EventSourceMatch.Matches(sourceName, wireEvent.SourceName))
|
||||
continue;
|
||||
truncated = true;
|
||||
}
|
||||
// Server-capped, the stream is already bounded at cap + 1 (by the gateway, or by the package client against an
|
||||
// older gateway): read it to its end so the call completes normally instead of being cancelled. Only a stream
|
||||
// running past the max_events it was sent is drained within the budget.
|
||||
if (serverCap > 0 && received <= serverCap) continue;
|
||||
if (++pastCap > _eventDrainBudget)
|
||||
{
|
||||
abandoned = true;
|
||||
break;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
var mapped = EventMapper.ToHistoricalEvent(wireEvent);
|
||||
|
||||
// Defensive client-side source filter: the gateway's SQL ReadEvents source filter
|
||||
// may not be present, so drop any event whose source does not match the request. The request's own
|
||||
// events and its attributes' (<source>.<attribute>, the name the gateway reports an AVEVA attribute
|
||||
// alarm under) match; a sibling that merely shares the prefix does not (#882, EventSourceMatch).
|
||||
if (!EventSourceMatch.Matches(sourceName, mapped.SourceName))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
collected.Add(mapped);
|
||||
}
|
||||
|
||||
if (abandoned)
|
||||
{
|
||||
Interlocked.Increment(ref _eventStreamsAbandoned);
|
||||
_logger.LogDebug(
|
||||
"HistorianGateway events read abandoned its stream {Budget} rows past the page cap (EventDrainBudget); " +
|
||||
"the gateway may evict its historian session.", _eventDrainBudget);
|
||||
}
|
||||
|
||||
return new EventDrain(collected, truncated, received);
|
||||
}
|
||||
|
||||
/// <summary>Whether a gateway failure is an <c>InvalidArgument</c> refusal (raw, or wrapped by the package client).</summary>
|
||||
private static bool IsInvalidArgument(Exception ex)
|
||||
{
|
||||
for (Exception? e = ex; e is not null; e = e.InnerException)
|
||||
if (e is RpcException { StatusCode: Grpc.Core.StatusCode.InvalidArgument }) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public HistorianHealthSnapshot GetHealthSnapshot()
|
||||
{
|
||||
|
||||
+1
-1
@@ -49,7 +49,7 @@ public static class GatewayHistorian
|
||||
maxRawValuesPerRead: options.EffectiveMaxRawValuesPerRead,
|
||||
// How far a capped events read drains its stream past the cap before abandoning it.
|
||||
eventDrainBudget: options.Gateway.EventDrainBudget,
|
||||
// AVEVA's native TimeAverage / Total — off until historiangw#27 is live-verified.
|
||||
// AVEVA's native TimeAverage / Total — on by default since historiangw#27 was live-verified (lmxopcua#694).
|
||||
enableNativeTimeWeighted: options.Gateway.EnableNativeTimeWeighted,
|
||||
// A slow ProbeAsync timer so the health snapshot reflects the gateway without waiting for a read.
|
||||
probeInterval: options.Gateway.ProbeInterval);
|
||||
|
||||
+6
-1
@@ -135,7 +135,8 @@ public sealed class HistorianGatewayClientAdapter : IHistorianGatewayClient, IDi
|
||||
Value = sourceName,
|
||||
};
|
||||
|
||||
return _inner.ReadEventsAsync(startUtc, endUtc, filter, ct);
|
||||
// max_events rides on the request (server-side cap, completes the stream normally); 0 = unlimited.
|
||||
return _inner.ReadEventsAsync(startUtc, endUtc, filter, Math.Max(0, maxEvents), ct);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -147,6 +148,10 @@ public sealed class HistorianGatewayClientAdapter : IHistorianGatewayClient, IDi
|
||||
public Task<WriteAck> SendEventAsync(HistorianEvent evt, CancellationToken ct) =>
|
||||
_inner.SendEventAsync(evt, ct);
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<SendEventsReply> SendEventsAsync(IReadOnlyList<HistorianEvent> events, CancellationToken ct) =>
|
||||
_inner.SendEventsAsync(events, ct);
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<TagOperationResults> EnsureTagsAsync(
|
||||
IReadOnlyList<HistorianTagDefinition> definitions, CancellationToken ct) =>
|
||||
|
||||
@@ -65,10 +65,11 @@ public interface IHistorianGatewayClient : IAsyncDisposable
|
||||
/// <param name="startUtc">Inclusive start of the time window (UTC).</param>
|
||||
/// <param name="endUtc">Exclusive end of the time window (UTC).</param>
|
||||
/// <param name="maxEvents">
|
||||
/// Caps the number of events returned. The gateway wire contract (<c>ReadEventsRequest</c>) has
|
||||
/// no per-call server cap, so this limit is enforced client-side by early stream termination:
|
||||
/// <c>0</c> or negative means no client-side limit (the gateway may still apply its configured
|
||||
/// <c>RuntimeDb:EventReadMaxRows</c>); a positive value stops draining after that many events.
|
||||
/// The most events to stream, sent as the server-side <c>ReadEventsRequest.max_events</c> (HistorianGateway
|
||||
/// Contracts/Client 0.4.0+): the gateway stops at that many and COMPLETES the stream normally, so a bounded read
|
||||
/// needs no cancellation. <c>0</c> or negative = unlimited (the gateway still applies its
|
||||
/// <c>Historian:ReadLimits:MaxEventsCeiling</c>). A gateway older than the field ignores it; the package client then
|
||||
/// stops reading at the cap itself, which abandons (cancels) the stream.
|
||||
/// </param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>An async stream of historian events within the window.</returns>
|
||||
@@ -95,6 +96,23 @@ public interface IHistorianGatewayClient : IAsyncDisposable
|
||||
/// <returns>The write acknowledgement returned by the gateway.</returns>
|
||||
Task<WriteAck> SendEventAsync(HistorianEvent evt, CancellationToken ct);
|
||||
|
||||
/// <summary>
|
||||
/// Sends a batch of historian events in one <c>SendEvents</c> RPC (HistorianGateway Contracts/Client 0.4.0+): one
|
||||
/// <see cref="EventWriteResult"/> per event, in request order. A systemic fault fails the whole call (events before it
|
||||
/// may already be committed); a request the gateway refuses up front (an event without <c>event_time</c>, a non-GUID
|
||||
/// id, a batch over its limits) is <c>InvalidArgument</c> with nothing sent.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The default implementation answers <c>Unimplemented</c> — what a gateway older than the RPC answers — so a seam
|
||||
/// implementation without a batch path (test doubles) makes callers fall back to <see cref="SendEventAsync"/>.
|
||||
/// </remarks>
|
||||
/// <param name="events">The events to send (non-empty).</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>The per-event results, in request order.</returns>
|
||||
Task<SendEventsReply> SendEventsAsync(IReadOnlyList<HistorianEvent> events, CancellationToken ct) =>
|
||||
Task.FromException<SendEventsReply>(new Grpc.Core.RpcException(
|
||||
new Grpc.Core.Status(Grpc.Core.StatusCode.Unimplemented, "SendEvents is not implemented by this gateway client.")));
|
||||
|
||||
/// <summary>Ensures the supplied tag definitions exist (create-or-update).</summary>
|
||||
/// <param name="definitions">The tag definitions to create or update.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
|
||||
@@ -37,13 +37,26 @@ namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway;
|
||||
/// row for is absent (not a fabricated <c>BadNoData</c>, unlike the SQL backends).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// ⚠️ Today historiangw#27 (the interval reaches AVEVA in ticks where it reads milliseconds) makes AVEVA answer ONE
|
||||
/// Good row per request — the whole-intervals read lands on its earliest interval with a whole-span value and the rest
|
||||
/// are absent — in both directions. The mapping needs no change once the gateway is fixed.
|
||||
/// <b>Whole milliseconds only on a shared grid.</b> The gateway sends AVEVA the interval in whole milliseconds
|
||||
/// (<c>max(1, round(ms))</c>), and AVEVA clips its last interval at the request's end (live, lmxopcua#694, after
|
||||
/// historiangw#27: a 0.4 ms window answers one row over exactly that 0.4 ms — <c>Integral</c> = value·0.0004 s — and a
|
||||
/// 1.4 ms window read at "1.4 ms" answers 1 ms rows). So a grid of several intervals is sent as one request only when
|
||||
/// the interval is a whole number of milliseconds; otherwise every interval is its own request over exactly its span
|
||||
/// (<see cref="MaxPerIntervalRequests"/> at most — more is <see cref="HistoryReadLimitExceededException"/>). A request
|
||||
/// for ONE interval (the clipped one, a window narrower than one interval, or each of those per-interval requests) is
|
||||
/// sent with its width rounded UP to whole milliseconds as the interval, so AVEVA answers exactly one row, clipped at
|
||||
/// the request's end: sent at its own width, a 3.0004 s interval reached AVEVA as 3,000 ms and answered a 3 s row plus
|
||||
/// a 0.4 ms one (live, lmxopcua#694 — the clipped interval's Total lost the 0.4 ms).
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal static class NativeAggregatePlan
|
||||
{
|
||||
/// <summary>
|
||||
/// The most intervals a native read whose interval is not a whole number of milliseconds may cover: each is its own
|
||||
/// gateway request.
|
||||
/// </summary>
|
||||
public const int MaxPerIntervalRequests = 1000;
|
||||
|
||||
/// <summary>One forward native request and the ascending bucket index of its first interval.</summary>
|
||||
/// <param name="StartUtc">Forward request start (inclusive).</param>
|
||||
/// <param name="EndUtc">Forward request end.</param>
|
||||
@@ -55,10 +68,32 @@ internal static class NativeAggregatePlan
|
||||
/// <summary>The forward requests serving a plan: the whole intervals first, then the clipped one (if any).</summary>
|
||||
/// <param name="plan">A plan with at least one bucket.</param>
|
||||
/// <returns>One or two forward requests.</returns>
|
||||
/// <exception cref="HistoryReadLimitExceededException">
|
||||
/// The interval is not a whole number of milliseconds and the window holds more than
|
||||
/// <see cref="MaxPerIntervalRequests"/> intervals.
|
||||
/// </exception>
|
||||
public static IReadOnlyList<Request> Requests(BucketPlan plan)
|
||||
{
|
||||
var interval = TimeSpan.FromTicks(plan.IntervalTicks);
|
||||
var n = plan.BucketCount;
|
||||
if (n > 1 && plan.IntervalTicks % TimeSpan.TicksPerMillisecond != 0)
|
||||
{
|
||||
// The gateway would round the interval to whole ms and AVEVA would answer on that grid, not the plan's: read
|
||||
// each interval over exactly its span instead (one interval per request is exact at any width).
|
||||
if (n > MaxPerIntervalRequests)
|
||||
throw new HistoryReadLimitExceededException(MaxPerIntervalRequests,
|
||||
$"A native TimeAverage/Total read whose interval is not a whole number of milliseconds is read one " +
|
||||
$"interval per gateway request, at most {MaxPerIntervalRequests}; this window holds {n}. Use a whole-millisecond " +
|
||||
"ProcessingInterval or a shorter window.");
|
||||
var each = new List<Request>((int)n);
|
||||
for (long i = 0; i < n; i++)
|
||||
{
|
||||
var lo = new DateTime(plan.LowerTicks(i), DateTimeKind.Utc);
|
||||
var hi = new DateTime(plan.UpperTicks(i), DateTimeKind.Utc);
|
||||
each.Add(new Request(lo, hi, OneInterval(hi - lo), i, 1));
|
||||
}
|
||||
return each;
|
||||
}
|
||||
var clipped = plan.IsTruncated(0) ? 0L : plan.IsTruncated(n - 1) ? n - 1 : -1L;
|
||||
if (clipped < 0) return [new Request(plan.StartUtc, plan.EndUtc, interval, 0, n)];
|
||||
|
||||
@@ -69,10 +104,23 @@ internal static class NativeAggregatePlan
|
||||
requests.Add(clipped == 0
|
||||
? new Request(clipHi, plan.EndUtc, interval, 1, n - 1) // reverse: the clipped one is the earliest
|
||||
: new Request(plan.StartUtc, clipLo, interval, 0, n - 1)); // forward: the clipped one is the latest
|
||||
requests.Add(new Request(clipLo, clipHi, clipHi - clipLo, clipped, 1));
|
||||
requests.Add(new Request(clipLo, clipHi, OneInterval(clipHi - clipLo), clipped, 1));
|
||||
return requests;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The interval to send for a request covering exactly one interval of <paramref name="width"/>: the width rounded UP
|
||||
/// to whole milliseconds (at least 1 ms), which the gateway sends unchanged, so AVEVA's single interval spans the whole
|
||||
/// request and is clipped at its end — never a whole-ms interval plus a sub-ms remainder row.
|
||||
/// </summary>
|
||||
/// <param name="width">The request's span (> 0).</param>
|
||||
/// <returns>The interval to send.</returns>
|
||||
internal static TimeSpan OneInterval(TimeSpan width)
|
||||
{
|
||||
var ms = Math.Max(1, (width.Ticks + TimeSpan.TicksPerMillisecond - 1) / TimeSpan.TicksPerMillisecond);
|
||||
return TimeSpan.FromTicks(ms * TimeSpan.TicksPerMillisecond);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Maps a request's rows onto the plan's buckets: rows outside the request window (AVEVA's pre-window row)
|
||||
/// are dropped; each in-window row is placed by its interval start (or, without one, its end).
|
||||
|
||||
Reference in New Issue
Block a user