Files
lmxopcua/tests/Drivers/ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests/GatewayNativeWholeMillisecondGridTests.cs
T
Joseph Doherty 53978ca638 feat(historian-gateway): native TimeAverage/Total on by default, SendEvents batches, server-side max_events (#694, #746)
#694 item 3 — historiangw#27 is fixed and now live-verified through OtOpcUa:
- ServerHistorian:Gateway:EnableNativeTimeWeighted defaults to true (false is
  the kill switch for a gateway older than 9f711c1). TimeAverage / Total are
  served and advertised again (12 aggregates).
- The gateway sends the interval as max(1, round(ms)) and AVEVA clips its last
  interval at the request end (live). NativeAggregatePlan therefore sends a
  multi-interval grid only for whole-ms intervals; a fractional interval is read
  one interval per request (at most 1,000, else BadTooManyOperations), and every
  one-interval request goes out rounded UP to whole ms so AVEVA answers one
  clipped row. Live, a 3.0004 s clipped interval sent at its own width lost its
  last 0.4 ms (Total 123 vs 123.0172).

#746 — HistorianGateway Contracts/Client 1.0.0 surface:
- Alarm history: GatewayAlarmHistorianWriter sends SendEvents batches of <=200.
  A failed per-event result is re-sent alone via SendEvent (classified as
  before); a refused batch (InvalidArgument etc.) goes event by event so a
  poison event dead-letters alone; a systemic fault retries the rest without
  sending more; Unimplemented (older gateway) falls back to SendEvent for good.
  IHistorianGatewayClient.SendEventsAsync defaults to Unimplemented.
- Events reads: a capped page sends cap + 1 as ReadEventsRequest.max_events, so
  the gateway completes the stream instead of the client cancelling it. An
  ambiguous capped answer (foreign rows filtered out of the server's cut) or a
  refused cap is re-read uncapped with the existing EventDrainBudget drain.

Tests: GatewayServerCappedEventsTests, GatewayAlarmWriterBatchTests,
GatewayNativeWholeMillisecondGridTests (gateway-faithful rounding fake), gate
tests flipped to the new default; every new test mutation-checked (each
mutation of the production change fails at least one). Live harness
AvevaGatewayHistorianFollowupsLiveGateTests (read-only) for #694/#700/#746.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
2026-11-13 00:09:34 -05:00

181 lines
9.3 KiB
C#

using Google.Protobuf.WellKnownTypes;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
using Xunit;
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
/// <summary>
/// lmxopcua#694: the gateway sends AVEVA the native interval in WHOLE milliseconds (<c>max(1, round(ms))</c>) and AVEVA
/// clips its last interval at the request's end — both observed live (a 1.5 ms interval answers on a 2 ms grid; a 0.4 ms
/// window answers one row over exactly 0.4 ms). So a multi-interval request is only correct for a whole-millisecond
/// interval: a fractional one is read one interval per request, up to
/// <see cref="NativeAggregatePlan.MaxPerIntervalRequests"/>, and every one-interval request is sent with its width
/// rounded UP to whole milliseconds so AVEVA answers it as ONE clipped row. The fake models exactly that gateway + AVEVA
/// behaviour (the gateway's round-half-to-even included) over a step-held series.
/// </summary>
public sealed class GatewayNativeWholeMillisecondGridTests
{
private static readonly CancellationToken Ct = CancellationToken.None;
private static readonly DateTime T0 = new(2026, 11, 11, 14, 2, 3, DateTimeKind.Utc);
private const uint GoodCalc = 0x00000401u;
private const uint Partial = 0x4u;
/// <summary>A value that changes every 0.5 ms (1, 2, 3, …): any wrong grid shows in the values.</summary>
private static double ValueAt(DateTime t) => 1 + Math.Floor((t - T0).Ticks / 5000.0);
private static double Integral(DateTime a, DateTime b)
{
double sum = 0;
for (var t = a; t < b;)
{
var next = T0.AddTicks(((t - T0).Ticks / 5000 + 1) * 5000);
if (next > b) next = b;
sum += ValueAt(t) * (next - t).TotalSeconds;
t = next;
}
return sum;
}
private static double Expected(RetrievalMode mode, DateTime a, DateTime b) =>
mode == RetrievalMode.Integral ? Integral(a, b) : Integral(a, b) / (b - a).TotalSeconds;
/// <summary>The gateway (whole-ms interval) in front of AVEVA (grid anchored at the request start, last interval clipped).</summary>
private static FakeHistorianGatewayClient GatewayAndAveva() => new()
{
AggregateSource = (s, e, mode, interval) =>
{
var wire = TimeSpan.FromMilliseconds(Math.Max(1, Math.Round(interval.TotalMilliseconds)));
var rows = new List<HistorianAggregateSample>();
for (var lo = s - wire; lo < e; lo += wire) // AVEVA's extra pre-window row first
{
var hi = lo + wire > e && lo >= s ? e : lo + wire;
rows.Add(new HistorianAggregateSample
{
Tag = "T",
StartTime = Timestamp.FromDateTime(lo),
EndTime = Timestamp.FromDateTime(hi),
Value = Expected(mode, lo, hi),
OpcQuality = 192,
RetrievalMode = mode,
});
}
return rows;
},
};
private static GatewayHistorianDataSource Source(FakeHistorianGatewayClient fake) =>
new(fake, NullLogger<GatewayHistorianDataSource>.Instance);
[Theory]
[InlineData(HistoryAggregateType.TimeAverage, RetrievalMode.TimeWeightedAverage)]
[InlineData(HistoryAggregateType.Total, RetrievalMode.Integral)]
public async Task A_fractional_millisecond_interval_is_read_one_interval_per_request(HistoryAggregateType agg, RetrievalMode mode)
{
var fake = GatewayAndAveva();
var interval = TimeSpan.FromTicks(15_000); // 1.5 ms — the gateway would send 2 ms
var s = (await Source(fake).ReadProcessedAsync("T", T0, T0.AddMilliseconds(6), interval, agg, Ct)).Samples;
s.Select(x => x.SourceTimestampUtc).ShouldBe(new DateTime?[] { T0, T0.AddTicks(15_000), T0.AddTicks(30_000), T0.AddTicks(45_000) });
s.Select(x => (double)x.Value!).ShouldBe(
Enumerable.Range(0, 4).Select(i => Expected(mode, T0.AddTicks(i * 15_000L), T0.AddTicks((i + 1) * 15_000L))).ToList(),
tolerance: 1e-9);
s.ShouldAllBe(x => x.StatusCode == GoodCalc);
fake.AggregateCalls.Select(c => (c.Start, c.End)).ShouldBe(
Enumerable.Range(0, 4).Select(i => (T0.AddTicks(i * 15_000L), T0.AddTicks((i + 1) * 15_000L))).ToList());
}
[Fact]
public async Task A_fractional_interval_reverse_read_is_exact_too()
{
var fake = GatewayAndAveva();
var m = RetrievalMode.TimeWeightedAverage;
var s = (await Source(fake).ReadProcessedAsync("T", T0.AddTicks(40_000), T0, TimeSpan.FromTicks(15_000),
HistoryAggregateType.TimeAverage, Ct)).Samples;
// (0, 4 ms] stepping back from 4 ms: (2.5, 4], (1, 2.5], and the clipped (0, 1] — newest first, later edges.
s.Select(x => x.SourceTimestampUtc).ShouldBe(new DateTime?[] { T0.AddTicks(40_000), T0.AddTicks(25_000), T0.AddTicks(10_000) });
s.Select(x => (double)x.Value!).ShouldBe(
[Expected(m, T0.AddTicks(25_000), T0.AddTicks(40_000)), Expected(m, T0.AddTicks(10_000), T0.AddTicks(25_000)),
Expected(m, T0, T0.AddTicks(10_000))],
tolerance: 1e-9);
s.Select(x => x.StatusCode).ShouldBe([GoodCalc, GoodCalc, GoodCalc | Partial]);
fake.ReadAggregateCallCount.ShouldBe(3);
}
[Fact]
public async Task A_whole_millisecond_interval_stays_one_request_with_a_sub_millisecond_clipped_interval()
{
var fake = GatewayAndAveva();
var m = RetrievalMode.Integral;
var s = (await Source(fake).ReadProcessedAsync("T", T0, T0.AddTicks(30_004), TimeSpan.FromMilliseconds(1),
HistoryAggregateType.Total, Ct)).Samples;
s.Count.ShouldBe(4);
s.Select(x => (double)x.Value!).ShouldBe(
[Expected(m, T0, T0.AddMilliseconds(1)), Expected(m, T0.AddMilliseconds(1), T0.AddMilliseconds(2)),
Expected(m, T0.AddMilliseconds(2), T0.AddMilliseconds(3)), Expected(m, T0.AddMilliseconds(3), T0.AddTicks(30_004))],
tolerance: 1e-12);
s[^1].StatusCode.ShouldBe(GoodCalc | Partial);
fake.AggregateCalls.Select(c => (c.Start, c.End, c.Interval)).ShouldBe([
(T0, T0.AddMilliseconds(3), TimeSpan.FromMilliseconds(1)),
(T0.AddMilliseconds(3), T0.AddTicks(30_004), TimeSpan.FromMilliseconds(1)), // the 0.4 µs width, rounded UP to 1 ms
]);
}
[Theory]
[InlineData(HistoryAggregateType.TimeAverage, RetrievalMode.TimeWeightedAverage)]
[InlineData(HistoryAggregateType.Total, RetrievalMode.Integral)]
public async Task A_clipped_interval_that_is_not_whole_milliseconds_is_one_row_over_its_whole_span(HistoryAggregateType agg,
RetrievalMode mode)
{
// Live (lmxopcua#694): [40 s, 43.0004 s) sent at its own width reached AVEVA as 3,000 ms and answered a 3 s row plus a
// 0.4 ms one; the first won, so the clipped Total lost its last 0.4 ms. Here: 4 ms intervals, a 2.0004 ms clipped one.
var fake = GatewayAndAveva();
var end = T0.AddTicks(60_004);
var s = (await Source(fake).ReadProcessedAsync("T", T0, end, TimeSpan.FromMilliseconds(4), agg, Ct)).Samples;
s.Count.ShouldBe(2);
((double)s[1].Value!).ShouldBe(Expected(mode, T0.AddMilliseconds(4), end), 1e-12);
s[1].StatusCode.ShouldBe(GoodCalc | Partial);
fake.AggregateCalls[^1].ShouldBe((T0.AddMilliseconds(4), end, mode, TimeSpan.FromMilliseconds(3)));
}
[Fact]
public async Task A_half_millisecond_interval_rounding_to_even_still_reads_each_interval_whole()
{
// 2.5 ms rounds (to even) to 2 ms on the gateway: sent at its width, each interval would answer a 2 ms row + 0.5 ms.
var fake = GatewayAndAveva();
var m = RetrievalMode.Integral;
var s = (await Source(fake).ReadProcessedAsync("T", T0, T0.AddMilliseconds(5), TimeSpan.FromTicks(25_000),
HistoryAggregateType.Total, Ct)).Samples;
s.Select(x => (double)x.Value!).ShouldBe(
[Expected(m, T0, T0.AddTicks(25_000)), Expected(m, T0.AddTicks(25_000), T0.AddMilliseconds(5))], tolerance: 1e-12);
fake.AggregateCalls.Select(c => c.Interval).ShouldAllBe(i => i == TimeSpan.FromMilliseconds(3));
}
[Fact]
public async Task Too_many_fractional_intervals_are_a_request_limit_unsent_and_not_a_backend_failure()
{
var fake = GatewayAndAveva();
var ds = Source(fake);
var interval = TimeSpan.FromTicks(15_000);
var end = T0.AddTicks(15_000L * (NativeAggregatePlan.MaxPerIntervalRequests + 1));
var ex = await Should.ThrowAsync<HistoryReadLimitExceededException>(
() => ds.ReadProcessedAsync("T", T0, end, interval, HistoryAggregateType.TimeAverage, Ct));
ex.Limit.ShouldBe(NativeAggregatePlan.MaxPerIntervalRequests);
fake.ReadAggregateCallCount.ShouldBe(0);
fake.TagExistsCallCount.ShouldBe(0);
ds.GetHealthSnapshot().TotalFailures.ShouldBe(0);
// Positive control: exactly the limit is served.
var ok = await ds.ReadProcessedAsync("T", T0, T0.AddTicks(15_000L * NativeAggregatePlan.MaxPerIntervalRequests), interval,
HistoryAggregateType.TimeAverage, Ct);
ok.Samples.Count.ShouldBe(NativeAggregatePlan.MaxPerIntervalRequests);
}
}