using Google.Protobuf.WellKnownTypes; using Microsoft.Extensions.Logging.Abstractions; using Shouldly; using Xunit; using ZB.MOM.WW.HistorianGateway.Contracts.Grpc; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests; /// /// The Gateway events read drains its stream past the cap (within a budget) instead of abandoning it — an abandoned /// gateway stream cancels the RPC and the gateway evicts its pooled historian session — and a caller cancellation on /// the events / native-AtTime arms is not a backend failure. /// public sealed class GatewayEventDrainTests { private static readonly DateTime T0 = new(2026, 10, 22, 8, 0, 0, DateTimeKind.Utc); private static CancellationToken Ct => TestContext.Current.CancellationToken; [Fact] public async Task A_capped_read_drains_the_stream_to_its_end_and_still_flags_truncation() { var fake = new FakeHistorianGatewayClient { Events = Series(50) }; var ds = new GatewayHistorianDataSource(fake, NullLogger.Instance); var r = await ds.ReadEventsAsync(null, T0, T0.AddHours(1), maxEvents: 10, Ct); r.Events.Select(e => e.EventId).ShouldBe(Enumerable.Range(0, 10).Select(i => $"E{i:000}")); r.ContinuationPoint.ShouldNotBeNull(); fake.EventStreamsCompleted.ShouldBe(1, "the stream must be consumed to its end, never broken out of"); fake.EventStreamsAbandoned.ShouldBe(0); fake.EventsYieldedCount.ShouldBe(50); ds.EventStreamsAbandoned.ShouldBe(0); } [Fact] public async Task Past_the_drain_budget_the_stream_is_abandoned_but_the_page_is_unchanged() { var fake = new FakeHistorianGatewayClient { Events = Series(100) }; var ds = new GatewayHistorianDataSource(fake, NullLogger.Instance, eventDrainBudget: 5); var r = await ds.ReadEventsAsync(null, T0, T0.AddHours(1), maxEvents: 10, Ct); r.Events.Count.ShouldBe(10); r.ContinuationPoint.ShouldNotBeNull(); fake.EventStreamsAbandoned.ShouldBe(1); fake.EventsYieldedCount.ShouldBe(10 + 5 + 1, "cap + budget, then the row that overran it"); ds.EventStreamsAbandoned.ShouldBe(1); } [Fact] public async Task Budget_zero_is_the_previous_abandon_at_the_cap() { var fake = new FakeHistorianGatewayClient { Events = Series(20) }; var ds = new GatewayHistorianDataSource(fake, NullLogger.Instance, eventDrainBudget: 0); var r = await ds.ReadEventsAsync(null, T0, T0.AddHours(1), maxEvents: 3, Ct); r.Events.Count.ShouldBe(3); r.ContinuationPoint.ShouldNotBeNull(); fake.EventsYieldedCount.ShouldBe(4); fake.EventStreamsAbandoned.ShouldBe(1); } [Fact] public async Task Foreign_source_rows_past_the_cap_are_drained_but_do_not_signal_truncation() { // The page is exactly the matching events; everything after it belongs to another source (an unfiltered gateway). var events = Series(3, "Pump1").Concat(Series(5, "Pump2", offsetSeconds: 10)).ToArray(); var fake = new FakeHistorianGatewayClient { Events = events }; var ds = new GatewayHistorianDataSource(fake, NullLogger.Instance); var r = await ds.ReadEventsAsync("Pump1", T0, T0.AddHours(1), maxEvents: 3, Ct); r.Events.Count.ShouldBe(3); r.ContinuationPoint.ShouldBeNull("no further MATCHING event exists"); fake.EventStreamsCompleted.ShouldBe(1); // Positive control: one more matching event at the end does signal truncation. fake.Events = events.Append(Ev(99, "Pump1")).ToArray(); (await ds.ReadEventsAsync("Pump1", T0, T0.AddHours(1), maxEvents: 3, Ct)).ContinuationPoint.ShouldNotBeNull(); } [Fact] public async Task A_cancelled_events_read_rethrows_and_is_not_a_backend_failure() { var fake = new FakeHistorianGatewayClient { Events = Series(5) }; var ds = new GatewayHistorianDataSource(fake, NullLogger.Instance); using var cts = new CancellationTokenSource(); await cts.CancelAsync(); await Should.ThrowAsync(() => ds.ReadEventsAsync(null, T0, T0.AddHours(1), 3, cts.Token)); var h = ds.GetHealthSnapshot(); h.TotalFailures.ShouldBe(0); h.ConsecutiveFailures.ShouldBe(0); // Positive control: a real fault on the same arm IS a failure. fake.ThrowOnRead = new InvalidOperationException("boom"); await Should.ThrowAsync(() => ds.ReadEventsAsync(null, T0, T0.AddHours(1), 3, Ct)); ds.GetHealthSnapshot().TotalFailures.ShouldBe(1); } [Fact] public async Task A_cancelled_native_at_time_read_rethrows_and_is_not_a_backend_failure() { using var cts = new CancellationTokenSource(); await cts.CancelAsync(); var fake = new FakeHistorianGatewayClient { ThrowOnRead = new OperationCanceledException(cts.Token) }; var ds = new GatewayHistorianDataSource(fake, NullLogger.Instance, atTimeFromRaw: false); await Should.ThrowAsync(() => ds.ReadAtTimeAsync("T", [T0], cts.Token)); ds.GetHealthSnapshot().TotalFailures.ShouldBe(0); // Positive control: the same arm counts a real fault. fake.ThrowOnRead = new InvalidOperationException("boom"); await Should.ThrowAsync(() => ds.ReadAtTimeAsync("T", [T0], Ct)); ds.GetHealthSnapshot().TotalFailures.ShouldBe(1); } /// /// The paged-vs-unpaged fuzz for the drain: random event series (bursts, tie clusters, foreign-source rows), random /// windows, caps and drain budgets, paged the way the server pages a forward events read (resume at the last event's /// time inclusive, skip the trailing ties emitted, ask cap + skip; a resumed page stalled on a tie cluster /// over-fetches the cluster and pages within it) against a gateway-faithful fake (ascending time, ties in store /// order, inclusive end, optionally no server-side source filter). The concatenated pages must equal the one-shot /// uncapped read exactly — boundary ties included — and every stream must be consumed to its end whenever the /// budget covers the overshoot. /// [Fact] public async Task Paged_events_equal_the_unpaged_read_and_drain_within_budget() { var iterations = int.TryParse(Environment.GetEnvironmentVariable("OTOPCUA_PAGING_FUZZ_ITERATIONS"), out var n) && n > 0 ? Math.Min(n, 4000) : 1500; var occurred = new Dictionary(); void Count(string k) => occurred[k] = occurred.GetValueOrDefault(k) + 1; for (var seed = 0; seed < iterations; seed++) { var rng = new Random(seed); var store = RandomStore(rng, out var source); var cap = rng.Next(6) switch { 0 => 1, 1 => 2, 2 => 3, 3 => rng.Next(4, 12), 4 => rng.Next(12, 60), _ => 500 }; var budget = rng.Next(4) switch { 0 => 0, 1 => rng.Next(1, 8), _ => GatewayHistorianDataSource.DefaultEventDrainBudget }; var (start, end) = RandomWindow(rng, store); var tag = $"seed {seed} cap {cap} budget {budget} source {source ?? "*"} [{start:O} .. {end:O}]"; var truthFake = new FakeHistorianGatewayClient { EventsSource = (s, e) => Window(store, s, e) }; var truthDs = new GatewayHistorianDataSource(truthFake, NullLogger.Instance); var truth = (await truthDs.ReadEventsAsync(source, start, end, 0, Ct)).Events; truth.ShouldAllBe(e => source == null || e.SourceName == source, tag); var fake = new FakeHistorianGatewayClient { EventsSource = (s, e) => Window(store, s, e) }; var ds = new GatewayHistorianDataSource(fake, NullLogger.Instance, eventDrainBudget: budget); var paged = await PageAsync(ds, source, start, end, cap, tag); paged.Select(Key).ShouldBe(truth.Select(Key), $"paged ≠ unpaged: {tag}"); if (budget >= GatewayHistorianDataSource.DefaultEventDrainBudget) { fake.EventStreamsAbandoned.ShouldBe(0, $"a stream was abandoned within budget: {tag}"); Count("drained"); } if (fake.EventStreamsAbandoned > 0) Count("abandoned past budget"); if (truth.Count == 0) Count("empty"); if (truth.Count > cap) Count("paged"); if (truth.GroupBy(e => e.EventTimeUtc).Any(g => g.Count() > cap)) Count("tie cluster over the cap"); if (source is not null && store.Any(e => e.SourceName != source)) Count("foreign rows"); } foreach (var k in new[] { "drained", "abandoned past budget", "empty", "paged", "tie cluster over the cap", "foreign rows" }) occurred.GetValueOrDefault(k).ShouldBeGreaterThan(0, $"the fuzz never produced '{k}' — it is vacuous for that case"); } /// The server's forward events paging, against the data source (see the fuzz summary). private static async Task> PageAsync( IHistorianDataSource ds, string? source, DateTime start, DateTime end, int cap, string tag) { var all = new List(); var skip = 0; var resumed = false; for (var page = 0; ; page++) { page.ShouldBeLessThan(10_000, $"paging must terminate: {tag}"); var r = await ds.ReadEventsAsync(source, start, end, cap + skip, Ct); var truncated = r.ContinuationPoint is not null; var events = r.Events.ToList(); if (resumed && skip > 0) { var i = 0; while (i < events.Count && i < skip && events[i].EventTimeUtc == start) i++; events.RemoveRange(0, i); } if (events.Count > cap) { events.RemoveRange(cap, events.Count - cap); truncated = true; } if (truncated && resumed && (events.Count == 0 || events[^1].EventTimeUtc == start)) { // Oversized tie cluster: read it whole ([T, T] on the gateway's inclusive end) and page within it. var cluster = (await ds.ReadEventsAsync(source, start, start, 100_000, Ct)).Events .Where(e => e.EventTimeUtc == start).ToList(); var slice = cluster.Skip(skip).Take(cap).ToList(); all.AddRange(slice); if (skip + slice.Count < cluster.Count) { skip += slice.Count; continue; } start = start.AddTicks(1); skip = 0; if (start > end) return all; continue; } all.AddRange(events); if (!truncated || events.Count == 0) return all; var boundary = events[^1].EventTimeUtc; var ties = 0; for (var i = events.Count - 1; i >= 0 && events[i].EventTimeUtc == boundary; i--) ties++; // A resumed page landing on the same boundary carries its earlier skip forward. skip = resumed && boundary == start ? skip + ties : ties; start = boundary; resumed = true; } } private static string Key(HistoricalEvent e) => $"{e.EventTimeUtc.Ticks}:{e.EventId}"; private static List RandomStore(Random rng, out string? source) { var count = rng.Next(5) == 0 ? 0 : rng.Next(1, 120); var mixed = rng.Next(3) == 0; source = rng.Next(3) == 0 ? null : "Src"; var t = T0; var store = new List(); for (var i = 0; i < count; i++) { // Bursts: often several events share one timestamp; sometimes a big cluster. var step = rng.Next(6) switch { 0 or 1 => 0, 2 => 1, 3 => rng.Next(1, 1000), _ => rng.Next(1, 10_000_000) }; t = t.AddTicks(step); var src = mixed && rng.Next(3) == 0 ? "Other" : "Src"; store.Add(new HistorianEvent { Id = $"E{i:0000}", SourceName = src, EventTime = Timestamp.FromDateTime(t), ReceivedTime = Timestamp.FromDateTime(t), }); if (rng.Next(25) == 0) for (var k = 0; k < rng.Next(5, 40); k++) store.Add(new HistorianEvent { Id = $"E{i:0000}-{k:00}", SourceName = src, EventTime = Timestamp.FromDateTime(t), ReceivedTime = Timestamp.FromDateTime(t), }); } return store; } private static (DateTime Start, DateTime End) RandomWindow(Random rng, List store) { if (store.Count == 0 || rng.Next(6) == 0) return (T0, T0.AddDays(2)); DateTime Pick() => rng.Next(3) == 0 ? T0.AddTicks(rng.NextInt64(0, 2_000_000_000)) : store[rng.Next(store.Count)].EventTime.ToDateTime(); var a = Pick(); var b = Pick(); return a <= b ? (a, b) : (b, a); } /// The gateway's ReadEvents window: ascending time, ties in store order, inclusive at both ends. private static IEnumerable Window(List store, DateTime start, DateTime end) => store.Where(e => e.EventTime.ToDateTime() >= start && e.EventTime.ToDateTime() <= end) .Select((e, i) => (e, i)).OrderBy(x => x.e.EventTime.ToDateTime()).ThenBy(x => x.i).Select(x => x.e); private static HistorianEvent[] Series(int count, string source = "Pump1", int offsetSeconds = 0) => Enumerable.Range(0, count).Select(i => Ev(i + offsetSeconds, source, $"E{i:000}")).ToArray(); private static HistorianEvent Ev(int seconds, string source, string? id = null) => new() { Id = id ?? $"X{seconds}", SourceName = source, EventTime = Timestamp.FromDateTime(T0.AddSeconds(seconds)), ReceivedTime = Timestamp.FromDateTime(T0.AddSeconds(seconds)), }; }