using Grpc.Core; using Microsoft.Extensions.Logging.Abstractions; using Shouldly; using Xunit; using ZB.MOM.WW.HistorianGateway.Client; using ZB.MOM.WW.HistorianGateway.Contracts.Grpc; using ZB.MOM.WW.OtOpcUa.Core.Abstractions; using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian; namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests; /// /// lmxopcua#746: alarm-history writes go to the gateway as SendEvents batches (Contracts/Client 0.4.0+) — one /// round trip per events — with per-event outcomes in input /// order. A failed per-event result is classified by a single SendEvent re-send; a refused batch is sent event by /// event (a poison event dead-letters alone); a systemic fault retries the rest without sending more; an older gateway /// (Unimplemented) falls back to single sends for good. /// public sealed class GatewayAlarmWriterBatchTests { private static CancellationToken Ct => TestContext.Current.CancellationToken; private static AlarmHistorianEvent Evt(int i) => new( $"alarm-{i}", "Area/Pump", "N", "LimitAlarm", AlarmSeverity.High, "Activated", $"m{i}", "u", null, new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(i)); private static int MessageIndex(HistorianEvent e) => int.Parse(e.Properties["Message"][1..]); private static SendEventsReply AllOk(IReadOnlyList events) { var reply = new SendEventsReply(); foreach (var e in events) reply.Results.Add(new EventWriteResult { Id = e.Id, Success = true }); return reply; } [Fact] public async Task A_batch_goes_out_as_SendEvents_calls_of_at_most_the_batch_size_with_no_single_sends() { var fake = new FakeHistorianGatewayClient { SendEventsSource = (events, _) => Task.FromResult(AllOk(events)) }; using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger.Instance); const int n = 407; var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, n).Select(Evt).ToList(), Ct); outcomes.ShouldAllBe(o => o == HistorianWriteOutcome.Ack); outcomes.Count.ShouldBe(n); // 200 per call: well inside the gateway's 10,000-event / 16 MiB request limits and the 4 MiB gRPC message default. fake.SendEventsCalls.Select(c => c.Count).ShouldBe([200, 200, 7]); fake.SendEventsCalls.SelectMany(c => c).Select(MessageIndex).ShouldBe(Enumerable.Range(0, n), "request order = input order"); fake.SendEventCallCount.ShouldBe(0, "no per-event round trips when the batch RPC answers"); writer.BatchUnsupported.ShouldBeFalse(); } [Fact] public async Task Queued_results_ack_and_a_failed_result_is_classified_by_a_single_re_send() { var fake = new FakeHistorianGatewayClient { SendEventsSource = (events, _) => { var reply = new SendEventsReply(); foreach (var e in events) reply.Results.Add(MessageIndex(e) switch { 1 => new EventWriteResult { Id = e.Id, Success = true, Queued = true }, 2 or 3 => new EventWriteResult { Id = e.Id, Success = false, Error = "Historian did not accept the write." }, _ => new EventWriteResult { Id = e.Id, Success = true }, }); return Task.FromResult(reply); }, // The single re-send: event 2 is rejected by the historian (FailedPrecondition, as SendEvent answers a Failed // write), event 3 is transient. SendEventSource = (e, _) => MessageIndex(e) == 2 ? Task.FromException(new RpcException(new Status(StatusCode.FailedPrecondition, "rejected"))) : Task.FromException(new HistorianGatewayUnavailableException("down", new RpcException(new Status(StatusCode.Unavailable, "down")))), }; using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger.Instance); var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, 5).Select(Evt).ToList(), Ct); outcomes.ShouldBe([ HistorianWriteOutcome.Ack, HistorianWriteOutcome.Ack, HistorianWriteOutcome.PermanentFail, HistorianWriteOutcome.RetryPlease, HistorianWriteOutcome.Ack, ]); fake.SendEventsCalls.Count.ShouldBe(1); fake.SendEventCallCount.ShouldBe(2, "only the two failed events are re-sent"); } [Fact] public async Task A_refused_batch_is_sent_event_by_event_so_a_poison_event_dead_letters_alone() { var fake = new FakeHistorianGatewayClient { // The gateway validates the whole request first: one bad event refuses the batch, nothing sent. SendEventsSource = (_, _) => Task.FromException( new HistorianGatewayException("events[2].id must be empty or a GUID.", new RpcException(new Status(StatusCode.InvalidArgument, "events[2].id must be empty or a GUID.")))), SendEventSource = (e, _) => MessageIndex(e) == 2 ? Task.FromException(new RpcException(new Status(StatusCode.InvalidArgument, "id must be empty or a GUID."))) : Task.FromResult(new WriteAck { Success = true }), }; using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger.Instance); var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, 4).Select(Evt).ToList(), Ct); outcomes.ShouldBe([HistorianWriteOutcome.Ack, HistorianWriteOutcome.Ack, HistorianWriteOutcome.PermanentFail, HistorianWriteOutcome.Ack]); fake.SendEventCallCount.ShouldBe(4); writer.BatchUnsupported.ShouldBeFalse("a refused batch is not an older gateway — the next drain batches again"); } [Fact] public async Task A_systemic_fault_retries_the_rest_and_sends_nothing_more() { var calls = 0; var fake = new FakeHistorianGatewayClient { SendEventsSource = (events, _) => Interlocked.Increment(ref calls) == 1 ? Task.FromResult(AllOk(events)) : Task.FromException(new HistorianGatewayUnavailableException("down", new RpcException(new Status(StatusCode.Unavailable, "down")))), }; using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger.Instance); var size = GatewayAlarmHistorianWriter.MaxEventsPerBatch; var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, size * 3).Select(Evt).ToList(), Ct); outcomes.Take(size).ShouldAllBe(o => o == HistorianWriteOutcome.Ack); outcomes.Skip(size).ShouldAllBe(o => o == HistorianWriteOutcome.RetryPlease); calls.ShouldBe(2, "the third chunk would hit the same outage"); fake.SendEventCallCount.ShouldBe(0, "a systemic fault must not fan out into per-event sends"); } [Fact] public async Task An_older_gateway_falls_back_to_single_sends_for_good() { // No SendEventsSource: the fake answers Unimplemented, as a gateway older than the RPC. var fake = new FakeHistorianGatewayClient(); using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger.Instance); (await writer.WriteBatchAsync(Enumerable.Range(0, 5).Select(Evt).ToList(), Ct)).ShouldAllBe(o => o == HistorianWriteOutcome.Ack); (await writer.WriteBatchAsync(Enumerable.Range(5, 5).Select(Evt).ToList(), Ct)).ShouldAllBe(o => o == HistorianWriteOutcome.Ack); writer.BatchUnsupported.ShouldBeTrue(); fake.SendEventsCalls.Count.ShouldBe(1, "Unimplemented is remembered — the second drain batch goes straight to single sends"); fake.SendEventCallCount.ShouldBe(10); } [Fact] public async Task A_cancellation_during_the_batch_call_retries_and_dead_letters_nothing() { using var cts = new CancellationTokenSource(); var fake = new FakeHistorianGatewayClient { SendEventsSource = async (_, ct) => { await cts.CancelAsync(); ct.ThrowIfCancellationRequested(); return new SendEventsReply(); }, }; using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger.Instance); var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, GatewayAlarmHistorianWriter.MaxEventsPerBatch + 3).Select(Evt).ToList(), cts.Token); outcomes.ShouldAllBe(o => o == HistorianWriteOutcome.RetryPlease); fake.SendEventsCalls.Count.ShouldBe(1); fake.SendEventCallCount.ShouldBe(0); } [Fact] public async Task A_short_reply_classifies_the_missing_events_by_single_sends() { var fake = new FakeHistorianGatewayClient { SendEventsSource = (events, _) => Task.FromResult(AllOk(events.Take(2).ToList())), }; using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger.Instance); var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, 4).Select(Evt).ToList(), Ct); outcomes.ShouldAllBe(o => o == HistorianWriteOutcome.Ack); fake.SendEventCallCount.ShouldBe(2); } }