using Grpc.Core;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
using Xunit;
using ZB.MOM.WW.HistorianGateway.Client;
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
///
/// lmxopcua#746: alarm-history writes go to the gateway as SendEvents batches (Contracts/Client 0.4.0+) — one
/// round trip per events — with per-event outcomes in input
/// order. A failed per-event result is classified by a single SendEvent re-send; a refused batch is sent event by
/// event (a poison event dead-letters alone); a systemic fault retries the rest without sending more; an older gateway
/// (Unimplemented) falls back to single sends for good.
///
public sealed class GatewayAlarmWriterBatchTests
{
private static CancellationToken Ct => TestContext.Current.CancellationToken;
private static AlarmHistorianEvent Evt(int i) => new(
$"alarm-{i}", "Area/Pump", "N", "LimitAlarm", AlarmSeverity.High, "Activated", $"m{i}", "u", null,
new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(i));
private static int MessageIndex(HistorianEvent e) => int.Parse(e.Properties["Message"][1..]);
private static SendEventsReply AllOk(IReadOnlyList events)
{
var reply = new SendEventsReply();
foreach (var e in events) reply.Results.Add(new EventWriteResult { Id = e.Id, Success = true });
return reply;
}
[Fact]
public async Task A_batch_goes_out_as_SendEvents_calls_of_at_most_the_batch_size_with_no_single_sends()
{
var fake = new FakeHistorianGatewayClient { SendEventsSource = (events, _) => Task.FromResult(AllOk(events)) };
using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger.Instance);
const int n = 407;
var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, n).Select(Evt).ToList(), Ct);
outcomes.ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
outcomes.Count.ShouldBe(n);
// 200 per call: well inside the gateway's 10,000-event / 16 MiB request limits and the 4 MiB gRPC message default.
fake.SendEventsCalls.Select(c => c.Count).ShouldBe([200, 200, 7]);
fake.SendEventsCalls.SelectMany(c => c).Select(MessageIndex).ShouldBe(Enumerable.Range(0, n), "request order = input order");
fake.SendEventCallCount.ShouldBe(0, "no per-event round trips when the batch RPC answers");
writer.BatchUnsupported.ShouldBeFalse();
}
[Fact]
public async Task Queued_results_ack_and_a_failed_result_is_classified_by_a_single_re_send()
{
var fake = new FakeHistorianGatewayClient
{
SendEventsSource = (events, _) =>
{
var reply = new SendEventsReply();
foreach (var e in events)
reply.Results.Add(MessageIndex(e) switch
{
1 => new EventWriteResult { Id = e.Id, Success = true, Queued = true },
2 or 3 => new EventWriteResult { Id = e.Id, Success = false, Error = "Historian did not accept the write." },
_ => new EventWriteResult { Id = e.Id, Success = true },
});
return Task.FromResult(reply);
},
// The single re-send: event 2 is rejected by the historian (FailedPrecondition, as SendEvent answers a Failed
// write), event 3 is transient.
SendEventSource = (e, _) => MessageIndex(e) == 2
? Task.FromException(new RpcException(new Status(StatusCode.FailedPrecondition, "rejected")))
: Task.FromException(new HistorianGatewayUnavailableException("down", new RpcException(new Status(StatusCode.Unavailable, "down")))),
};
using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger.Instance);
var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, 5).Select(Evt).ToList(), Ct);
outcomes.ShouldBe([
HistorianWriteOutcome.Ack, HistorianWriteOutcome.Ack, HistorianWriteOutcome.PermanentFail,
HistorianWriteOutcome.RetryPlease, HistorianWriteOutcome.Ack,
]);
fake.SendEventsCalls.Count.ShouldBe(1);
fake.SendEventCallCount.ShouldBe(2, "only the two failed events are re-sent");
}
[Fact]
public async Task A_refused_batch_is_sent_event_by_event_so_a_poison_event_dead_letters_alone()
{
var fake = new FakeHistorianGatewayClient
{
// The gateway validates the whole request first: one bad event refuses the batch, nothing sent.
SendEventsSource = (_, _) => Task.FromException(
new HistorianGatewayException("events[2].id must be empty or a GUID.",
new RpcException(new Status(StatusCode.InvalidArgument, "events[2].id must be empty or a GUID.")))),
SendEventSource = (e, _) => MessageIndex(e) == 2
? Task.FromException(new RpcException(new Status(StatusCode.InvalidArgument, "id must be empty or a GUID.")))
: Task.FromResult(new WriteAck { Success = true }),
};
using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger.Instance);
var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, 4).Select(Evt).ToList(), Ct);
outcomes.ShouldBe([HistorianWriteOutcome.Ack, HistorianWriteOutcome.Ack, HistorianWriteOutcome.PermanentFail, HistorianWriteOutcome.Ack]);
fake.SendEventCallCount.ShouldBe(4);
writer.BatchUnsupported.ShouldBeFalse("a refused batch is not an older gateway — the next drain batches again");
}
[Fact]
public async Task A_systemic_fault_retries_the_rest_and_sends_nothing_more()
{
var calls = 0;
var fake = new FakeHistorianGatewayClient
{
SendEventsSource = (events, _) => Interlocked.Increment(ref calls) == 1
? Task.FromResult(AllOk(events))
: Task.FromException(new HistorianGatewayUnavailableException("down",
new RpcException(new Status(StatusCode.Unavailable, "down")))),
};
using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger.Instance);
var size = GatewayAlarmHistorianWriter.MaxEventsPerBatch;
var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, size * 3).Select(Evt).ToList(), Ct);
outcomes.Take(size).ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
outcomes.Skip(size).ShouldAllBe(o => o == HistorianWriteOutcome.RetryPlease);
calls.ShouldBe(2, "the third chunk would hit the same outage");
fake.SendEventCallCount.ShouldBe(0, "a systemic fault must not fan out into per-event sends");
}
[Fact]
public async Task An_older_gateway_falls_back_to_single_sends_for_good()
{
// No SendEventsSource: the fake answers Unimplemented, as a gateway older than the RPC.
var fake = new FakeHistorianGatewayClient();
using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger.Instance);
(await writer.WriteBatchAsync(Enumerable.Range(0, 5).Select(Evt).ToList(), Ct)).ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
(await writer.WriteBatchAsync(Enumerable.Range(5, 5).Select(Evt).ToList(), Ct)).ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
writer.BatchUnsupported.ShouldBeTrue();
fake.SendEventsCalls.Count.ShouldBe(1, "Unimplemented is remembered — the second drain batch goes straight to single sends");
fake.SendEventCallCount.ShouldBe(10);
}
[Fact]
public async Task A_cancellation_during_the_batch_call_retries_and_dead_letters_nothing()
{
using var cts = new CancellationTokenSource();
var fake = new FakeHistorianGatewayClient
{
SendEventsSource = async (_, ct) =>
{
await cts.CancelAsync();
ct.ThrowIfCancellationRequested();
return new SendEventsReply();
},
};
using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger.Instance);
var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, GatewayAlarmHistorianWriter.MaxEventsPerBatch + 3).Select(Evt).ToList(), cts.Token);
outcomes.ShouldAllBe(o => o == HistorianWriteOutcome.RetryPlease);
fake.SendEventsCalls.Count.ShouldBe(1);
fake.SendEventCallCount.ShouldBe(0);
}
[Fact]
public async Task A_short_reply_classifies_the_missing_events_by_single_sends()
{
var fake = new FakeHistorianGatewayClient
{
SendEventsSource = (events, _) => Task.FromResult(AllOk(events.Take(2).ToList())),
};
using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger.Instance);
var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, 4).Select(Evt).ToList(), Ct);
outcomes.ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
fake.SendEventCallCount.ShouldBe(2);
}
}