perf(historian): batch alarm-history writes; pipeline gateway SendEvent (lmxopcua#746)

All three alarm-history writers inserted one event per round trip, serially
(SqlServer also opened a pooled connection per event), so a 10000-event
alarm storm drained at 10000 x RTT while the backlog sat in LocalDb.

- Sql.Common.AlarmBatchWrite: one statement per chunk; a chunk that fails
  PermanentFail-class is re-sent event by event so only the poison event
  dead-letters; a transient failure retries the chunk; a cancellation never
  dead-letters.
- SqlServer: multi-row insert-if-absent over a VALUES set (150 events per
  statement, under the 2100-parameter limit), duplicate ids in a batch
  written once; a duplicate-key race isolates per event, where it is Ack.
  No new table type, so no schema migration (the review suggested a TVP;
  the VALUES form gets the same one-round-trip-per-chunk without one).
- QuestDb: multi-row INSERTs of 500 events; pre-1970 events still
  dead-letter unsent.
- Gateway: no batch RPC exists, so sends are pipelined, at most 8 in
  flight, outcomes kept in input order.

Rebased from the unpushed remed/lane-H2 branch (originally authored 2026-10-22
in session_01Ub9hGmYLQvn9XETzw7CMsT) onto current master.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-10-22 16:54:37 -04:00
parent ee314d1b9e
commit 74afc3f90a
10 changed files with 503 additions and 89 deletions
@@ -227,8 +227,16 @@ public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
: Task.FromResult(WriteLiveValuesResult);
}
/// <summary>When set, SendEvent is served by it (an asynchronous, observable gateway — concurrency tests).</summary>
public Func<HistorianEvent, CancellationToken, Task<WriteAck>>? SendEventSource;
public Task<WriteAck> SendEventAsync(HistorianEvent evt, CancellationToken ct)
{
if (SendEventSource is { } source)
{
Interlocked.Increment(ref SendEventCallCount);
return source(evt, ct);
}
LastSendEvent = evt;
SendEventCallCount++;
return SendEventThrows is not null
@@ -0,0 +1,101 @@
using Grpc.Core;
using Microsoft.Extensions.Logging.Abstractions;
using Shouldly;
using Xunit;
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
/// <summary>
/// The gateway has no batch SendEvent RPC, so the alarm writer pipelines its per-event sends — at most
/// <see cref="GatewayAlarmHistorianWriter.MaxSendsInFlight"/> at once — instead of one round trip after another. Every
/// event still gets its own outcome, in input order.
/// </summary>
public sealed class GatewayAlarmWriterPipeliningTests
{
private static CancellationToken Ct => TestContext.Current.CancellationToken;
private static AlarmHistorianEvent Evt(int i) => new(
$"alarm-{i}", "Area/Pump", "N", "LimitAlarm", AlarmSeverity.High, "Activated", $"m{i}", "u", null,
new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(i));
[Fact]
public async Task Sends_overlap_up_to_the_bound_and_outcomes_keep_input_order()
{
var inFlight = 0;
var maxInFlight = 0;
var fake = new FakeHistorianGatewayClient
{
SendEventSource = async (e, ct) =>
{
var now = Interlocked.Increment(ref inFlight);
InterlockedMax(ref maxInFlight, now);
await Task.Delay(20, ct);
Interlocked.Decrement(ref inFlight);
// Odd messages are soft failures: their outcome must land on their own slot.
return e.Properties.TryGetValue("Message", out var m) && int.Parse(m[1..]) % 2 == 1
? new WriteAck { Success = false }
: new WriteAck { Success = true };
},
};
using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger<GatewayAlarmHistorianWriter>.Instance);
var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, 40).Select(Evt).ToList(), Ct);
outcomes.Count.ShouldBe(40);
for (var i = 0; i < 40; i++)
outcomes[i].ShouldBe(i % 2 == 1 ? HistorianWriteOutcome.RetryPlease : HistorianWriteOutcome.Ack, $"event {i}");
maxInFlight.ShouldBeGreaterThan(1, "the sends must overlap — a sequential drain is 40 round trips");
maxInFlight.ShouldBeLessThanOrEqualTo(GatewayAlarmHistorianWriter.MaxSendsInFlight);
fake.SendEventCallCount.ShouldBe(40);
}
[Fact]
public async Task A_cancellation_mid_batch_sends_nothing_more_and_dead_letters_nothing()
{
using var cts = new CancellationTokenSource();
var sent = 0;
var fake = new FakeHistorianGatewayClient
{
SendEventSource = async (_, ct) =>
{
if (Interlocked.Increment(ref sent) == 3) await cts.CancelAsync();
await Task.Delay(10, ct);
return new WriteAck { Success = true };
},
};
using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger<GatewayAlarmHistorianWriter>.Instance);
var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, 40).Select(Evt).ToList(), cts.Token);
outcomes.Count.ShouldBe(40);
outcomes.ShouldNotContain(HistorianWriteOutcome.PermanentFail);
outcomes.Count(o => o == HistorianWriteOutcome.RetryPlease).ShouldBeGreaterThan(0);
sent.ShouldBeLessThanOrEqualTo(GatewayAlarmHistorianWriter.MaxSendsInFlight + 2, "no new send starts after the cancel");
}
[Fact]
public async Task A_poison_event_dead_letters_alone()
{
var fake = new FakeHistorianGatewayClient
{
SendEventSource = (e, _) => e.Properties.TryGetValue("Message", out var m) && m == "m5"
? Task.FromException<WriteAck>(new RpcException(new Status(StatusCode.InvalidArgument, "bad")))
: Task.FromResult(new WriteAck { Success = true }),
};
using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger<GatewayAlarmHistorianWriter>.Instance);
var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, 10).Select(Evt).ToList(), Ct);
outcomes[5].ShouldBe(HistorianWriteOutcome.PermanentFail);
outcomes.Where((_, i) => i != 5).ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
}
private static void InterlockedMax(ref int target, int value)
{
int current;
while ((current = Volatile.Read(ref target)) < value && Interlocked.CompareExchange(ref target, value, current) != current) { }
}
}
@@ -32,6 +32,26 @@ public sealed class AlarmWriterTests(QuestDbFixture fx)
return Convert.ToInt64(await cmd.ExecuteScalarAsync());
}
[Fact]
public async Task A_batch_is_one_insert_per_chunk_and_a_redelivery_converges()
{
fx.SkipIfUnavailable();
var (w, c) = Make();
var events = Enumerable.Range(0, 1200).Select(i => Ev(kind: $"k{i}", at: T.AddSeconds(i))).ToList();
events.Insert(3, new AlarmHistorianEvent("alarm-1", "Plant/Line/Eq", "HiTemp", "LimitAlarm", AlarmSeverity.High,
"old", "msg", "user", null, new DateTime(1969, 12, 31, 0, 0, 0, DateTimeKind.Utc))); // pre-1970: never sent
var r = await w.WriteBatchAsync(events, CancellationToken.None);
r[3].ShouldBe(HistorianWriteOutcome.PermanentFail);
r.Where((_, i) => i != 3).ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
w.Statements.ShouldBe(3, "1,200 events = 500 + 500 + 200, not 1,200 round trips");
(await CountAsync(c)).ShouldBe(1200);
(await w.WriteBatchAsync(events.Where((_, i) => i != 3).ToList(), CancellationToken.None))
.ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
(await CountAsync(c)).ShouldBe(1200, "DEDUP UPSERT KEYS(ts, event_id) converges a redelivered batch");
}
[Fact]
public async Task Event_round_trips_and_a_repeat_is_one_row_with_every_write_Ack()
{
@@ -0,0 +1,102 @@
using Shouldly;
using Xunit;
using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests;
/// <summary>
/// The shared batched alarm write: one statement per chunk instead of one round trip per event, with the per-event
/// outcomes the store-and-forward drain needs kept intact (a poison event is isolated, an outage retries everything,
/// a cancellation never dead-letters).
/// </summary>
public sealed class AlarmBatchWriteTests
{
private sealed class PoisonException : Exception;
private sealed class OutageException : Exception;
private static HistorianWriteOutcome Classify(Exception ex) =>
ex is PoisonException ? HistorianWriteOutcome.PermanentFail : HistorianWriteOutcome.RetryPlease;
private static HistorianWriteOutcome?[] Undecided(int n) => new HistorianWriteOutcome?[n];
[Fact]
public async Task A_clean_batch_is_one_statement_per_chunk_and_all_Ack()
{
var statements = new List<int[]>();
var r = await AlarmBatchWrite.RunAsync(Undecided(25), 10,
(ix, _) => { statements.Add(ix.ToArray()); return Task.CompletedTask; }, Classify, null, CancellationToken.None);
r.ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
statements.Select(s => s.Length).ShouldBe([10, 10, 5]);
statements.SelectMany(s => s).ShouldBe(Enumerable.Range(0, 25));
}
[Fact]
public async Task A_poison_event_is_isolated_and_its_siblings_still_Ack()
{
var statements = 0;
var r = await AlarmBatchWrite.RunAsync(Undecided(6), 10, (ix, _) =>
{
statements++;
if (ix.Contains(3)) throw new PoisonException();
return Task.CompletedTask;
}, Classify, null, CancellationToken.None);
r.ShouldBe([HistorianWriteOutcome.Ack, HistorianWriteOutcome.Ack, HistorianWriteOutcome.Ack,
HistorianWriteOutcome.PermanentFail, HistorianWriteOutcome.Ack, HistorianWriteOutcome.Ack]);
statements.ShouldBe(1 + 6, "the failed chunk, then one statement per event");
}
[Fact]
public async Task An_outage_retries_the_whole_chunk_without_re_sending_event_by_event()
{
var statements = 0;
var r = await AlarmBatchWrite.RunAsync(Undecided(6), 10,
(_, _) => { statements++; throw new OutageException(); }, Classify, null, CancellationToken.None);
r.ShouldAllBe(o => o == HistorianWriteOutcome.RetryPlease);
statements.ShouldBe(1);
}
[Fact]
public async Task Pre_decided_slots_are_kept_and_never_sent()
{
var sent = new List<int>();
var outcomes = Undecided(4);
outcomes[1] = HistorianWriteOutcome.PermanentFail; // e.g. a pre-1970 time on QuestDB
var r = await AlarmBatchWrite.RunAsync(outcomes, 10,
(ix, _) => { sent.AddRange(ix); return Task.CompletedTask; }, Classify, null, CancellationToken.None);
r.ShouldBe([HistorianWriteOutcome.Ack, HistorianWriteOutcome.PermanentFail, HistorianWriteOutcome.Ack, HistorianWriteOutcome.Ack]);
sent.ShouldBe([0, 2, 3]);
}
[Fact]
public async Task A_cancellation_mid_batch_retries_everything_undecided_and_dead_letters_nothing()
{
using var cts = new CancellationTokenSource();
var r = await AlarmBatchWrite.RunAsync(Undecided(5), 2, (ix, ct) =>
{
if (ix.Contains(2)) { cts.Cancel(); ct.ThrowIfCancellationRequested(); }
return Task.CompletedTask;
}, Classify, null, cts.Token);
r.ShouldBe([HistorianWriteOutcome.Ack, HistorianWriteOutcome.Ack,
HistorianWriteOutcome.RetryPlease, HistorianWriteOutcome.RetryPlease, HistorianWriteOutcome.RetryPlease]);
}
[Fact]
public async Task A_cancellation_during_poison_isolation_retries_the_rest()
{
using var cts = new CancellationTokenSource();
var r = await AlarmBatchWrite.RunAsync(Undecided(4), 10, (ix, ct) =>
{
if (ix.Count > 1) throw new PoisonException();
if (ix[0] == 1) { cts.Cancel(); ct.ThrowIfCancellationRequested(); }
return Task.CompletedTask;
}, Classify, null, cts.Token);
r.ShouldBe([HistorianWriteOutcome.Ack, HistorianWriteOutcome.RetryPlease, HistorianWriteOutcome.RetryPlease,
HistorianWriteOutcome.RetryPlease]);
}
}
@@ -63,6 +63,31 @@ public sealed class AlarmWriterTests(SqlHistorianFixture fx)
r.ShouldBe([HistorianWriteOutcome.Ack, HistorianWriteOutcome.PermanentFail, HistorianWriteOutcome.Ack]);
}
[Fact]
public async Task A_batch_is_one_statement_per_chunk_and_a_redelivery_converges()
{
fx.SkipIfUnavailable();
var (w, s) = Make();
var events = Enumerable.Range(0, 400)
.Select(i => new AlarmHistorianEvent("alarm-1", "Plant/Line/Eq", "HiTemp", "LimitAlarm", AlarmSeverity.High,
"Activated", $"m{i}", "user", null, T.AddSeconds(i)))
.ToList();
(await w.WriteBatchAsync(events, CancellationToken.None)).ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
w.Statements.ShouldBe(3, "400 events = 150 + 150 + 100, not 400 round trips");
// Redelivered (the at-least-once drain), with an in-batch duplicate: every event Acks, no row is added.
var again = events.Append(events[0]).ToList();
var r = await w.WriteBatchAsync(again, CancellationToken.None);
r.Count.ShouldBe(401);
r.ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
await using var c = new SqlConnection(fx.ConnectionString);
await c.OpenAsync();
await using var cmd = c.CreateCommand();
cmd.CommandText = $"SELECT COUNT(*) FROM [{s}].Event";
Convert.ToInt32(await cmd.ExecuteScalarAsync()).ShouldBe(400);
}
[Fact]
public async Task Concurrent_writers_of_the_same_event_all_Ack_with_one_row()
{