perf(historian): batch alarm-history writes; pipeline gateway SendEvent (lmxopcua#746)
All three alarm-history writers inserted one event per round trip, serially (SqlServer also opened a pooled connection per event), so a 10000-event alarm storm drained at 10000 x RTT while the backlog sat in LocalDb. - Sql.Common.AlarmBatchWrite: one statement per chunk; a chunk that fails PermanentFail-class is re-sent event by event so only the poison event dead-letters; a transient failure retries the chunk; a cancellation never dead-letters. - SqlServer: multi-row insert-if-absent over a VALUES set (150 events per statement, under the 2100-parameter limit), duplicate ids in a batch written once; a duplicate-key race isolates per event, where it is Ack. No new table type, so no schema migration (the review suggested a TVP; the VALUES form gets the same one-round-trip-per-chunk without one). - QuestDb: multi-row INSERTs of 500 events; pre-1970 events still dead-letter unsent. - Gateway: no batch RPC exists, so sends are pipelined, at most 8 in flight, outcomes kept in input order. Rebased from the unpushed remed/lane-H2 branch (originally authored 2026-10-22 in session_01Ub9hGmYLQvn9XETzw7CMsT) onto current master. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
+8
@@ -227,8 +227,16 @@ public sealed class FakeHistorianGatewayClient : IHistorianGatewayClient
|
||||
: Task.FromResult(WriteLiveValuesResult);
|
||||
}
|
||||
|
||||
/// <summary>When set, SendEvent is served by it (an asynchronous, observable gateway — concurrency tests).</summary>
|
||||
public Func<HistorianEvent, CancellationToken, Task<WriteAck>>? SendEventSource;
|
||||
|
||||
public Task<WriteAck> SendEventAsync(HistorianEvent evt, CancellationToken ct)
|
||||
{
|
||||
if (SendEventSource is { } source)
|
||||
{
|
||||
Interlocked.Increment(ref SendEventCallCount);
|
||||
return source(evt, ct);
|
||||
}
|
||||
LastSendEvent = evt;
|
||||
SendEventCallCount++;
|
||||
return SendEventThrows is not null
|
||||
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.HistorianGateway.Contracts.Grpc;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.Abstractions;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Gateway.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The gateway has no batch SendEvent RPC, so the alarm writer pipelines its per-event sends — at most
|
||||
/// <see cref="GatewayAlarmHistorianWriter.MaxSendsInFlight"/> at once — instead of one round trip after another. Every
|
||||
/// event still gets its own outcome, in input order.
|
||||
/// </summary>
|
||||
public sealed class GatewayAlarmWriterPipeliningTests
|
||||
{
|
||||
private static CancellationToken Ct => TestContext.Current.CancellationToken;
|
||||
|
||||
private static AlarmHistorianEvent Evt(int i) => new(
|
||||
$"alarm-{i}", "Area/Pump", "N", "LimitAlarm", AlarmSeverity.High, "Activated", $"m{i}", "u", null,
|
||||
new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddSeconds(i));
|
||||
|
||||
[Fact]
|
||||
public async Task Sends_overlap_up_to_the_bound_and_outcomes_keep_input_order()
|
||||
{
|
||||
var inFlight = 0;
|
||||
var maxInFlight = 0;
|
||||
var fake = new FakeHistorianGatewayClient
|
||||
{
|
||||
SendEventSource = async (e, ct) =>
|
||||
{
|
||||
var now = Interlocked.Increment(ref inFlight);
|
||||
InterlockedMax(ref maxInFlight, now);
|
||||
await Task.Delay(20, ct);
|
||||
Interlocked.Decrement(ref inFlight);
|
||||
// Odd messages are soft failures: their outcome must land on their own slot.
|
||||
return e.Properties.TryGetValue("Message", out var m) && int.Parse(m[1..]) % 2 == 1
|
||||
? new WriteAck { Success = false }
|
||||
: new WriteAck { Success = true };
|
||||
},
|
||||
};
|
||||
using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger<GatewayAlarmHistorianWriter>.Instance);
|
||||
|
||||
var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, 40).Select(Evt).ToList(), Ct);
|
||||
|
||||
outcomes.Count.ShouldBe(40);
|
||||
for (var i = 0; i < 40; i++)
|
||||
outcomes[i].ShouldBe(i % 2 == 1 ? HistorianWriteOutcome.RetryPlease : HistorianWriteOutcome.Ack, $"event {i}");
|
||||
maxInFlight.ShouldBeGreaterThan(1, "the sends must overlap — a sequential drain is 40 round trips");
|
||||
maxInFlight.ShouldBeLessThanOrEqualTo(GatewayAlarmHistorianWriter.MaxSendsInFlight);
|
||||
fake.SendEventCallCount.ShouldBe(40);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_cancellation_mid_batch_sends_nothing_more_and_dead_letters_nothing()
|
||||
{
|
||||
using var cts = new CancellationTokenSource();
|
||||
var sent = 0;
|
||||
var fake = new FakeHistorianGatewayClient
|
||||
{
|
||||
SendEventSource = async (_, ct) =>
|
||||
{
|
||||
if (Interlocked.Increment(ref sent) == 3) await cts.CancelAsync();
|
||||
await Task.Delay(10, ct);
|
||||
return new WriteAck { Success = true };
|
||||
},
|
||||
};
|
||||
using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger<GatewayAlarmHistorianWriter>.Instance);
|
||||
|
||||
var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, 40).Select(Evt).ToList(), cts.Token);
|
||||
|
||||
outcomes.Count.ShouldBe(40);
|
||||
outcomes.ShouldNotContain(HistorianWriteOutcome.PermanentFail);
|
||||
outcomes.Count(o => o == HistorianWriteOutcome.RetryPlease).ShouldBeGreaterThan(0);
|
||||
sent.ShouldBeLessThanOrEqualTo(GatewayAlarmHistorianWriter.MaxSendsInFlight + 2, "no new send starts after the cancel");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_poison_event_dead_letters_alone()
|
||||
{
|
||||
var fake = new FakeHistorianGatewayClient
|
||||
{
|
||||
SendEventSource = (e, _) => e.Properties.TryGetValue("Message", out var m) && m == "m5"
|
||||
? Task.FromException<WriteAck>(new RpcException(new Status(StatusCode.InvalidArgument, "bad")))
|
||||
: Task.FromResult(new WriteAck { Success = true }),
|
||||
};
|
||||
using var writer = new GatewayAlarmHistorianWriter(fake, NullLogger<GatewayAlarmHistorianWriter>.Instance);
|
||||
|
||||
var outcomes = await writer.WriteBatchAsync(Enumerable.Range(0, 10).Select(Evt).ToList(), Ct);
|
||||
|
||||
outcomes[5].ShouldBe(HistorianWriteOutcome.PermanentFail);
|
||||
outcomes.Where((_, i) => i != 5).ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
|
||||
}
|
||||
|
||||
private static void InterlockedMax(ref int target, int value)
|
||||
{
|
||||
int current;
|
||||
while ((current = Volatile.Read(ref target)) < value && Interlocked.CompareExchange(ref target, value, current) != current) { }
|
||||
}
|
||||
}
|
||||
+20
@@ -32,6 +32,26 @@ public sealed class AlarmWriterTests(QuestDbFixture fx)
|
||||
return Convert.ToInt64(await cmd.ExecuteScalarAsync());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_batch_is_one_insert_per_chunk_and_a_redelivery_converges()
|
||||
{
|
||||
fx.SkipIfUnavailable();
|
||||
var (w, c) = Make();
|
||||
var events = Enumerable.Range(0, 1200).Select(i => Ev(kind: $"k{i}", at: T.AddSeconds(i))).ToList();
|
||||
events.Insert(3, new AlarmHistorianEvent("alarm-1", "Plant/Line/Eq", "HiTemp", "LimitAlarm", AlarmSeverity.High,
|
||||
"old", "msg", "user", null, new DateTime(1969, 12, 31, 0, 0, 0, DateTimeKind.Utc))); // pre-1970: never sent
|
||||
|
||||
var r = await w.WriteBatchAsync(events, CancellationToken.None);
|
||||
r[3].ShouldBe(HistorianWriteOutcome.PermanentFail);
|
||||
r.Where((_, i) => i != 3).ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
|
||||
w.Statements.ShouldBe(3, "1,200 events = 500 + 500 + 200, not 1,200 round trips");
|
||||
(await CountAsync(c)).ShouldBe(1200);
|
||||
|
||||
(await w.WriteBatchAsync(events.Where((_, i) => i != 3).ToList(), CancellationToken.None))
|
||||
.ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
|
||||
(await CountAsync(c)).ShouldBe(1200, "DEDUP UPSERT KEYS(ts, event_id) converges a redelivered batch");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Event_round_trips_and_a_repeat_is_one_row_with_every_write_Ack()
|
||||
{
|
||||
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
using Shouldly;
|
||||
using Xunit;
|
||||
using ZB.MOM.WW.OtOpcUa.Core.AlarmHistorian;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.Driver.Historian.Sql.Common.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The shared batched alarm write: one statement per chunk instead of one round trip per event, with the per-event
|
||||
/// outcomes the store-and-forward drain needs kept intact (a poison event is isolated, an outage retries everything,
|
||||
/// a cancellation never dead-letters).
|
||||
/// </summary>
|
||||
public sealed class AlarmBatchWriteTests
|
||||
{
|
||||
private sealed class PoisonException : Exception;
|
||||
private sealed class OutageException : Exception;
|
||||
|
||||
private static HistorianWriteOutcome Classify(Exception ex) =>
|
||||
ex is PoisonException ? HistorianWriteOutcome.PermanentFail : HistorianWriteOutcome.RetryPlease;
|
||||
|
||||
private static HistorianWriteOutcome?[] Undecided(int n) => new HistorianWriteOutcome?[n];
|
||||
|
||||
[Fact]
|
||||
public async Task A_clean_batch_is_one_statement_per_chunk_and_all_Ack()
|
||||
{
|
||||
var statements = new List<int[]>();
|
||||
var r = await AlarmBatchWrite.RunAsync(Undecided(25), 10,
|
||||
(ix, _) => { statements.Add(ix.ToArray()); return Task.CompletedTask; }, Classify, null, CancellationToken.None);
|
||||
|
||||
r.ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
|
||||
statements.Select(s => s.Length).ShouldBe([10, 10, 5]);
|
||||
statements.SelectMany(s => s).ShouldBe(Enumerable.Range(0, 25));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_poison_event_is_isolated_and_its_siblings_still_Ack()
|
||||
{
|
||||
var statements = 0;
|
||||
var r = await AlarmBatchWrite.RunAsync(Undecided(6), 10, (ix, _) =>
|
||||
{
|
||||
statements++;
|
||||
if (ix.Contains(3)) throw new PoisonException();
|
||||
return Task.CompletedTask;
|
||||
}, Classify, null, CancellationToken.None);
|
||||
|
||||
r.ShouldBe([HistorianWriteOutcome.Ack, HistorianWriteOutcome.Ack, HistorianWriteOutcome.Ack,
|
||||
HistorianWriteOutcome.PermanentFail, HistorianWriteOutcome.Ack, HistorianWriteOutcome.Ack]);
|
||||
statements.ShouldBe(1 + 6, "the failed chunk, then one statement per event");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task An_outage_retries_the_whole_chunk_without_re_sending_event_by_event()
|
||||
{
|
||||
var statements = 0;
|
||||
var r = await AlarmBatchWrite.RunAsync(Undecided(6), 10,
|
||||
(_, _) => { statements++; throw new OutageException(); }, Classify, null, CancellationToken.None);
|
||||
|
||||
r.ShouldAllBe(o => o == HistorianWriteOutcome.RetryPlease);
|
||||
statements.ShouldBe(1);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Pre_decided_slots_are_kept_and_never_sent()
|
||||
{
|
||||
var sent = new List<int>();
|
||||
var outcomes = Undecided(4);
|
||||
outcomes[1] = HistorianWriteOutcome.PermanentFail; // e.g. a pre-1970 time on QuestDB
|
||||
var r = await AlarmBatchWrite.RunAsync(outcomes, 10,
|
||||
(ix, _) => { sent.AddRange(ix); return Task.CompletedTask; }, Classify, null, CancellationToken.None);
|
||||
|
||||
r.ShouldBe([HistorianWriteOutcome.Ack, HistorianWriteOutcome.PermanentFail, HistorianWriteOutcome.Ack, HistorianWriteOutcome.Ack]);
|
||||
sent.ShouldBe([0, 2, 3]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_cancellation_mid_batch_retries_everything_undecided_and_dead_letters_nothing()
|
||||
{
|
||||
using var cts = new CancellationTokenSource();
|
||||
var r = await AlarmBatchWrite.RunAsync(Undecided(5), 2, (ix, ct) =>
|
||||
{
|
||||
if (ix.Contains(2)) { cts.Cancel(); ct.ThrowIfCancellationRequested(); }
|
||||
return Task.CompletedTask;
|
||||
}, Classify, null, cts.Token);
|
||||
|
||||
r.ShouldBe([HistorianWriteOutcome.Ack, HistorianWriteOutcome.Ack,
|
||||
HistorianWriteOutcome.RetryPlease, HistorianWriteOutcome.RetryPlease, HistorianWriteOutcome.RetryPlease]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_cancellation_during_poison_isolation_retries_the_rest()
|
||||
{
|
||||
using var cts = new CancellationTokenSource();
|
||||
var r = await AlarmBatchWrite.RunAsync(Undecided(4), 10, (ix, ct) =>
|
||||
{
|
||||
if (ix.Count > 1) throw new PoisonException();
|
||||
if (ix[0] == 1) { cts.Cancel(); ct.ThrowIfCancellationRequested(); }
|
||||
return Task.CompletedTask;
|
||||
}, Classify, null, cts.Token);
|
||||
|
||||
r.ShouldBe([HistorianWriteOutcome.Ack, HistorianWriteOutcome.RetryPlease, HistorianWriteOutcome.RetryPlease,
|
||||
HistorianWriteOutcome.RetryPlease]);
|
||||
}
|
||||
}
|
||||
+25
@@ -63,6 +63,31 @@ public sealed class AlarmWriterTests(SqlHistorianFixture fx)
|
||||
r.ShouldBe([HistorianWriteOutcome.Ack, HistorianWriteOutcome.PermanentFail, HistorianWriteOutcome.Ack]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task A_batch_is_one_statement_per_chunk_and_a_redelivery_converges()
|
||||
{
|
||||
fx.SkipIfUnavailable();
|
||||
var (w, s) = Make();
|
||||
var events = Enumerable.Range(0, 400)
|
||||
.Select(i => new AlarmHistorianEvent("alarm-1", "Plant/Line/Eq", "HiTemp", "LimitAlarm", AlarmSeverity.High,
|
||||
"Activated", $"m{i}", "user", null, T.AddSeconds(i)))
|
||||
.ToList();
|
||||
|
||||
(await w.WriteBatchAsync(events, CancellationToken.None)).ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
|
||||
w.Statements.ShouldBe(3, "400 events = 150 + 150 + 100, not 400 round trips");
|
||||
|
||||
// Redelivered (the at-least-once drain), with an in-batch duplicate: every event Acks, no row is added.
|
||||
var again = events.Append(events[0]).ToList();
|
||||
var r = await w.WriteBatchAsync(again, CancellationToken.None);
|
||||
r.Count.ShouldBe(401);
|
||||
r.ShouldAllBe(o => o == HistorianWriteOutcome.Ack);
|
||||
await using var c = new SqlConnection(fx.ConnectionString);
|
||||
await c.OpenAsync();
|
||||
await using var cmd = c.CreateCommand();
|
||||
cmd.CommandText = $"SELECT COUNT(*) FROM [{s}].Event";
|
||||
Convert.ToInt32(await cmd.ExecuteScalarAsync()).ShouldBe(400);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Concurrent_writers_of_the_same_event_all_Ack_with_one_row()
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user