perf(comms): alarms-only seed, capped buffers, at-least-once audit pull
This commit is contained in:
Joseph Doherty
+112
-23
@@ -13,9 +13,10 @@ using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Bundle A A2 tests for <see cref="SiteStreamGrpcServer.PullAuditEvents"/>.
|
||||
/// Verifies the request → ISiteAuditQueue.ReadPendingSinceAsync → response →
|
||||
/// MarkReconciledAsync round-trip through the gRPC handler. The queue is an
|
||||
/// Tests for <see cref="SiteStreamGrpcServer.PullAuditEvents"/>: the request →
|
||||
/// <c>ISiteAuditQueue.ReadPendingSinceAsync</c> → response round-trip, plus the WP2.3
|
||||
/// at-least-once contract — rows are retired by the NEXT pull's cursor
|
||||
/// (<c>MarkReconciledUpToAsync</c>), never by the act of serving them. The queue is an
|
||||
/// NSubstitute stub so the tests never touch SQLite.
|
||||
/// </summary>
|
||||
public class SiteStreamPullAuditEventsTests : TestKit
|
||||
@@ -63,11 +64,12 @@ public class SiteStreamPullAuditEventsTests : TestKit
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PullAuditEvents_With5PendingRows_ReturnsAllFiveDtos_AndFlipsToReconciled()
|
||||
public async Task PullAuditEvents_With5PendingRows_ReturnsAllFiveDtos_AndDoesNotFlipThem()
|
||||
{
|
||||
var queue = Substitute.For<ISiteAuditQueue>();
|
||||
var events = Enumerable.Range(0, 5).Select(_ => NewEvent()).ToList();
|
||||
queue.ReadPendingSinceAsync(Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
|
||||
queue.ReadPendingSinceAsync(
|
||||
Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<string?>(), Arg.Any<CancellationToken>())
|
||||
.Returns((IReadOnlyList<AuditEvent>)events);
|
||||
|
||||
var server = CreateServer();
|
||||
@@ -86,11 +88,100 @@ public class SiteStreamPullAuditEventsTests : TestKit
|
||||
var expectedIds = events.Select(e => e.EventId.ToString()).ToHashSet();
|
||||
Assert.True(expectedIds.SetEquals(response.Events.Select(d => d.EventId).ToHashSet()));
|
||||
|
||||
// Verify MarkReconciledAsync received the same 5 ids (best-effort flip).
|
||||
await queue.Received(1).MarkReconciledAsync(
|
||||
Arg.Is<IReadOnlyList<Guid>>(ids => ids.Count == 5 &&
|
||||
ids.ToHashSet().SetEquals(events.Select(e => e.EventId))),
|
||||
Arg.Any<CancellationToken>());
|
||||
// AT-LEAST-ONCE: serving rows is NOT proof of receipt. The per-id flip is gone
|
||||
// entirely; only a later cursor retires rows.
|
||||
await queue.DidNotReceive().MarkReconciledAsync(
|
||||
Arg.Any<IReadOnlyList<Guid>>(), Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PullAuditEvents_FaultBetweenResponseAndNextPull_ReservesTheSameRows()
|
||||
{
|
||||
// The failure this closes: central receives the batch, then dies before committing
|
||||
// it, so its cursor never advances. Pre-fix the site had already flipped the rows to
|
||||
// Reconciled while serving them, and ReadPendingSinceAsync would never return them
|
||||
// again — the rows were silently lost. Now the unchanged cursor means no flip, and
|
||||
// the identical batch is served again.
|
||||
var queue = Substitute.For<ISiteAuditQueue>();
|
||||
var since = DateTime.SpecifyKind(new DateTime(2026, 5, 20, 9, 30, 0), DateTimeKind.Utc);
|
||||
var events = Enumerable.Range(0, 3).Select(_ => NewEvent()).ToList();
|
||||
queue.ReadPendingSinceAsync(
|
||||
Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<string?>(), Arg.Any<CancellationToken>())
|
||||
.Returns((IReadOnlyList<AuditEvent>)events);
|
||||
|
||||
var server = CreateServer();
|
||||
server.SetSiteAuditQueue(queue);
|
||||
|
||||
var request = new PullAuditEventsRequest
|
||||
{
|
||||
SinceUtc = Timestamp.FromDateTime(since),
|
||||
BatchSize = 100,
|
||||
};
|
||||
|
||||
var first = await server.PullAuditEvents(request, NewContext());
|
||||
// …central faults here; it never commits, so it re-pulls with the SAME cursor.
|
||||
var second = await server.PullAuditEvents(request, NewContext());
|
||||
|
||||
Assert.Equal(3, first.Events.Count);
|
||||
Assert.Equal(
|
||||
first.Events.Select(e => e.EventId).ToHashSet(),
|
||||
second.Events.Select(e => e.EventId).ToHashSet());
|
||||
|
||||
// Neither pull retired anything past the (unchanged) cursor: the flip is bounded by
|
||||
// the cursor value, so replaying the same cursor can never retire the served rows.
|
||||
await queue.Received(2).MarkReconciledUpToAsync(
|
||||
since, null, Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PullAuditEvents_AdvancedCursor_RetiresEverythingUpToIt_BeforeReading()
|
||||
{
|
||||
// The cursor central sends back IS the receipt: everything at or before it has been
|
||||
// ingested, so those rows are flipped — and flipped BEFORE the read, so they do not
|
||||
// consume this batch's budget.
|
||||
var queue = Substitute.For<ISiteAuditQueue>();
|
||||
queue.ReadPendingSinceAsync(
|
||||
Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<string?>(), Arg.Any<CancellationToken>())
|
||||
.Returns((IReadOnlyList<AuditEvent>)Array.Empty<AuditEvent>());
|
||||
|
||||
var server = CreateServer();
|
||||
server.SetSiteAuditQueue(queue);
|
||||
|
||||
var cursorTime = DateTime.SpecifyKind(new DateTime(2026, 5, 20, 10, 0, 0), DateTimeKind.Utc);
|
||||
var cursorId = Guid.NewGuid().ToString();
|
||||
var request = new PullAuditEventsRequest
|
||||
{
|
||||
SinceUtc = Timestamp.FromDateTime(cursorTime),
|
||||
BatchSize = 100,
|
||||
AfterId = cursorId,
|
||||
};
|
||||
|
||||
await server.PullAuditEvents(request, NewContext());
|
||||
|
||||
await queue.Received(1).MarkReconciledUpToAsync(
|
||||
cursorTime, cursorId, Arg.Any<CancellationToken>());
|
||||
// The keyset cursor is passed straight through to the read as well.
|
||||
await queue.Received(1).ReadPendingSinceAsync(
|
||||
cursorTime, 100, cursorId, Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PullAuditEvents_FirstEverPull_DoesNotFlipAnything()
|
||||
{
|
||||
// since == MinValue means "from the beginning of recorded history" — central has
|
||||
// consumed nothing yet, so there is nothing to retire.
|
||||
var queue = Substitute.For<ISiteAuditQueue>();
|
||||
queue.ReadPendingSinceAsync(
|
||||
Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<string?>(), Arg.Any<CancellationToken>())
|
||||
.Returns((IReadOnlyList<AuditEvent>)Array.Empty<AuditEvent>());
|
||||
|
||||
var server = CreateServer();
|
||||
server.SetSiteAuditQueue(queue);
|
||||
|
||||
await server.PullAuditEvents(new PullAuditEventsRequest { BatchSize = 10 }, NewContext());
|
||||
|
||||
await queue.DidNotReceive().MarkReconciledUpToAsync(
|
||||
Arg.Any<DateTime>(), Arg.Any<string?>(), Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -102,7 +193,8 @@ public class SiteStreamPullAuditEventsTests : TestKit
|
||||
// yields an empty gRPC response.
|
||||
var queue = Substitute.For<ISiteAuditQueue>();
|
||||
var capturedSince = DateTime.MinValue;
|
||||
queue.ReadPendingSinceAsync(Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
|
||||
queue.ReadPendingSinceAsync(
|
||||
Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<string?>(), Arg.Any<CancellationToken>())
|
||||
.Returns(call =>
|
||||
{
|
||||
capturedSince = call.ArgAt<DateTime>(0);
|
||||
@@ -124,9 +216,6 @@ public class SiteStreamPullAuditEventsTests : TestKit
|
||||
Assert.Empty(response.Events);
|
||||
Assert.False(response.MoreAvailable);
|
||||
Assert.Equal(since, capturedSince);
|
||||
// Empty result → no MarkReconciledAsync call (no rows to flip).
|
||||
await queue.DidNotReceive().MarkReconciledAsync(
|
||||
Arg.Any<IReadOnlyList<Guid>>(), Arg.Any<CancellationToken>());
|
||||
}
|
||||
|
||||
[Fact]
|
||||
@@ -134,7 +223,8 @@ public class SiteStreamPullAuditEventsTests : TestKit
|
||||
{
|
||||
var queue = Substitute.For<ISiteAuditQueue>();
|
||||
var events = Enumerable.Range(0, 3).Select(_ => NewEvent()).ToList();
|
||||
queue.ReadPendingSinceAsync(Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
|
||||
queue.ReadPendingSinceAsync(
|
||||
Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<string?>(), Arg.Any<CancellationToken>())
|
||||
.Returns((IReadOnlyList<AuditEvent>)events);
|
||||
|
||||
var server = CreateServer();
|
||||
@@ -154,16 +244,17 @@ public class SiteStreamPullAuditEventsTests : TestKit
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PullAuditEvents_MarkReconciledThrows_ResponseStillReturned()
|
||||
public async Task PullAuditEvents_MarkReconciledUpToThrows_ResponseStillReturned()
|
||||
{
|
||||
// The Reconciled flip is best-effort — if it fails, the response must
|
||||
// still surface so central can ingest the rows (and dedup on EventId
|
||||
// when it pulls them again).
|
||||
// The retire step is best-effort — if it fails, the pull must still serve rows.
|
||||
// Worst case the same rows are shipped again and central dedups on EventId.
|
||||
var queue = Substitute.For<ISiteAuditQueue>();
|
||||
var events = Enumerable.Range(0, 2).Select(_ => NewEvent()).ToList();
|
||||
queue.ReadPendingSinceAsync(Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<CancellationToken>())
|
||||
queue.ReadPendingSinceAsync(
|
||||
Arg.Any<DateTime>(), Arg.Any<int>(), Arg.Any<string?>(), Arg.Any<CancellationToken>())
|
||||
.Returns((IReadOnlyList<AuditEvent>)events);
|
||||
queue.MarkReconciledAsync(Arg.Any<IReadOnlyList<Guid>>(), Arg.Any<CancellationToken>())
|
||||
queue.MarkReconciledUpToAsync(
|
||||
Arg.Any<DateTime>(), Arg.Any<string?>(), Arg.Any<CancellationToken>())
|
||||
.ThrowsAsync(new InvalidOperationException("SQLite disposed mid-call"));
|
||||
|
||||
var server = CreateServer();
|
||||
@@ -175,8 +266,6 @@ public class SiteStreamPullAuditEventsTests : TestKit
|
||||
BatchSize = 100,
|
||||
};
|
||||
|
||||
// Must NOT throw — the response is built before the flip and returned
|
||||
// regardless of the flip outcome.
|
||||
var response = await server.PullAuditEvents(request, NewContext());
|
||||
|
||||
Assert.Equal(2, response.Events.Count);
|
||||
|
||||
Reference in New Issue
Block a user