dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

perf(comms): alarms-only seed, capped buffers, at-least-once audit pull

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-14 21:10:19 -04:00
parent 1040dc0fcc
commit 2ce0ad7ed1
30 changed files with 1941 additions and 482 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Integration/OutageReconciliationTests.cs
+26 -23
View File
@@ -89,28 +89,25 @@ public class OutageReconciliationTests : TestKit, IClassFixture<MsSqlMigrationFi
{
CallCount++;
var rows = await _siteQueue
.ReadPendingSinceAsync(sinceUtc, batchSize, ct)
.ConfigureAwait(false);
// Commit immediately on the site side — once the actor has the
// batch in hand it will InsertIfNotExistsAsync centrally; if the
// central insert later throws on a specific row, idempotency
// guarantees the next pull cycle does NOT re-fetch the row (it's
// already Reconciled on the site) but also does not surface the
// failure here. The brief calls this "ack-after-persist" — the
// production gRPC server will flip to Reconciled inside its
// PullAuditEvents handler after the central side has acknowledged
// (per Bundle A's race-fix, central is idempotent on EventId).
//
// MoreAvailable is true iff the read filled the batch — the actor
// uses this to decide whether to follow up on the next tick.
if (rows.Count > 0)
// Mirrors SiteStreamGrpcServer.PullAuditEvents exactly (WP2.3): the
// INCOMING cursor is central's receipt, so everything at or before it
// is retired FIRST; the rows this call serves are NOT retired, because
// nothing yet proves central consumed them. A fault between here and
// central's commit therefore re-serves them on the next tick instead of
// losing them. The actor sends no after_id, so the cursor is a bare
// timestamp under the inclusive >= read contract and only rows strictly
// older than it are provably received.
if (sinceUtc > DateTime.MinValue)
{
var ids = rows.Select(e => e.EventId).ToList();
await _siteQueue.MarkReconciledAsync(ids, ct).ConfigureAwait(false);
await _siteQueue.MarkReconciledUpToAsync(sinceUtc, null, ct).ConfigureAwait(false);
}
var rows = await _siteQueue
.ReadPendingSinceAsync(sinceUtc, batchSize, afterId: null, ct)
.ConfigureAwait(false);
// MoreAvailable is true iff the read filled the batch — the actor
// uses this to decide whether to follow up on the next tick.
return new PullAuditEventsResponse(rows, MoreAvailable: rows.Count >= batchSize);
}
}
@@ -251,13 +248,19 @@ public class OutageReconciliationTests : TestKit, IClassFixture<MsSqlMigrationFi
duration: TimeSpan.FromSeconds(30),
interval: TimeSpan.FromMilliseconds(200));
// Step 4: assert site rows flipped to Reconciled.
// ReadPendingAsync only returns Pending rows; after a full drain
// it must be empty.
// Step 4: assert site rows flipped to Reconciled once central's cursor
// proved receipt. Exactly ONE row can legitimately remain Pending: the
// newest, which sits AT the cursor instant. Central sends only a
// timestamp cursor (no after_id yet — see the central-side follow-up),
// and under the inclusive >= read contract a bare timestamp cannot
// prove the rows AT that instant were consumed, so the site keeps
// serving that row until a newer one advances the cursor. Re-serving is
// harmless: central dedups on EventId (asserted in step 5).
await AwaitAssertAsync(async () =>
{
var stillPending = await sqliteWriter.ReadPendingAsync(totalEvents + 10);
Assert.Empty(stillPending);
Assert.True(stillPending.Count <= 1,
$"expected at most the boundary row to remain Pending, found {stillPending.Count}");
},
duration: TimeSpan.FromSeconds(10),
interval: TimeSpan.FromMilliseconds(100));
tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Site/SiteAuditRetentionServiceTests.cs
+4 -1
View File
@@ -160,10 +160,13 @@ public class SiteAuditRetentionServiceTests
=> throw new NotSupportedException();
public Task MarkForwardedAsync(IReadOnlyList<Guid> eventIds, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task<IReadOnlyList<AuditEvent>> ReadPendingSinceAsync(DateTime sinceUtc, int batchSize, CancellationToken ct = default)
public Task<IReadOnlyList<AuditEvent>> ReadPendingSinceAsync(
DateTime sinceUtc, int batchSize, string? afterId = null, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task MarkReconciledAsync(IReadOnlyList<Guid> eventIds, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task<int> MarkReconciledUpToAsync(DateTime sinceUtc, string? afterId, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task<SiteAuditBacklogSnapshot> GetBacklogStatsAsync(CancellationToken ct = default)
=> throw new NotSupportedException();
}
tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Site/SqliteAuditWriterWriteTests.cs
+117
View File
@@ -531,6 +531,123 @@ public class SqliteAuditWriterWriteTests
Assert.Equal(pending.EventId, rows[0].EventId);
}
// ----- WP2.3: composite keyset cursor + cursor-proved retirement ----- //
[Fact]
public async Task ReadPendingSinceAsync_WithAfterId_SkipsRowsAtOrBeforeTheCompositeCursor()
{
// A batch of rows sharing ONE exact instant used to pin the inclusive-timestamp
// cursor forever: every pull re-served the same page and the backlog never drained.
var (writer, _) = CreateWriter(nameof(ReadPendingSinceAsync_WithAfterId_SkipsRowsAtOrBeforeTheCompositeCursor));
await using var _w = writer;
var instant = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc);
var sameInstant = Enumerable.Range(0, 5)
.Select(_ => NewEvent(occurredAtUtc: instant))
.ToList();
foreach (var e in sameInstant) await writer.WriteAsync(e);
// First page under the composite order (OccurredAtUtc, EventId).
var page1 = await writer.ReadPendingSinceAsync(instant, batchSize: 2);
Assert.Equal(2, page1.Count);
// Second page continues strictly after the last row of the first — no repeats,
// no stall, even though every row shares the same instant.
var page2 = await writer.ReadPendingSinceAsync(
instant, batchSize: 2, afterId: page1[^1].EventId.ToString());
Assert.Equal(2, page2.Count);
Assert.Empty(page2.Select(r => r.EventId).Intersect(page1.Select(r => r.EventId)));
var page3 = await writer.ReadPendingSinceAsync(
instant, batchSize: 2, afterId: page2[^1].EventId.ToString());
Assert.Single(page3); // the 5th and last
var allIds = page1.Concat(page2).Concat(page3).Select(r => r.EventId).ToHashSet();
Assert.Equal(sameInstant.Select(e => e.EventId).ToHashSet(), allIds);
}
[Fact]
public async Task MarkReconciledUpToAsync_WithCursorId_RetiresOnlyRowsAtOrBeforeIt()
{
var (writer, _) = CreateWriter(nameof(MarkReconciledUpToAsync_WithCursorId_RetiresOnlyRowsAtOrBeforeIt));
await using var _w = writer;
var instant = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc);
var older = NewEvent(occurredAtUtc: instant.AddSeconds(-10));
var atInstant = Enumerable.Range(0, 4).Select(_ => NewEvent(occurredAtUtc: instant)).ToList();
var newer = NewEvent(occurredAtUtc: instant.AddSeconds(10));
await writer.WriteAsync(older);
foreach (var e in atInstant) await writer.WriteAsync(e);
await writer.WriteAsync(newer);
// Central consumed the older row plus the first two at the shared instant.
var consumed = await writer.ReadPendingSinceAsync(instant, batchSize: 2);
var cursorId = consumed[^1].EventId.ToString();
var flipped = await writer.MarkReconciledUpToAsync(instant, cursorId);
// older + the two consumed at the instant.
Assert.Equal(3, flipped);
var remaining = await writer.ReadPendingSinceAsync(DateTime.MinValue, batchSize: 100);
var remainingIds = remaining.Select(r => r.EventId).ToHashSet();
Assert.DoesNotContain(older.EventId, remainingIds);
foreach (var e in consumed) Assert.DoesNotContain(e.EventId, remainingIds);
Assert.Contains(newer.EventId, remainingIds);
Assert.Equal(3, remaining.Count); // the two un-consumed at the instant + newer
}
[Fact]
public async Task MarkReconciledUpToAsync_WithoutCursorId_LeavesTheBoundaryInstantServable()
{
// With a bare timestamp cursor under the inclusive >= read contract, the rows AT
// the cursor instant may be only half-consumed, so they must stay servable. Only
// strictly-older rows are provably received.
var (writer, _) = CreateWriter(nameof(MarkReconciledUpToAsync_WithoutCursorId_LeavesTheBoundaryInstantServable));
await using var _w = writer;
var instant = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc);
var older = NewEvent(occurredAtUtc: instant.AddSeconds(-5));
var boundary = NewEvent(occurredAtUtc: instant);
await writer.WriteAsync(older);
await writer.WriteAsync(boundary);
var flipped = await writer.MarkReconciledUpToAsync(instant, afterId: null);
Assert.Equal(1, flipped);
var remaining = await writer.ReadPendingSinceAsync(DateTime.MinValue, batchSize: 100);
var row = Assert.Single(remaining);
Assert.Equal(boundary.EventId, row.EventId);
}
[Fact]
public async Task ServingRows_WithoutAnAdvancingCursor_KeepsThemServable_AtLeastOnce()
{
// The at-least-once contract end to end at the storage layer: reading a batch
// changes no state, so a central that faults before committing gets the identical
// batch on its next pull. Only an advanced cursor retires rows.
var (writer, _) = CreateWriter(nameof(ServingRows_WithoutAnAdvancingCursor_KeepsThemServable_AtLeastOnce));
await using var _w = writer;
var since = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc);
var events = Enumerable.Range(1, 3)
.Select(i => NewEvent(occurredAtUtc: since.AddSeconds(i)))
.ToList();
foreach (var e in events) await writer.WriteAsync(e);
var first = await writer.ReadPendingSinceAsync(since, batchSize: 100);
// ... central faults here; its cursor never moves, so it replays the same call.
await writer.MarkReconciledUpToAsync(since, afterId: null);
var second = await writer.ReadPendingSinceAsync(since, batchSize: 100);
Assert.Equal(3, first.Count);
Assert.Equal(
first.Select(r => r.EventId).ToHashSet(),
second.Select(r => r.EventId).ToHashSet());
}
[Fact]
public async Task ReadPendingSinceAsync_InvalidBatchSize_Throws()
{