dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch 'fix/cached-telemetry-drain-hot-loop'

Browse Source
This commit is contained in:
Joseph Doherty
2026-07-27 15:50:42 -04:00
parent 63c16d6912 62cddcfa56
commit 663d03e89c
4 changed files with 328 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Site/Telemetry/SiteAuditTelemetryActorTests.cs
+137 -4
View File
@@ -362,11 +362,22 @@ public class SiteAuditTelemetryActorTests : TestKit
}
[Fact]
public async Task CachedDrain_OrphanRow_NoTrackingSnapshot_IsSkipped_DoesNotCrash()
public async Task CachedDrain_OrphanRow_PastGrace_IsAbandoned_AndTheValidRowStillFlows()
{
// RETARGETED, not rewritten from scratch: this test previously asserted
// the orphan "is skipped and stays Pending", which is precisely the
// defect fixed here (known-issues/2026-07-20-cached-telemetry-drain-
// hot-loop.md) — a Pending row that can never resolve is re-read every
// drain forever and starves the queue behind it. The half of the test
// that still holds (the valid row flows, the drain does not crash) is
// kept verbatim; only the orphan's fate is inverted.
//
// NewCachedEvent stamps occurredAtUtc in the past, well beyond the
// default 300 s grace, so this orphan is past the abandon threshold.
//
// Arrange — two cached audit rows: one with a tracking snapshot, one
// orphaned (the tracking store returns null). The orphaned row must be
// skipped without aborting the batch — the valid row still flows.
// abandoned without aborting the batch — the valid row still flows.
var orphan = NewCachedEvent(AuditKind.CachedSubmit);
var valid = NewCachedEvent(AuditKind.CachedResolve);
@@ -403,8 +414,8 @@ public class SiteAuditTelemetryActorTests : TestKit
// Act
CreateActorWithCachedDrain();
// Assert — exactly one push containing ONLY the valid row; the orphan
// is skipped and stays Pending (not in MarkForwardedAsync's id list).
// Assert — exactly one push containing ONLY the valid row: the orphan
// has no operational half to send.
await AwaitAssertAsync(async () =>
{
await _client.Received(1).IngestCachedTelemetryAsync(
@@ -415,9 +426,131 @@ public class SiteAuditTelemetryActorTests : TestKit
Assert.Single(capturedBatch!.Packets);
Assert.Equal(valid.EventId.ToString(), capturedBatch.Packets[0].AuditEvent.EventId);
// The valid row is marked Forwarded because central ack'd it...
await _queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 1 && g[0] == valid.EventId),
Arg.Any<CancellationToken>());
// ...and the orphan is marked Forwarded too, in its own call, so it
// LEAVES the cached queue. Its audit half is still delivered by the
// reconciliation pull, which covers Forwarded rows as well as Pending.
await _queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == 1 && g[0] == orphan.EventId),
Arg.Any<CancellationToken>());
}
[Fact]
public async Task CachedDrain_OrphanRow_InsideGrace_IsRetried_NotAbandoned()
{
// The other side of the grace window, and the reason abandonment is not
// immediate: a missing snapshot is normally a brief write race (the
// audit row lands microseconds before the tracking row). Abandoning on
// the first failed lookup would throw away the operational half of
// every cached call that happened to lose that race.
var fresh = ScadaBridgeAuditEventFactory.Create(
channel: AuditChannel.ApiOutbound,
kind: AuditKind.CachedSubmit,
status: AuditStatus.Submitted,
eventId: Guid.NewGuid(),
occurredAtUtc: DateTime.UtcNow, // inside the grace window
target: "ERP.GetOrder",
sourceSiteId: "site-1",
correlationId: Guid.NewGuid());
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
_queue.ReadPendingCachedTelemetryAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<AuditEvent>>(new[] { fresh }));
_trackingStore.GetStatusAsync(
new TrackedOperationId(fresh.CorrelationId!.Value), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<TrackingStatusSnapshot?>(null));
CreateActorWithCachedDrain();
// Give the drain several ticks to prove it keeps retrying rather than
// abandoning. A positive control guards against vacuity: the row must
// actually have been READ more than once, otherwise "never marked
// Forwarded" would also be true of a drain that never ran at all.
await AwaitAssertAsync(async () =>
{
await _queue.Received(Quantity.Within(2, int.MaxValue))
.ReadPendingCachedTelemetryAsync(Arg.Any<int>(), Arg.Any<CancellationToken>());
}, TimeSpan.FromSeconds(6));
await _queue.DidNotReceive().MarkForwardedAsync(
Arg.Any<IReadOnlyList<Guid>>(), Arg.Any<CancellationToken>());
}
[Fact]
public async Task CachedDrain_FullBatchOfUnresolvableRows_DoesNotStarveTheQueue()
{
// The actual regression. The queue is read oldest-first with a fixed
// BatchSize, so before the fix a batch's worth of permanently
// unresolvable rows at the head meant every subsequent drain re-read
// exactly those rows, failed identically, and never reached the newer
// rows behind them — a permanent stall of the cached-telemetry path,
// plus one log line per row per pass (~2,800/min measured on a rig).
var stuck = Enumerable.Range(0, 4).Select(_ => NewCachedEvent()).ToList();
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
_queue.ReadPendingCachedTelemetryAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<AuditEvent>>(stuck));
foreach (var row in stuck)
{
_trackingStore.GetStatusAsync(
new TrackedOperationId(row.CorrelationId!.Value), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<TrackingStatusSnapshot?>(null));
}
CreateActorWithCachedDrain();
// Every row is abandoned in ONE call, so the head of the queue clears
// and the next drain can see past it.
await AwaitAssertAsync(async () =>
{
await _queue.Received(1).MarkForwardedAsync(
Arg.Is<IReadOnlyList<Guid>>(g => g.Count == stuck.Count),
Arg.Any<CancellationToken>());
}, TimeSpan.FromSeconds(5));
// Nothing was pushed — there was no operational half to build for any
// of them — but the drain still made progress rather than spinning.
await _client.DidNotReceive().IngestCachedTelemetryAsync(
Arg.Any<CachedTelemetryBatch>(), Arg.Any<CancellationToken>());
}
[Fact]
public async Task CachedDrain_TrackingStoreThrow_DoesNotAbandonTheRow()
{
// A throw is a STORE fault (locked, corrupt, mid-restore), not a
// verdict about this row: the snapshot may well exist and be readable
// a second later. Abandoning here would discard the operational half
// of every in-flight cached call during a transient SQLite lock — so
// the throw path deliberately never abandons, however old the row is.
var row = NewCachedEvent(); // deliberately older than the grace window
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
_queue.ReadPendingCachedTelemetryAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
.Returns(Task.FromResult<IReadOnlyList<AuditEvent>>(new[] { row }));
_trackingStore.GetStatusAsync(
new TrackedOperationId(row.CorrelationId!.Value), Arg.Any<CancellationToken>())
.ThrowsAsync(new InvalidOperationException("tracking store unavailable"));
CreateActorWithCachedDrain();
// Positive control again: prove the drain actually ran repeatedly, so
// "never abandoned" is a real observation rather than an artefact of a
// drain that never executed.
await AwaitAssertAsync(async () =>
{
await _queue.Received(Quantity.Within(2, int.MaxValue))
.ReadPendingCachedTelemetryAsync(Arg.Any<int>(), Arg.Any<CancellationToken>());
}, TimeSpan.FromSeconds(6));
await _queue.DidNotReceive().MarkForwardedAsync(
Arg.Any<IReadOnlyList<Guid>>(), Arg.Any<CancellationToken>());
}
[Fact]