dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

perf(comms+audit): close phase-2 residuals — direct ingest path, monotonic timeouts, synthetic probe, not-reporting set, cursor-exact audit pull

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-14 21:38:23 -04:00
parent 4cd1441984
commit a5882753dd
38 changed files with 1254 additions and 443 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/GrpcPullAuditEventsClientTests.cs
+8 -8
View File
@@ -117,7 +117,7 @@ public class GrpcPullAuditEventsClientTests
invoker,
NullLogger<GrpcPullAuditEventsClient>.Instance);
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, ct: CancellationToken.None);
// Endpoint resolution + request shaping.
Assert.Equal("http://site-a:8083", invoker.Endpoint);
@@ -141,7 +141,7 @@ public class GrpcPullAuditEventsClientTests
invoker,
NullLogger<GrpcPullAuditEventsClient>.Instance);
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, ct: CancellationToken.None);
Assert.Empty(result.Events);
Assert.False(result.MoreAvailable);
@@ -161,7 +161,7 @@ public class GrpcPullAuditEventsClientTests
NullLogger<GrpcPullAuditEventsClient>.Instance);
// MUST NOT throw — per the IPullAuditEventsClient contract.
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, ct: CancellationToken.None);
Assert.Empty(result.Events);
Assert.False(result.MoreAvailable);
@@ -178,7 +178,7 @@ public class GrpcPullAuditEventsClientTests
invoker,
NullLogger<GrpcPullAuditEventsClient>.Instance);
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, ct: CancellationToken.None);
Assert.Empty(result.Events);
Assert.False(result.MoreAvailable);
@@ -197,7 +197,7 @@ public class GrpcPullAuditEventsClientTests
invoker,
NullLogger<GrpcPullAuditEventsClient>.Instance);
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, ct: CancellationToken.None);
Assert.Empty(result.Events);
Assert.False(result.MoreAvailable);
@@ -221,7 +221,7 @@ public class GrpcPullAuditEventsClientTests
NullLogger<GrpcPullAuditEventsClient>.Instance);
// MUST NOT throw — must dial successfully.
var result = await sut.PullAsync("site-a", minUnspecified, batchSize: 256, CancellationToken.None);
var result = await sut.PullAsync("site-a", minUnspecified, afterId: null, batchSize: 256, ct: CancellationToken.None);
Assert.Equal(1, invoker.CallCount);
Assert.Equal("http://site-a:8083", invoker.Endpoint);
@@ -252,7 +252,7 @@ public class GrpcPullAuditEventsClientTests
invoker,
NullLogger<GrpcPullAuditEventsClient>.Instance);
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, ct: CancellationToken.None);
Assert.Equal(new[] { "http://node-a:8083", "http://node-b:8083" }, invoker.Dialed);
var evt = Assert.Single(result.Events);
@@ -271,7 +271,7 @@ public class GrpcPullAuditEventsClientTests
invoker,
NullLogger<GrpcPullAuditEventsClient>.Instance);
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, ct: CancellationToken.None);
Assert.Empty(result.Events);
Assert.Equal(new[] { "http://node-a:8083" }, invoker.Dialed);
tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/SiteAuditReconciliationActorTests.cs
+101 -3
View File
@@ -194,7 +194,7 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
/// </summary>
private sealed class ScriptedPullClient : IPullAuditEventsClient
{
public List<(string SiteId, DateTime SinceUtc, int BatchSize)> Calls { get; } = new();
public List<(string SiteId, DateTime SinceUtc, string? AfterId, int BatchSize)> Calls { get; } = new();
private readonly Dictionary<string, Queue<PullAuditEventsResponse>> _scripted = new();
private readonly Dictionary<string, Exception> _throwOnSite = new();
@@ -211,9 +211,9 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
}
public Task<PullAuditEventsResponse> PullAsync(
string siteId, DateTime sinceUtc, int batchSize, CancellationToken ct)
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
{
Calls.Add((siteId, sinceUtc, batchSize));
Calls.Add((siteId, sinceUtc, afterId, batchSize));
if (_throwOnSite.TryGetValue(siteId, out var ex))
{
throw ex;
@@ -425,6 +425,104 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
Assert.Equal(DateTime.MinValue, client.Calls[0].SinceUtc);
Assert.Equal(t3, client.Calls[1].SinceUtc);
// The composite half travels too (arch-review phase-2 residual #5): the first pull has
// no cursor at all, the second carries the id of the row at t3 so the site can retire
// it exactly instead of leaving the boundary instant permanently servable.
Assert.Null(client.Calls[0].AfterId);
Assert.Equal(e3.EventId.ToString(), client.Calls[1].AfterId);
}
[Fact]
public void Cursor_AdvancesOnTheIdTiebreak_WhenEveryRowSharesOneInstant()
{
// The case a bare timestamp cursor could never drain: a burst all stamped at the same
// instant. The timestamp cannot move, so only the id half can — and it must, or the
// next pull re-serves the identical window forever.
var sites = new StaticEnumerator(new SiteEntry("siteA", "http://siteA:8083"));
var t = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);
// Deterministic ids so "the greatest ordinal" is a fact, not a coin flip.
var low = NewEvent("siteA", t, Guid.Parse("11111111-1111-1111-1111-111111111111"));
var high = NewEvent("siteA", t, Guid.Parse("99999999-9999-9999-9999-999999999999"));
var mid = NewEvent("siteA", t, Guid.Parse("55555555-5555-5555-5555-555555555555"));
var client = new ScriptedPullClient().Script("siteA",
new PullAuditEventsResponse(new[] { low, high, mid }, MoreAvailable: true));
var repo = new RecordingRepo();
CreateActor(sites, client, repo, FastTickOptions());
AwaitAssert(() => Assert.True(client.Calls.Count >= 2,
$"need at least 2 pulls, got {client.Calls.Count}"),
duration: TimeSpan.FromSeconds(5),
interval: TimeSpan.FromMilliseconds(50));
Assert.Equal(t, client.Calls[1].SinceUtc);
Assert.Equal(high.EventId.ToString(), client.Calls[1].AfterId);
}
[Fact]
public void BothCursorHalves_AreHeldBack_WhileARowIsStillBeingRetried()
{
// A held-back cursor must hold BOTH halves: advancing the id while pinning the
// timestamp would skip past the very rows being retried.
var sites = new StaticEnumerator(new SiteEntry("siteA", "http://siteA:8083"));
var t = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);
var evt = NewEvent("siteA", t);
var client = new ScriptedPullClient().Script("siteA",
new PullAuditEventsResponse(new[] { evt }, MoreAvailable: false));
var repo = new AlwaysThrowingRepo();
CreateActor(sites, client, repo, FastTickOptions());
AwaitAssert(() => Assert.True(client.Calls.Count >= 2,
$"need at least 2 pulls, got {client.Calls.Count}"),
duration: TimeSpan.FromSeconds(5),
interval: TimeSpan.FromMilliseconds(50));
Assert.Equal(DateTime.MinValue, client.Calls[1].SinceUtc);
Assert.Null(client.Calls[1].AfterId);
}
/// <summary>Repository whose every insert throws, so the retry hold-back path is taken.</summary>
private sealed class AlwaysThrowingRepo : IAuditLogRepository
{
public Task InsertIfNotExistsAsync(AuditEvent evt, CancellationToken ct = default) =>
throw new InvalidOperationException("central insert failed");
public Task<IReadOnlyList<AuditEvent>> QueryAsync(
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task<long> SwitchOutPartitionAsync(
DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task<long> PurgeChannelOlderThanAsync(
string channel, DateTime threshold, int batchSize, TimeSpan? commandTimeout = null,
CancellationToken ct = default)
=> throw new NotSupportedException();
public Task<long> BackfillSourceNodeAsync(
string sentinel, DateTime before, int batchSize, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task<IReadOnlyList<DateTime>> GetPartitionBoundariesOlderThanAsync(
DateTime threshold, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task<ZB.MOM.WW.ScadaBridge.Commons.Types.AuditLogKpiSnapshot> GetKpiSnapshotAsync(
TimeSpan window, DateTime? nowUtc = null, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task<IReadOnlyList<ExecutionTreeNode>> GetExecutionTreeAsync(
Guid executionId, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task<IReadOnlyList<string>> GetDistinctSourceNodesAsync(CancellationToken ct = default)
=> throw new NotSupportedException();
}
// ---------------------------------------------------------------------
tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Integration/OutageReconciliationTests.cs
+6 -6
View File
@@ -85,7 +85,7 @@ public class OutageReconciliationTests : TestKit, IClassFixture<MsSqlMigrationFi
}
public async Task<PullAuditEventsResponse> PullAsync(
string siteId, DateTime sinceUtc, int batchSize, CancellationToken ct)
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
{
CallCount++;
@@ -94,16 +94,16 @@ public class OutageReconciliationTests : TestKit, IClassFixture<MsSqlMigrationFi
// is retired FIRST; the rows this call serves are NOT retired, because
// nothing yet proves central consumed them. A fault between here and
// central's commit therefore re-serves them on the next tick instead of
// losing them. The actor sends no after_id, so the cursor is a bare
// timestamp under the inclusive >= read contract and only rows strictly
// older than it are provably received.
// losing them. The actor now sends the composite (timestamp, id) cursor,
// so retirement is exact: the rows AT the cursor instant are proven
// received too, where a bare timestamp could only prove strictly-older ones.
if (sinceUtc > DateTime.MinValue)
{
await _siteQueue.MarkReconciledUpToAsync(sinceUtc, null, ct).ConfigureAwait(false);
await _siteQueue.MarkReconciledUpToAsync(sinceUtc, afterId, ct).ConfigureAwait(false);
}
var rows = await _siteQueue
.ReadPendingSinceAsync(sinceUtc, batchSize, afterId: null, ct)
.ReadPendingSinceAsync(sinceUtc, batchSize, afterId, ct)
.ConfigureAwait(false);
// MoreAvailable is true iff the read filled the batch — the actor