fix(site-call-audit): composite keyset cursor eliminates the single-timestamp reconciliation pin; pinned state now a published event
Reconciliation cursor becomes composite (UpdatedAtUtc, TrackedOperationId); IPullSiteCallsClient/GrpcPullSiteCallsClient forward the Task-15 after_id keyset (additive param, null preserves the legacy inclusive >= contract). A burst sharing one exact UpdatedAtUtc now drains via the id tiebreak instead of pinning forever. A legacy site that ignores after_id is latched + published as SiteCallReconciliationPinnedChanged on the EventStream (transition-only), replacing the prior silent log line.
This commit is contained in:
Joseph Doherty
@@ -278,6 +278,14 @@ non-draining (e.g., telemetry actor wedged), central issues a
|
||||
are flipped to `ForwardState = 'Reconciled'` site-side. Same self-healing
|
||||
pattern as Site Call Audit's reconciliation of `SiteCalls`.
|
||||
|
||||
> **Cursor keyset (tracked follow-up).** The `PullAuditEvents` cursor is still a
|
||||
> single `sinceUtc` timestamp. Site Call Audit's pull now uses a composite
|
||||
> `(UpdatedAtUtc, TrackedOperationId)` keyset to avoid a single-timestamp pin
|
||||
> (see Component-SiteCallAudit.md → Reconciliation). The same keyset should be
|
||||
> applied here, but it is lower urgency because the audit cursor already re-pulls
|
||||
> idempotently on `EventId` — a saturated single-timestamp window re-inserts
|
||||
> harmless no-ops rather than losing rows.
|
||||
|
||||
### Central direct-write (central-originated events)
|
||||
|
||||
Events originating at central never touch site SQLite. Inbound API writes one
|
||||
|
||||
@@ -98,6 +98,21 @@ reconnect — pulls "all tracking rows changed since cursor X" from each site.
|
||||
Gaps left by lost telemetry self-heal. Central converges to the site; the site
|
||||
never depends on central.
|
||||
|
||||
The per-site cursor is a **composite `(UpdatedAtUtc, TrackedOperationId)`
|
||||
keyset**, not a single timestamp. Each pull asks for rows strictly greater than
|
||||
the cursor pair and advances it to the maximum row seen; a burst of more rows
|
||||
than one batch all sharing one exact `UpdatedAtUtc` therefore drains via the
|
||||
`TrackedOperationId` tiebreak instead of pinning the timestamp forever. The
|
||||
`after_id` keyset field is additive on the pull contract — a first pull (or a
|
||||
**legacy** site that predates it) sends no `after_id` and keeps the inclusive
|
||||
`>=` timestamp behaviour. When such a legacy site keeps reporting
|
||||
`MoreAvailable` yet the composite cursor cannot advance, the actor latches the
|
||||
site as *pinned* and publishes `SiteCallReconciliationPinnedChanged(siteId,
|
||||
Pinned)` on the EventStream (transition-only, mirroring
|
||||
`SiteAuditTelemetryStalledChanged`) — the un-drainable tail is a
|
||||
health-observable condition rather than a silent log line, and the latch clears
|
||||
with `Pinned=false` once a later tick makes progress.
|
||||
|
||||
## Retry / Discard Relay
|
||||
|
||||
Parked cached calls live in the owning site's S&F buffer. Operator Retry/Discard
|
||||
|
||||
@@ -80,6 +80,7 @@ public sealed class GrpcPullSiteCallsClient : IPullSiteCallsClient
|
||||
public async Task<PullSiteCallsResponse> PullAsync(
|
||||
string siteId,
|
||||
DateTime sinceUtc,
|
||||
string? afterId,
|
||||
int batchSize,
|
||||
CancellationToken ct)
|
||||
{
|
||||
@@ -100,6 +101,10 @@ public sealed class GrpcPullSiteCallsClient : IPullSiteCallsClient
|
||||
// EnsureUtc keeps Timestamp.FromDateTime happy (it requires UTC kind).
|
||||
SinceUtc = Timestamp.FromDateTime(EnsureUtc(sinceUtc)),
|
||||
BatchSize = batchSize,
|
||||
// Composite-keyset tiebreak (Task 16). proto3 has no nullable string —
|
||||
// an unset/empty AfterId is the site's signal to keep the legacy
|
||||
// inclusive-timestamp contract (also what a first pull sends).
|
||||
AfterId = afterId ?? string.Empty,
|
||||
};
|
||||
|
||||
ProtoPullResponse reply;
|
||||
|
||||
@@ -46,12 +46,22 @@ public interface IPullSiteCallsClient
|
||||
/// </summary>
|
||||
/// <param name="siteId">The identifier of the site to pull cached-call operational rows from.</param>
|
||||
/// <param name="sinceUtc">Only rows with an <c>UpdatedAtUtc</c> at or after this cursor time are returned.</param>
|
||||
/// <param name="afterId">
|
||||
/// The composite-keyset tiebreak cursor (Task 16). When non-null it is the
|
||||
/// <c>TrackedOperationId</c> of the last row already consumed at
|
||||
/// <paramref name="sinceUtc"/>; the site returns only rows strictly greater
|
||||
/// than the composite <c>(UpdatedAtUtc, TrackedOperationId)</c> pair, so a
|
||||
/// burst sharing one exact <c>UpdatedAtUtc</c> drains via the id tiebreak
|
||||
/// instead of pinning the inclusive-timestamp cursor. Null on the first pull
|
||||
/// (or against a legacy site) preserves the inclusive <c>>=</c> contract.
|
||||
/// </param>
|
||||
/// <param name="batchSize">Maximum number of rows to return per call.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that resolves to the next reconciliation batch with a <c>MoreAvailable</c> flag.</returns>
|
||||
Task<PullSiteCallsResponse> PullAsync(
|
||||
string siteId,
|
||||
DateTime sinceUtc,
|
||||
string? afterId,
|
||||
int batchSize,
|
||||
CancellationToken ct);
|
||||
}
|
||||
|
||||
@@ -131,17 +131,32 @@ public class SiteCallAuditActor : ReceiveActor
|
||||
private readonly bool _backgroundTimersEnabled;
|
||||
|
||||
/// <summary>
|
||||
/// Per-site reconciliation watermark — the highest
|
||||
/// <see cref="SiteCall.UpdatedAtUtc"/> seen for that site on a previous
|
||||
/// tick. The next tick asks for rows at or after this cursor; idempotent
|
||||
/// monotonic <see cref="ISiteCallAuditRepository.UpsertAsync"/> swallows any
|
||||
/// duplicate-with-same-timestamp rows. In-memory for the singleton's
|
||||
/// lifetime — a failover / restart resets every cursor to
|
||||
/// <see cref="DateTime.MinValue"/>, which is conservative but correct
|
||||
/// (the next tick re-pulls and idempotent upsert dedupes). Mirrors
|
||||
/// <c>SiteAuditReconciliationActor</c>.
|
||||
/// Per-site reconciliation watermark — the composite
|
||||
/// <c>(UpdatedAtUtc, TrackedOperationId)</c> keyset of the highest row seen
|
||||
/// for that site on a previous tick (Task 16). The next tick asks for rows
|
||||
/// strictly greater than this pair, so a burst sharing one exact
|
||||
/// <see cref="SiteCall.UpdatedAtUtc"/> drains via the id tiebreak rather than
|
||||
/// pinning the timestamp forever; idempotent monotonic
|
||||
/// <see cref="ISiteCallAuditRepository.UpsertAsync"/> still swallows any
|
||||
/// boundary duplicate. In-memory for the singleton's lifetime — a
|
||||
/// failover / restart resets every cursor to
|
||||
/// <c>(DateTime.MinValue, null)</c>, which is conservative but correct (the
|
||||
/// next tick re-pulls from the start and idempotent upsert dedupes). Mirrors
|
||||
/// <c>SiteAuditReconciliationActor</c>'s cursor (now composite).
|
||||
/// </summary>
|
||||
private readonly Dictionary<string, DateTime> _reconciliationCursors = new();
|
||||
private readonly Dictionary<string, (DateTime Since, string? AfterId)> _reconciliationCursors = new();
|
||||
|
||||
/// <summary>
|
||||
/// Per-site "pinned" latch (Task 16) — <c>true</c> once a site's composite
|
||||
/// cursor stopped advancing while the site still reported
|
||||
/// <c>MoreAvailable=true</c> (in practice only a legacy site that ignores the
|
||||
/// <c>after_id</c> keyset field). Tracks the latch so only <em>transitions</em>
|
||||
/// publish <see cref="SiteCallReconciliationPinnedChanged"/> on the EventStream
|
||||
/// (pinned on latch, unpinned when progress resumes), mirroring the sibling
|
||||
/// <c>SiteAuditReconciliationActor</c>'s stalled-state machine — no flood of
|
||||
/// redundant notifications.
|
||||
/// </summary>
|
||||
private readonly Dictionary<string, bool> _reconciliationPinned = new();
|
||||
|
||||
private ICancelable? _reconciliationTimer;
|
||||
private ICancelable? _purgeTimer;
|
||||
@@ -581,52 +596,56 @@ public class SiteCallAuditActor : ReceiveActor
|
||||
/// not the source of truth, so a slow site simply lags rather than corrupts.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Inclusive cursor boundary.</b> The cursor is advanced to the maximum
|
||||
/// <see cref="SiteCall.UpdatedAtUtc"/> seen, and the pull asks for rows at or
|
||||
/// after it (<c>since</c> is <c>>=</c>, not <c>></c>). The row whose
|
||||
/// timestamp equals the cursor is therefore re-pulled on the next tick and
|
||||
/// deduplicated by the idempotent monotonic upsert — the same inclusive-boundary
|
||||
/// contract as <c>SiteAuditReconciliationActor</c>'s cursor.
|
||||
/// <b>Composite keyset cursor (Task 16).</b> The cursor is the
|
||||
/// <c>(UpdatedAtUtc, TrackedOperationId)</c> pair of the maximum row seen, and
|
||||
/// the pull asks for rows strictly greater than it. The boundary row is NOT
|
||||
/// re-pulled (unlike the prior inclusive-timestamp cursor), and — critically —
|
||||
/// a burst of more than <see cref="SiteCallAuditOptions.ReconciliationBatchSize"/>
|
||||
/// rows sharing one exact <see cref="SiteCall.UpdatedAtUtc"/> now drains via the
|
||||
/// id tiebreak instead of pinning the timestamp forever. The idempotent monotonic
|
||||
/// upsert still dedupes any overlap. Matches the composite keyset the site's
|
||||
/// <c>ReadChangedSinceAsync</c> honours (Task 15).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Consumes <see cref="PullSiteCallsResponse.MoreAvailable"/>
|
||||
/// to guarantee forward progress.</b> Whereas the prior implementation ignored
|
||||
/// the flag entirely and relied solely on the tick cadence, this method now
|
||||
/// continues pulling within the same tick while the site reports
|
||||
/// <c>MoreAvailable=true</c>, bounded by
|
||||
/// <see cref="MaxReconciliationPagesPerTick"/>. This closes the
|
||||
/// single-timestamp-saturation edge: if a backlog larger than
|
||||
/// <see cref="SiteCallAuditOptions.ReconciliationBatchSize"/> all shares one
|
||||
/// exact <see cref="SiteCall.UpdatedAtUtc"/>, the inclusive max-timestamp cursor
|
||||
/// cannot advance, so the previous code re-pulled the identical window forever
|
||||
/// across ticks and never drained the tail. Here, a saturated batch whose
|
||||
/// observed max timestamp did NOT advance past <c>since</c> is detected as a
|
||||
/// no-progress pin: the loop stops and logs a Warning (the same observability
|
||||
/// intent as the sibling's stalled signal, without its EventStream state
|
||||
/// machine), so the pathological site surfaces rather than spinning silently.
|
||||
/// This diverges from <c>SiteAuditReconciliationActor</c>, which reads
|
||||
/// <c>MoreAvailable</c> to drive a <c>SiteAuditTelemetryStalledChanged</c>
|
||||
/// stalled-detection state machine instead of a within-tick continuation drain.
|
||||
/// to guarantee forward progress.</b> The method continues pulling within the
|
||||
/// same tick while the site reports <c>MoreAvailable=true</c>, bounded by
|
||||
/// <see cref="MaxReconciliationPagesPerTick"/> so a misbehaving site can never
|
||||
/// spin the dispatcher. Each page advances the in-flight composite cursor. If a
|
||||
/// page fails to advance the cursor yet the site still reports more available,
|
||||
/// the site is NOT honouring the <c>after_id</c> keyset (a legacy pre-Task-15
|
||||
/// site): continuing would re-pull the identical window forever, so the loop
|
||||
/// latches the site as <em>pinned</em> and publishes
|
||||
/// <see cref="SiteCallReconciliationPinnedChanged"/> on the EventStream — the
|
||||
/// dead-end becomes a health-observable condition rather than a silent log line.
|
||||
/// When a later tick makes progress the latch clears with <c>Pinned=false</c>.
|
||||
/// This aligns with <c>SiteAuditReconciliationActor</c>'s
|
||||
/// <c>SiteAuditTelemetryStalledChanged</c> transition-only publication.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
private async Task ReconcileSiteAsync(
|
||||
SiteEntry site, IPullSiteCallsClient client, ISiteCallAuditRepository repository)
|
||||
{
|
||||
var cursor = _reconciliationCursors.TryGetValue(site.SiteId, out var c) ? c : DateTime.MinValue;
|
||||
var cursor = _reconciliationCursors.TryGetValue(site.SiteId, out var c)
|
||||
? c
|
||||
: (Since: DateTime.MinValue, AfterId: (string?)null);
|
||||
|
||||
// Drain within the tick while the site keeps reporting
|
||||
// MoreAvailable, bounded by MaxReconciliationPagesPerTick so a misbehaving
|
||||
// site can never spin the dispatcher. Each page advances the in-flight
|
||||
// cursor; a saturated page that fails to advance the cursor is the
|
||||
// single-timestamp no-progress pin — break and surface it.
|
||||
// composite cursor; a page that fails to advance it while MoreAvailable is
|
||||
// still true means the site ignores after_id (legacy) — latch + publish
|
||||
// the pinned state and break.
|
||||
for (var page = 0; page < MaxReconciliationPagesPerTick; page++)
|
||||
{
|
||||
var since = cursor;
|
||||
var since = cursor.Since;
|
||||
var afterId = cursor.AfterId;
|
||||
var response = await client
|
||||
.PullAsync(site.SiteId, since, _options.ReconciliationBatchSize, CancellationToken.None)
|
||||
.PullAsync(site.SiteId, since, afterId, _options.ReconciliationBatchSize, CancellationToken.None)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
var maxUpdated = since;
|
||||
var maxAfterId = afterId;
|
||||
var nowUtc = DateTime.UtcNow;
|
||||
foreach (var row in response.SiteCalls)
|
||||
{
|
||||
@@ -637,40 +656,57 @@ public class SiteCallAuditActor : ReceiveActor
|
||||
var siteCall = row with { IngestedAtUtc = nowUtc };
|
||||
await repository.UpsertAsync(siteCall).ConfigureAwait(false);
|
||||
|
||||
if (row.UpdatedAtUtc > maxUpdated)
|
||||
// Advance the composite max: greater timestamp wins; on a tie the
|
||||
// greater TrackedOperationId (ordinal, matching the site's SQLite
|
||||
// BINARY text collation) wins.
|
||||
var rowId = row.TrackedOperationId.ToString();
|
||||
if (row.UpdatedAtUtc > maxUpdated ||
|
||||
(row.UpdatedAtUtc == maxUpdated && string.CompareOrdinal(rowId, maxAfterId) > 0))
|
||||
{
|
||||
maxUpdated = row.UpdatedAtUtc;
|
||||
maxAfterId = rowId;
|
||||
}
|
||||
}
|
||||
|
||||
// Did the composite cursor move? (timestamp advanced OR — at the same
|
||||
// timestamp — the id tiebreak advanced). CompareOrdinal handles nulls
|
||||
// (CompareOrdinal(x, null) > 0 for any non-null x).
|
||||
var advanced = maxUpdated > since || string.CompareOrdinal(maxAfterId, afterId) != 0;
|
||||
|
||||
// Persist the advanced cursor after every page so a fault on a later
|
||||
// page (caught per-site upstream) still keeps the rows already drained.
|
||||
cursor = maxUpdated;
|
||||
cursor = (maxUpdated, maxAfterId);
|
||||
_reconciliationCursors[site.SiteId] = cursor;
|
||||
|
||||
// Progress resumed → clear any prior pinned latch (publishes the
|
||||
// Pinned=false transition exactly once).
|
||||
if (advanced)
|
||||
{
|
||||
SetReconciliationPinned(site.SiteId, pinned: false);
|
||||
}
|
||||
|
||||
if (!response.MoreAvailable)
|
||||
{
|
||||
// Backlog fully drained for this site this tick.
|
||||
return;
|
||||
}
|
||||
|
||||
if (maxUpdated <= since)
|
||||
if (!advanced)
|
||||
{
|
||||
// No-progress pin: the site saturated the batch yet the max
|
||||
// observed UpdatedAtUtc did not advance past the inclusive cursor
|
||||
// (a burst of > batch-size rows sharing one exact timestamp).
|
||||
// Continuing would re-pull the identical window forever, so stop
|
||||
// and surface it — the inclusive max-timestamp cursor cannot make
|
||||
// progress on this input without a composite (timestamp,id)
|
||||
// keyset, which the pull contract does not yet support.
|
||||
// The composite cursor did not advance yet the site still reports
|
||||
// MoreAvailable — the site ignored after_id (a legacy pre-keyset
|
||||
// site), so a continuation would re-pull the identical window
|
||||
// forever. Latch + publish the pinned state and stop; the next tick
|
||||
// re-checks and unpins if the site catches up.
|
||||
SetReconciliationPinned(site.SiteId, pinned: true);
|
||||
_logger.LogWarning(
|
||||
"SiteCallAudit reconciliation for site {SiteId} cannot make progress: a saturated "
|
||||
+ "batch of more than {BatchSize} rows shares a single UpdatedAtUtc ({CursorUtc:o}), "
|
||||
+ "so the inclusive cursor is pinned. The backlog tail beyond the batch ceiling will "
|
||||
+ "not reconcile until those rows' timestamps differ.",
|
||||
"SiteCallAudit reconciliation for site {SiteId} is pinned: the composite "
|
||||
+ "(UpdatedAtUtc, TrackedOperationId) cursor did not advance past ({CursorUtc:o}, {AfterId}) "
|
||||
+ "yet the site reports more rows available — the site does not honour the after_id keyset "
|
||||
+ "(legacy). The backlog tail will not reconcile until the site is upgraded.",
|
||||
site.SiteId,
|
||||
_options.ReconciliationBatchSize,
|
||||
since);
|
||||
since,
|
||||
afterId ?? "(none)");
|
||||
return;
|
||||
}
|
||||
}
|
||||
@@ -687,6 +723,25 @@ public class SiteCallAuditActor : ReceiveActor
|
||||
MaxReconciliationPagesPerTick);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Latches the per-site reconciliation pinned state and publishes
|
||||
/// <see cref="SiteCallReconciliationPinnedChanged"/> on the actor system
|
||||
/// EventStream ONLY on a transition — mirroring
|
||||
/// <c>SiteAuditReconciliationActor.UpdateStalledState</c> so a downstream
|
||||
/// health subscriber never sees a flood of same-state notifications.
|
||||
/// </summary>
|
||||
private void SetReconciliationPinned(string siteId, bool pinned)
|
||||
{
|
||||
var was = _reconciliationPinned.TryGetValue(siteId, out var prior) && prior;
|
||||
if (was == pinned)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_reconciliationPinned[siteId] = pinned;
|
||||
Context.System.EventStream.Publish(new SiteCallReconciliationPinnedChanged(siteId, pinned));
|
||||
}
|
||||
|
||||
// ── Piece B: daily terminal-row purge scheduler ──
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteCallAudit;
|
||||
|
||||
/// <summary>
|
||||
/// Published on the actor system EventStream when a site's Site Call Audit
|
||||
/// reconciliation puller transitions into or out of the "pinned" state — the
|
||||
/// composite <c>(UpdatedAtUtc, TrackedOperationId)</c> keyset cursor could not
|
||||
/// advance yet the site still reports <c>MoreAvailable=true</c>, so the backlog
|
||||
/// tail cannot be drained. In practice this only happens against a legacy site
|
||||
/// that ignores the <c>after_id</c> keyset field (a modern site drains a
|
||||
/// single-timestamp burst via the id tiebreak). Mirrors
|
||||
/// <c>SiteAuditTelemetryStalledChanged</c>: only transitions are published
|
||||
/// (<c>Pinned=true</c> on latch, <c>Pinned=false</c> when progress resumes), so
|
||||
/// a downstream health subscriber never sees a flood of redundant notifications.
|
||||
/// The dead-end that the previous implementation only logged is now a
|
||||
/// health-observable condition.
|
||||
/// </summary>
|
||||
/// <param name="SiteId">The site whose reconciliation cursor pinned (or unpinned).</param>
|
||||
/// <param name="Pinned">True on entering the pinned state; false when the cursor makes progress again.</param>
|
||||
public sealed record SiteCallReconciliationPinnedChanged(string SiteId, bool Pinned);
|
||||
@@ -98,7 +98,7 @@ public class GrpcPullSiteCallsClientTests
|
||||
invoker,
|
||||
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
||||
|
||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
||||
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, CancellationToken.None);
|
||||
|
||||
// Endpoint resolution + request shaping.
|
||||
Assert.Equal("http://site-a:8083", invoker.Endpoint);
|
||||
@@ -131,7 +131,7 @@ public class GrpcPullSiteCallsClientTests
|
||||
invoker,
|
||||
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
||||
|
||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
||||
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, CancellationToken.None);
|
||||
|
||||
Assert.Empty(result.SiteCalls);
|
||||
Assert.False(result.MoreAvailable);
|
||||
@@ -150,7 +150,7 @@ public class GrpcPullSiteCallsClientTests
|
||||
invoker,
|
||||
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
||||
|
||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
||||
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, CancellationToken.None);
|
||||
|
||||
Assert.Empty(result.SiteCalls);
|
||||
Assert.False(result.MoreAvailable);
|
||||
@@ -165,7 +165,7 @@ public class GrpcPullSiteCallsClientTests
|
||||
invoker,
|
||||
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
||||
|
||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
||||
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, CancellationToken.None);
|
||||
|
||||
Assert.Empty(result.SiteCalls);
|
||||
Assert.False(result.MoreAvailable);
|
||||
@@ -180,7 +180,7 @@ public class GrpcPullSiteCallsClientTests
|
||||
invoker,
|
||||
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
||||
|
||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
||||
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, CancellationToken.None);
|
||||
|
||||
Assert.Empty(result.SiteCalls);
|
||||
Assert.False(result.MoreAvailable);
|
||||
@@ -211,7 +211,7 @@ public class GrpcPullSiteCallsClientTests
|
||||
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
||||
|
||||
// Must NOT throw — the bad row is dropped, the good rows are returned.
|
||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
||||
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, CancellationToken.None);
|
||||
|
||||
Assert.Equal(2, result.SiteCalls.Count);
|
||||
// Survivors are oldest-first and SourceSite re-stamped from the dialed siteId.
|
||||
@@ -239,7 +239,7 @@ public class GrpcPullSiteCallsClientTests
|
||||
invoker,
|
||||
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
||||
|
||||
var result = await sut.PullAsync("site-a", minUnspecified, batchSize: 256, CancellationToken.None);
|
||||
var result = await sut.PullAsync("site-a", minUnspecified, afterId: null, batchSize: 256, CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, invoker.CallCount);
|
||||
Assert.Equal("http://site-a:8083", invoker.Endpoint);
|
||||
|
||||
@@ -42,7 +42,7 @@ public class SiteCallAuditPurgeTests : TestKit
|
||||
private sealed class NoOpPullClient : IPullSiteCallsClient
|
||||
{
|
||||
public Task<PullSiteCallsResponse> PullAsync(
|
||||
string siteId, DateTime sinceUtc, int batchSize, CancellationToken ct) =>
|
||||
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct) =>
|
||||
Task.FromResult(new PullSiteCallsResponse(Array.Empty<SiteCall>(), MoreAvailable: false));
|
||||
}
|
||||
|
||||
|
||||
+150
-4
@@ -74,7 +74,7 @@ public class SiteCallAuditReconciliationTests : TestKit
|
||||
/// </summary>
|
||||
private sealed class ScriptedPullClient : IPullSiteCallsClient
|
||||
{
|
||||
public List<(string SiteId, DateTime SinceUtc, int BatchSize)> Calls { get; } = new();
|
||||
public List<(string SiteId, DateTime SinceUtc, string? AfterId, int BatchSize)> Calls { get; } = new();
|
||||
private readonly Dictionary<string, Queue<PullSiteCallsResponse>> _scripted = new();
|
||||
private readonly Dictionary<string, Exception> _throwOnSite = new();
|
||||
|
||||
@@ -91,9 +91,9 @@ public class SiteCallAuditReconciliationTests : TestKit
|
||||
}
|
||||
|
||||
public Task<PullSiteCallsResponse> PullAsync(
|
||||
string siteId, DateTime sinceUtc, int batchSize, CancellationToken ct)
|
||||
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
|
||||
{
|
||||
Calls.Add((siteId, sinceUtc, batchSize));
|
||||
Calls.Add((siteId, sinceUtc, afterId, batchSize));
|
||||
if (_throwOnSite.TryGetValue(siteId, out var ex))
|
||||
{
|
||||
throw ex;
|
||||
@@ -107,6 +107,67 @@ public class SiteCallAuditReconciliationTests : TestKit
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Keyset-honouring in-memory pull client (Task 16). Holds one static set of
|
||||
/// rows and answers each pull as a real site would: returns the next
|
||||
/// <c>batchSize</c> rows strictly greater than the composite
|
||||
/// <c>(sinceUtc, afterId)</c> cursor under the deterministic
|
||||
/// <c>(UpdatedAtUtc, TrackedOperationId ordinal)</c> ordering, with
|
||||
/// <c>MoreAvailable=true</c> when unseen rows remain. This is the modern site
|
||||
/// that DOES honour <c>after_id</c>, so a burst sharing one exact
|
||||
/// <c>UpdatedAtUtc</c> drains fully via the id tiebreak — no single-timestamp
|
||||
/// pin.
|
||||
/// </summary>
|
||||
private sealed class KeysetHonoringPullClient : IPullSiteCallsClient
|
||||
{
|
||||
private readonly IReadOnlyList<SiteCall> _rows;
|
||||
public int CallCount { get; private set; }
|
||||
|
||||
public KeysetHonoringPullClient(IEnumerable<SiteCall> rows) =>
|
||||
_rows = rows
|
||||
.OrderBy(r => r.UpdatedAtUtc)
|
||||
.ThenBy(r => r.TrackedOperationId.ToString(), StringComparer.Ordinal)
|
||||
.ToList();
|
||||
|
||||
public Task<PullSiteCallsResponse> PullAsync(
|
||||
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
|
||||
{
|
||||
CallCount++;
|
||||
var greater = _rows
|
||||
.Where(r =>
|
||||
r.UpdatedAtUtc > sinceUtc ||
|
||||
(r.UpdatedAtUtc == sinceUtc &&
|
||||
string.CompareOrdinal(r.TrackedOperationId.ToString(), afterId) > 0))
|
||||
.ToList();
|
||||
var page = greater.Take(batchSize).ToList();
|
||||
var more = greater.Count > page.Count;
|
||||
return Task.FromResult(
|
||||
new PullSiteCallsResponse(page, MoreAvailable: more));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Legacy pull client (Task 16) that IGNORES <c>afterId</c> and always
|
||||
/// returns the same saturated page (<c>MoreAvailable=true</c>) — models a
|
||||
/// pre-keyset site the central actor cannot drain past. The composite cursor
|
||||
/// cannot advance, so the actor must latch and publish
|
||||
/// <see cref="SiteCallReconciliationPinnedChanged"/> instead of spinning.
|
||||
/// </summary>
|
||||
private sealed class LegacyIgnoresAfterIdPullClient : IPullSiteCallsClient
|
||||
{
|
||||
private readonly IReadOnlyList<SiteCall> _rows;
|
||||
public int CallCount { get; private set; }
|
||||
|
||||
public LegacyIgnoresAfterIdPullClient(IReadOnlyList<SiteCall> rows) => _rows = rows;
|
||||
|
||||
public Task<PullSiteCallsResponse> PullAsync(
|
||||
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
|
||||
{
|
||||
CallCount++;
|
||||
return Task.FromResult(new PullSiteCallsResponse(_rows, MoreAvailable: true));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pull client that ALWAYS returns the same saturated response
|
||||
/// (<c>MoreAvailable=true</c>) regardless of the <c>since</c> cursor —
|
||||
@@ -124,7 +185,7 @@ public class SiteCallAuditReconciliationTests : TestKit
|
||||
public SaturatedPinPullClient(IReadOnlyList<SiteCall> rows) => _rows = rows;
|
||||
|
||||
public Task<PullSiteCallsResponse> PullAsync(
|
||||
string siteId, DateTime sinceUtc, int batchSize, CancellationToken ct)
|
||||
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
|
||||
{
|
||||
CallCount++;
|
||||
return Task.FromResult(new PullSiteCallsResponse(_rows, MoreAvailable: true));
|
||||
@@ -436,4 +497,89 @@ public class SiteCallAuditReconciliationTests : TestKit
|
||||
$"a single-timestamp saturation pin must break the within-tick drain, not spin to the "
|
||||
+ $"page ceiling; got {client.CallCount} pulls (an unbounded within-tick loop would be 50+)");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 7. Task 16: a keyset-honouring site drains a single-timestamp burst
|
||||
// LARGER than the batch size in ONE tick — the composite (UpdatedAtUtc,
|
||||
// TrackedOperationId) cursor advances by id even when every row shares
|
||||
// one exact timestamp, so the tail no longer pins forever.
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void ReconciliationTick_KeysetCursor_DrainsSingleTimestampBurstBeyondBatchSize()
|
||||
{
|
||||
var siteId = "siteA";
|
||||
// 2 × batchSize rows all sharing ONE exact UpdatedAtUtc: the pre-keyset
|
||||
// inclusive-timestamp cursor could never advance past this window.
|
||||
var ts = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);
|
||||
var rows = Enumerable.Range(0, 4)
|
||||
.Select(_ => NewRow(TrackedOperationId.New(), siteId, updatedAtUtc: ts))
|
||||
.ToList();
|
||||
|
||||
var sites = new StaticEnumerator(new SiteEntry(siteId, "http://siteA:8083"));
|
||||
var client = new KeysetHonoringPullClient(rows);
|
||||
var repo = new RecordingRepo();
|
||||
|
||||
// batchSize = 2 → the 4-row burst needs the id tiebreak to page through.
|
||||
// Slow tick so the full drain must be the within-tick continuation loop,
|
||||
// not multiple ticks.
|
||||
var options = new SiteCallAuditOptions
|
||||
{
|
||||
ReconciliationIntervalOverride = TimeSpan.FromSeconds(2),
|
||||
ReconciliationBatchSize = 2,
|
||||
};
|
||||
|
||||
CreateActor(sites, client, repo, options);
|
||||
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
foreach (var r in rows)
|
||||
{
|
||||
Assert.True(repo.Upserted.ContainsKey(r.TrackedOperationId),
|
||||
"every row in the single-timestamp burst must reconcile via the "
|
||||
+ "composite keyset cursor within one tick");
|
||||
}
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(3),
|
||||
interval: TimeSpan.FromMilliseconds(50));
|
||||
|
||||
// Two pages (2 + 2) drained the burst; well under the 50-page ceiling.
|
||||
Assert.True(client.CallCount is >= 2 and < 10,
|
||||
$"expected the burst to drain in a couple of keyset pages, got {client.CallCount}");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 8. Task 16: a LEGACY site that ignores after_id can never be drained, so
|
||||
// the actor latches and publishes SiteCallReconciliationPinnedChanged
|
||||
// instead of silently logging (the dead-end is now health-observable).
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void ReconciliationTick_LegacySiteIgnoresAfterId_PublishesPinnedChanged()
|
||||
{
|
||||
var siteId = "siteLegacy";
|
||||
var ts = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);
|
||||
var r1 = NewRow(TrackedOperationId.New(), siteId, updatedAtUtc: ts);
|
||||
var r2 = NewRow(TrackedOperationId.New(), siteId, updatedAtUtc: ts);
|
||||
|
||||
var sites = new StaticEnumerator(new SiteEntry(siteId, "http://siteLegacy:8083"));
|
||||
var client = new LegacyIgnoresAfterIdPullClient(new[] { r1, r2 });
|
||||
var repo = new RecordingRepo();
|
||||
|
||||
// Subscribe BEFORE the actor starts ticking so the transition isn't missed.
|
||||
Sys.EventStream.Subscribe(TestActor, typeof(SiteCallReconciliationPinnedChanged));
|
||||
|
||||
var options = new SiteCallAuditOptions
|
||||
{
|
||||
ReconciliationIntervalOverride = TimeSpan.FromMilliseconds(100),
|
||||
ReconciliationBatchSize = 2,
|
||||
};
|
||||
|
||||
CreateActor(sites, client, repo, options);
|
||||
|
||||
var evt = ExpectMsg<SiteCallReconciliationPinnedChanged>(TimeSpan.FromSeconds(5));
|
||||
Assert.Equal(siteId, evt.SiteId);
|
||||
Assert.True(evt.Pinned, "a legacy site that ignores after_id must publish Pinned=true");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user