fix(site-call-audit): composite keyset cursor eliminates the single-timestamp reconciliation pin; pinned state now a published event
Reconciliation cursor becomes composite (UpdatedAtUtc, TrackedOperationId); IPullSiteCallsClient/GrpcPullSiteCallsClient forward the Task-15 after_id keyset (additive param, null preserves the legacy inclusive >= contract). A burst sharing one exact UpdatedAtUtc now drains via the id tiebreak instead of pinning forever. A legacy site that ignores after_id is latched + published as SiteCallReconciliationPinnedChanged on the EventStream (transition-only), replacing the prior silent log line.
This commit is contained in:
@@ -278,6 +278,14 @@ non-draining (e.g., telemetry actor wedged), central issues a
|
|||||||
are flipped to `ForwardState = 'Reconciled'` site-side. Same self-healing
|
are flipped to `ForwardState = 'Reconciled'` site-side. Same self-healing
|
||||||
pattern as Site Call Audit's reconciliation of `SiteCalls`.
|
pattern as Site Call Audit's reconciliation of `SiteCalls`.
|
||||||
|
|
||||||
|
> **Cursor keyset (tracked follow-up).** The `PullAuditEvents` cursor is still a
|
||||||
|
> single `sinceUtc` timestamp. Site Call Audit's pull now uses a composite
|
||||||
|
> `(UpdatedAtUtc, TrackedOperationId)` keyset to avoid a single-timestamp pin
|
||||||
|
> (see Component-SiteCallAudit.md → Reconciliation). The same keyset should be
|
||||||
|
> applied here, but it is lower urgency because the audit cursor already re-pulls
|
||||||
|
> idempotently on `EventId` — a saturated single-timestamp window re-inserts
|
||||||
|
> harmless no-ops rather than losing rows.
|
||||||
|
|
||||||
### Central direct-write (central-originated events)
|
### Central direct-write (central-originated events)
|
||||||
|
|
||||||
Events originating at central never touch site SQLite. Inbound API writes one
|
Events originating at central never touch site SQLite. Inbound API writes one
|
||||||
|
|||||||
@@ -98,6 +98,21 @@ reconnect — pulls "all tracking rows changed since cursor X" from each site.
|
|||||||
Gaps left by lost telemetry self-heal. Central converges to the site; the site
|
Gaps left by lost telemetry self-heal. Central converges to the site; the site
|
||||||
never depends on central.
|
never depends on central.
|
||||||
|
|
||||||
|
The per-site cursor is a **composite `(UpdatedAtUtc, TrackedOperationId)`
|
||||||
|
keyset**, not a single timestamp. Each pull asks for rows strictly greater than
|
||||||
|
the cursor pair and advances it to the maximum row seen; a burst of more rows
|
||||||
|
than one batch all sharing one exact `UpdatedAtUtc` therefore drains via the
|
||||||
|
`TrackedOperationId` tiebreak instead of pinning the timestamp forever. The
|
||||||
|
`after_id` keyset field is additive on the pull contract — a first pull (or a
|
||||||
|
**legacy** site that predates it) sends no `after_id` and keeps the inclusive
|
||||||
|
`>=` timestamp behaviour. When such a legacy site keeps reporting
|
||||||
|
`MoreAvailable` yet the composite cursor cannot advance, the actor latches the
|
||||||
|
site as *pinned* and publishes `SiteCallReconciliationPinnedChanged(siteId,
|
||||||
|
Pinned)` on the EventStream (transition-only, mirroring
|
||||||
|
`SiteAuditTelemetryStalledChanged`) — the un-drainable tail is a
|
||||||
|
health-observable condition rather than a silent log line, and the latch clears
|
||||||
|
with `Pinned=false` once a later tick makes progress.
|
||||||
|
|
||||||
## Retry / Discard Relay
|
## Retry / Discard Relay
|
||||||
|
|
||||||
Parked cached calls live in the owning site's S&F buffer. Operator Retry/Discard
|
Parked cached calls live in the owning site's S&F buffer. Operator Retry/Discard
|
||||||
|
|||||||
@@ -80,6 +80,7 @@ public sealed class GrpcPullSiteCallsClient : IPullSiteCallsClient
|
|||||||
public async Task<PullSiteCallsResponse> PullAsync(
|
public async Task<PullSiteCallsResponse> PullAsync(
|
||||||
string siteId,
|
string siteId,
|
||||||
DateTime sinceUtc,
|
DateTime sinceUtc,
|
||||||
|
string? afterId,
|
||||||
int batchSize,
|
int batchSize,
|
||||||
CancellationToken ct)
|
CancellationToken ct)
|
||||||
{
|
{
|
||||||
@@ -100,6 +101,10 @@ public sealed class GrpcPullSiteCallsClient : IPullSiteCallsClient
|
|||||||
// EnsureUtc keeps Timestamp.FromDateTime happy (it requires UTC kind).
|
// EnsureUtc keeps Timestamp.FromDateTime happy (it requires UTC kind).
|
||||||
SinceUtc = Timestamp.FromDateTime(EnsureUtc(sinceUtc)),
|
SinceUtc = Timestamp.FromDateTime(EnsureUtc(sinceUtc)),
|
||||||
BatchSize = batchSize,
|
BatchSize = batchSize,
|
||||||
|
// Composite-keyset tiebreak (Task 16). proto3 has no nullable string —
|
||||||
|
// an unset/empty AfterId is the site's signal to keep the legacy
|
||||||
|
// inclusive-timestamp contract (also what a first pull sends).
|
||||||
|
AfterId = afterId ?? string.Empty,
|
||||||
};
|
};
|
||||||
|
|
||||||
ProtoPullResponse reply;
|
ProtoPullResponse reply;
|
||||||
|
|||||||
@@ -46,12 +46,22 @@ public interface IPullSiteCallsClient
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="siteId">The identifier of the site to pull cached-call operational rows from.</param>
|
/// <param name="siteId">The identifier of the site to pull cached-call operational rows from.</param>
|
||||||
/// <param name="sinceUtc">Only rows with an <c>UpdatedAtUtc</c> at or after this cursor time are returned.</param>
|
/// <param name="sinceUtc">Only rows with an <c>UpdatedAtUtc</c> at or after this cursor time are returned.</param>
|
||||||
|
/// <param name="afterId">
|
||||||
|
/// The composite-keyset tiebreak cursor (Task 16). When non-null it is the
|
||||||
|
/// <c>TrackedOperationId</c> of the last row already consumed at
|
||||||
|
/// <paramref name="sinceUtc"/>; the site returns only rows strictly greater
|
||||||
|
/// than the composite <c>(UpdatedAtUtc, TrackedOperationId)</c> pair, so a
|
||||||
|
/// burst sharing one exact <c>UpdatedAtUtc</c> drains via the id tiebreak
|
||||||
|
/// instead of pinning the inclusive-timestamp cursor. Null on the first pull
|
||||||
|
/// (or against a legacy site) preserves the inclusive <c>>=</c> contract.
|
||||||
|
/// </param>
|
||||||
/// <param name="batchSize">Maximum number of rows to return per call.</param>
|
/// <param name="batchSize">Maximum number of rows to return per call.</param>
|
||||||
/// <param name="ct">Cancellation token.</param>
|
/// <param name="ct">Cancellation token.</param>
|
||||||
/// <returns>A task that resolves to the next reconciliation batch with a <c>MoreAvailable</c> flag.</returns>
|
/// <returns>A task that resolves to the next reconciliation batch with a <c>MoreAvailable</c> flag.</returns>
|
||||||
Task<PullSiteCallsResponse> PullAsync(
|
Task<PullSiteCallsResponse> PullAsync(
|
||||||
string siteId,
|
string siteId,
|
||||||
DateTime sinceUtc,
|
DateTime sinceUtc,
|
||||||
|
string? afterId,
|
||||||
int batchSize,
|
int batchSize,
|
||||||
CancellationToken ct);
|
CancellationToken ct);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -131,17 +131,32 @@ public class SiteCallAuditActor : ReceiveActor
|
|||||||
private readonly bool _backgroundTimersEnabled;
|
private readonly bool _backgroundTimersEnabled;
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Per-site reconciliation watermark — the highest
|
/// Per-site reconciliation watermark — the composite
|
||||||
/// <see cref="SiteCall.UpdatedAtUtc"/> seen for that site on a previous
|
/// <c>(UpdatedAtUtc, TrackedOperationId)</c> keyset of the highest row seen
|
||||||
/// tick. The next tick asks for rows at or after this cursor; idempotent
|
/// for that site on a previous tick (Task 16). The next tick asks for rows
|
||||||
/// monotonic <see cref="ISiteCallAuditRepository.UpsertAsync"/> swallows any
|
/// strictly greater than this pair, so a burst sharing one exact
|
||||||
/// duplicate-with-same-timestamp rows. In-memory for the singleton's
|
/// <see cref="SiteCall.UpdatedAtUtc"/> drains via the id tiebreak rather than
|
||||||
/// lifetime — a failover / restart resets every cursor to
|
/// pinning the timestamp forever; idempotent monotonic
|
||||||
/// <see cref="DateTime.MinValue"/>, which is conservative but correct
|
/// <see cref="ISiteCallAuditRepository.UpsertAsync"/> still swallows any
|
||||||
/// (the next tick re-pulls and idempotent upsert dedupes). Mirrors
|
/// boundary duplicate. In-memory for the singleton's lifetime — a
|
||||||
/// <c>SiteAuditReconciliationActor</c>.
|
/// failover / restart resets every cursor to
|
||||||
|
/// <c>(DateTime.MinValue, null)</c>, which is conservative but correct (the
|
||||||
|
/// next tick re-pulls from the start and idempotent upsert dedupes). Mirrors
|
||||||
|
/// <c>SiteAuditReconciliationActor</c>'s cursor (now composite).
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private readonly Dictionary<string, DateTime> _reconciliationCursors = new();
|
private readonly Dictionary<string, (DateTime Since, string? AfterId)> _reconciliationCursors = new();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Per-site "pinned" latch (Task 16) — <c>true</c> once a site's composite
|
||||||
|
/// cursor stopped advancing while the site still reported
|
||||||
|
/// <c>MoreAvailable=true</c> (in practice only a legacy site that ignores the
|
||||||
|
/// <c>after_id</c> keyset field). Tracks the latch so only <em>transitions</em>
|
||||||
|
/// publish <see cref="SiteCallReconciliationPinnedChanged"/> on the EventStream
|
||||||
|
/// (pinned on latch, unpinned when progress resumes), mirroring the sibling
|
||||||
|
/// <c>SiteAuditReconciliationActor</c>'s stalled-state machine — no flood of
|
||||||
|
/// redundant notifications.
|
||||||
|
/// </summary>
|
||||||
|
private readonly Dictionary<string, bool> _reconciliationPinned = new();
|
||||||
|
|
||||||
private ICancelable? _reconciliationTimer;
|
private ICancelable? _reconciliationTimer;
|
||||||
private ICancelable? _purgeTimer;
|
private ICancelable? _purgeTimer;
|
||||||
@@ -581,52 +596,56 @@ public class SiteCallAuditActor : ReceiveActor
|
|||||||
/// not the source of truth, so a slow site simply lags rather than corrupts.
|
/// not the source of truth, so a slow site simply lags rather than corrupts.
|
||||||
/// </para>
|
/// </para>
|
||||||
/// <para>
|
/// <para>
|
||||||
/// <b>Inclusive cursor boundary.</b> The cursor is advanced to the maximum
|
/// <b>Composite keyset cursor (Task 16).</b> The cursor is the
|
||||||
/// <see cref="SiteCall.UpdatedAtUtc"/> seen, and the pull asks for rows at or
|
/// <c>(UpdatedAtUtc, TrackedOperationId)</c> pair of the maximum row seen, and
|
||||||
/// after it (<c>since</c> is <c>>=</c>, not <c>></c>). The row whose
|
/// the pull asks for rows strictly greater than it. The boundary row is NOT
|
||||||
/// timestamp equals the cursor is therefore re-pulled on the next tick and
|
/// re-pulled (unlike the prior inclusive-timestamp cursor), and — critically —
|
||||||
/// deduplicated by the idempotent monotonic upsert — the same inclusive-boundary
|
/// a burst of more than <see cref="SiteCallAuditOptions.ReconciliationBatchSize"/>
|
||||||
/// contract as <c>SiteAuditReconciliationActor</c>'s cursor.
|
/// rows sharing one exact <see cref="SiteCall.UpdatedAtUtc"/> now drains via the
|
||||||
|
/// id tiebreak instead of pinning the timestamp forever. The idempotent monotonic
|
||||||
|
/// upsert still dedupes any overlap. Matches the composite keyset the site's
|
||||||
|
/// <c>ReadChangedSinceAsync</c> honours (Task 15).
|
||||||
/// </para>
|
/// </para>
|
||||||
/// <para>
|
/// <para>
|
||||||
/// <b>Consumes <see cref="PullSiteCallsResponse.MoreAvailable"/>
|
/// <b>Consumes <see cref="PullSiteCallsResponse.MoreAvailable"/>
|
||||||
/// to guarantee forward progress.</b> Whereas the prior implementation ignored
|
/// to guarantee forward progress.</b> The method continues pulling within the
|
||||||
/// the flag entirely and relied solely on the tick cadence, this method now
|
/// same tick while the site reports <c>MoreAvailable=true</c>, bounded by
|
||||||
/// continues pulling within the same tick while the site reports
|
/// <see cref="MaxReconciliationPagesPerTick"/> so a misbehaving site can never
|
||||||
/// <c>MoreAvailable=true</c>, bounded by
|
/// spin the dispatcher. Each page advances the in-flight composite cursor. If a
|
||||||
/// <see cref="MaxReconciliationPagesPerTick"/>. This closes the
|
/// page fails to advance the cursor yet the site still reports more available,
|
||||||
/// single-timestamp-saturation edge: if a backlog larger than
|
/// the site is NOT honouring the <c>after_id</c> keyset (a legacy pre-Task-15
|
||||||
/// <see cref="SiteCallAuditOptions.ReconciliationBatchSize"/> all shares one
|
/// site): continuing would re-pull the identical window forever, so the loop
|
||||||
/// exact <see cref="SiteCall.UpdatedAtUtc"/>, the inclusive max-timestamp cursor
|
/// latches the site as <em>pinned</em> and publishes
|
||||||
/// cannot advance, so the previous code re-pulled the identical window forever
|
/// <see cref="SiteCallReconciliationPinnedChanged"/> on the EventStream — the
|
||||||
/// across ticks and never drained the tail. Here, a saturated batch whose
|
/// dead-end becomes a health-observable condition rather than a silent log line.
|
||||||
/// observed max timestamp did NOT advance past <c>since</c> is detected as a
|
/// When a later tick makes progress the latch clears with <c>Pinned=false</c>.
|
||||||
/// no-progress pin: the loop stops and logs a Warning (the same observability
|
/// This aligns with <c>SiteAuditReconciliationActor</c>'s
|
||||||
/// intent as the sibling's stalled signal, without its EventStream state
|
/// <c>SiteAuditTelemetryStalledChanged</c> transition-only publication.
|
||||||
/// machine), so the pathological site surfaces rather than spinning silently.
|
|
||||||
/// This diverges from <c>SiteAuditReconciliationActor</c>, which reads
|
|
||||||
/// <c>MoreAvailable</c> to drive a <c>SiteAuditTelemetryStalledChanged</c>
|
|
||||||
/// stalled-detection state machine instead of a within-tick continuation drain.
|
|
||||||
/// </para>
|
/// </para>
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
private async Task ReconcileSiteAsync(
|
private async Task ReconcileSiteAsync(
|
||||||
SiteEntry site, IPullSiteCallsClient client, ISiteCallAuditRepository repository)
|
SiteEntry site, IPullSiteCallsClient client, ISiteCallAuditRepository repository)
|
||||||
{
|
{
|
||||||
var cursor = _reconciliationCursors.TryGetValue(site.SiteId, out var c) ? c : DateTime.MinValue;
|
var cursor = _reconciliationCursors.TryGetValue(site.SiteId, out var c)
|
||||||
|
? c
|
||||||
|
: (Since: DateTime.MinValue, AfterId: (string?)null);
|
||||||
|
|
||||||
// Drain within the tick while the site keeps reporting
|
// Drain within the tick while the site keeps reporting
|
||||||
// MoreAvailable, bounded by MaxReconciliationPagesPerTick so a misbehaving
|
// MoreAvailable, bounded by MaxReconciliationPagesPerTick so a misbehaving
|
||||||
// site can never spin the dispatcher. Each page advances the in-flight
|
// site can never spin the dispatcher. Each page advances the in-flight
|
||||||
// cursor; a saturated page that fails to advance the cursor is the
|
// composite cursor; a page that fails to advance it while MoreAvailable is
|
||||||
// single-timestamp no-progress pin — break and surface it.
|
// still true means the site ignores after_id (legacy) — latch + publish
|
||||||
|
// the pinned state and break.
|
||||||
for (var page = 0; page < MaxReconciliationPagesPerTick; page++)
|
for (var page = 0; page < MaxReconciliationPagesPerTick; page++)
|
||||||
{
|
{
|
||||||
var since = cursor;
|
var since = cursor.Since;
|
||||||
|
var afterId = cursor.AfterId;
|
||||||
var response = await client
|
var response = await client
|
||||||
.PullAsync(site.SiteId, since, _options.ReconciliationBatchSize, CancellationToken.None)
|
.PullAsync(site.SiteId, since, afterId, _options.ReconciliationBatchSize, CancellationToken.None)
|
||||||
.ConfigureAwait(false);
|
.ConfigureAwait(false);
|
||||||
|
|
||||||
var maxUpdated = since;
|
var maxUpdated = since;
|
||||||
|
var maxAfterId = afterId;
|
||||||
var nowUtc = DateTime.UtcNow;
|
var nowUtc = DateTime.UtcNow;
|
||||||
foreach (var row in response.SiteCalls)
|
foreach (var row in response.SiteCalls)
|
||||||
{
|
{
|
||||||
@@ -637,40 +656,57 @@ public class SiteCallAuditActor : ReceiveActor
|
|||||||
var siteCall = row with { IngestedAtUtc = nowUtc };
|
var siteCall = row with { IngestedAtUtc = nowUtc };
|
||||||
await repository.UpsertAsync(siteCall).ConfigureAwait(false);
|
await repository.UpsertAsync(siteCall).ConfigureAwait(false);
|
||||||
|
|
||||||
if (row.UpdatedAtUtc > maxUpdated)
|
// Advance the composite max: greater timestamp wins; on a tie the
|
||||||
|
// greater TrackedOperationId (ordinal, matching the site's SQLite
|
||||||
|
// BINARY text collation) wins.
|
||||||
|
var rowId = row.TrackedOperationId.ToString();
|
||||||
|
if (row.UpdatedAtUtc > maxUpdated ||
|
||||||
|
(row.UpdatedAtUtc == maxUpdated && string.CompareOrdinal(rowId, maxAfterId) > 0))
|
||||||
{
|
{
|
||||||
maxUpdated = row.UpdatedAtUtc;
|
maxUpdated = row.UpdatedAtUtc;
|
||||||
|
maxAfterId = rowId;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Did the composite cursor move? (timestamp advanced OR — at the same
|
||||||
|
// timestamp — the id tiebreak advanced). CompareOrdinal handles nulls
|
||||||
|
// (CompareOrdinal(x, null) > 0 for any non-null x).
|
||||||
|
var advanced = maxUpdated > since || string.CompareOrdinal(maxAfterId, afterId) != 0;
|
||||||
|
|
||||||
// Persist the advanced cursor after every page so a fault on a later
|
// Persist the advanced cursor after every page so a fault on a later
|
||||||
// page (caught per-site upstream) still keeps the rows already drained.
|
// page (caught per-site upstream) still keeps the rows already drained.
|
||||||
cursor = maxUpdated;
|
cursor = (maxUpdated, maxAfterId);
|
||||||
_reconciliationCursors[site.SiteId] = cursor;
|
_reconciliationCursors[site.SiteId] = cursor;
|
||||||
|
|
||||||
|
// Progress resumed → clear any prior pinned latch (publishes the
|
||||||
|
// Pinned=false transition exactly once).
|
||||||
|
if (advanced)
|
||||||
|
{
|
||||||
|
SetReconciliationPinned(site.SiteId, pinned: false);
|
||||||
|
}
|
||||||
|
|
||||||
if (!response.MoreAvailable)
|
if (!response.MoreAvailable)
|
||||||
{
|
{
|
||||||
// Backlog fully drained for this site this tick.
|
// Backlog fully drained for this site this tick.
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (maxUpdated <= since)
|
if (!advanced)
|
||||||
{
|
{
|
||||||
// No-progress pin: the site saturated the batch yet the max
|
// The composite cursor did not advance yet the site still reports
|
||||||
// observed UpdatedAtUtc did not advance past the inclusive cursor
|
// MoreAvailable — the site ignored after_id (a legacy pre-keyset
|
||||||
// (a burst of > batch-size rows sharing one exact timestamp).
|
// site), so a continuation would re-pull the identical window
|
||||||
// Continuing would re-pull the identical window forever, so stop
|
// forever. Latch + publish the pinned state and stop; the next tick
|
||||||
// and surface it — the inclusive max-timestamp cursor cannot make
|
// re-checks and unpins if the site catches up.
|
||||||
// progress on this input without a composite (timestamp,id)
|
SetReconciliationPinned(site.SiteId, pinned: true);
|
||||||
// keyset, which the pull contract does not yet support.
|
|
||||||
_logger.LogWarning(
|
_logger.LogWarning(
|
||||||
"SiteCallAudit reconciliation for site {SiteId} cannot make progress: a saturated "
|
"SiteCallAudit reconciliation for site {SiteId} is pinned: the composite "
|
||||||
+ "batch of more than {BatchSize} rows shares a single UpdatedAtUtc ({CursorUtc:o}), "
|
+ "(UpdatedAtUtc, TrackedOperationId) cursor did not advance past ({CursorUtc:o}, {AfterId}) "
|
||||||
+ "so the inclusive cursor is pinned. The backlog tail beyond the batch ceiling will "
|
+ "yet the site reports more rows available — the site does not honour the after_id keyset "
|
||||||
+ "not reconcile until those rows' timestamps differ.",
|
+ "(legacy). The backlog tail will not reconcile until the site is upgraded.",
|
||||||
site.SiteId,
|
site.SiteId,
|
||||||
_options.ReconciliationBatchSize,
|
since,
|
||||||
since);
|
afterId ?? "(none)");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -687,6 +723,25 @@ public class SiteCallAuditActor : ReceiveActor
|
|||||||
MaxReconciliationPagesPerTick);
|
MaxReconciliationPagesPerTick);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Latches the per-site reconciliation pinned state and publishes
|
||||||
|
/// <see cref="SiteCallReconciliationPinnedChanged"/> on the actor system
|
||||||
|
/// EventStream ONLY on a transition — mirroring
|
||||||
|
/// <c>SiteAuditReconciliationActor.UpdateStalledState</c> so a downstream
|
||||||
|
/// health subscriber never sees a flood of same-state notifications.
|
||||||
|
/// </summary>
|
||||||
|
private void SetReconciliationPinned(string siteId, bool pinned)
|
||||||
|
{
|
||||||
|
var was = _reconciliationPinned.TryGetValue(siteId, out var prior) && prior;
|
||||||
|
if (was == pinned)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
_reconciliationPinned[siteId] = pinned;
|
||||||
|
Context.System.EventStream.Publish(new SiteCallReconciliationPinnedChanged(siteId, pinned));
|
||||||
|
}
|
||||||
|
|
||||||
// ── Piece B: daily terminal-row purge scheduler ──
|
// ── Piece B: daily terminal-row purge scheduler ──
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
namespace ZB.MOM.WW.ScadaBridge.SiteCallAudit;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Published on the actor system EventStream when a site's Site Call Audit
|
||||||
|
/// reconciliation puller transitions into or out of the "pinned" state — the
|
||||||
|
/// composite <c>(UpdatedAtUtc, TrackedOperationId)</c> keyset cursor could not
|
||||||
|
/// advance yet the site still reports <c>MoreAvailable=true</c>, so the backlog
|
||||||
|
/// tail cannot be drained. In practice this only happens against a legacy site
|
||||||
|
/// that ignores the <c>after_id</c> keyset field (a modern site drains a
|
||||||
|
/// single-timestamp burst via the id tiebreak). Mirrors
|
||||||
|
/// <c>SiteAuditTelemetryStalledChanged</c>: only transitions are published
|
||||||
|
/// (<c>Pinned=true</c> on latch, <c>Pinned=false</c> when progress resumes), so
|
||||||
|
/// a downstream health subscriber never sees a flood of redundant notifications.
|
||||||
|
/// The dead-end that the previous implementation only logged is now a
|
||||||
|
/// health-observable condition.
|
||||||
|
/// </summary>
|
||||||
|
/// <param name="SiteId">The site whose reconciliation cursor pinned (or unpinned).</param>
|
||||||
|
/// <param name="Pinned">True on entering the pinned state; false when the cursor makes progress again.</param>
|
||||||
|
public sealed record SiteCallReconciliationPinnedChanged(string SiteId, bool Pinned);
|
||||||
@@ -98,7 +98,7 @@ public class GrpcPullSiteCallsClientTests
|
|||||||
invoker,
|
invoker,
|
||||||
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
||||||
|
|
||||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, CancellationToken.None);
|
||||||
|
|
||||||
// Endpoint resolution + request shaping.
|
// Endpoint resolution + request shaping.
|
||||||
Assert.Equal("http://site-a:8083", invoker.Endpoint);
|
Assert.Equal("http://site-a:8083", invoker.Endpoint);
|
||||||
@@ -131,7 +131,7 @@ public class GrpcPullSiteCallsClientTests
|
|||||||
invoker,
|
invoker,
|
||||||
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
||||||
|
|
||||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, CancellationToken.None);
|
||||||
|
|
||||||
Assert.Empty(result.SiteCalls);
|
Assert.Empty(result.SiteCalls);
|
||||||
Assert.False(result.MoreAvailable);
|
Assert.False(result.MoreAvailable);
|
||||||
@@ -150,7 +150,7 @@ public class GrpcPullSiteCallsClientTests
|
|||||||
invoker,
|
invoker,
|
||||||
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
||||||
|
|
||||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, CancellationToken.None);
|
||||||
|
|
||||||
Assert.Empty(result.SiteCalls);
|
Assert.Empty(result.SiteCalls);
|
||||||
Assert.False(result.MoreAvailable);
|
Assert.False(result.MoreAvailable);
|
||||||
@@ -165,7 +165,7 @@ public class GrpcPullSiteCallsClientTests
|
|||||||
invoker,
|
invoker,
|
||||||
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
||||||
|
|
||||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, CancellationToken.None);
|
||||||
|
|
||||||
Assert.Empty(result.SiteCalls);
|
Assert.Empty(result.SiteCalls);
|
||||||
Assert.False(result.MoreAvailable);
|
Assert.False(result.MoreAvailable);
|
||||||
@@ -180,7 +180,7 @@ public class GrpcPullSiteCallsClientTests
|
|||||||
invoker,
|
invoker,
|
||||||
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
||||||
|
|
||||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, CancellationToken.None);
|
||||||
|
|
||||||
Assert.Empty(result.SiteCalls);
|
Assert.Empty(result.SiteCalls);
|
||||||
Assert.False(result.MoreAvailable);
|
Assert.False(result.MoreAvailable);
|
||||||
@@ -211,7 +211,7 @@ public class GrpcPullSiteCallsClientTests
|
|||||||
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
||||||
|
|
||||||
// Must NOT throw — the bad row is dropped, the good rows are returned.
|
// Must NOT throw — the bad row is dropped, the good rows are returned.
|
||||||
var result = await sut.PullAsync("site-a", BaseTime, batchSize: 256, CancellationToken.None);
|
var result = await sut.PullAsync("site-a", BaseTime, afterId: null, batchSize: 256, CancellationToken.None);
|
||||||
|
|
||||||
Assert.Equal(2, result.SiteCalls.Count);
|
Assert.Equal(2, result.SiteCalls.Count);
|
||||||
// Survivors are oldest-first and SourceSite re-stamped from the dialed siteId.
|
// Survivors are oldest-first and SourceSite re-stamped from the dialed siteId.
|
||||||
@@ -239,7 +239,7 @@ public class GrpcPullSiteCallsClientTests
|
|||||||
invoker,
|
invoker,
|
||||||
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
NullLogger<GrpcPullSiteCallsClient>.Instance);
|
||||||
|
|
||||||
var result = await sut.PullAsync("site-a", minUnspecified, batchSize: 256, CancellationToken.None);
|
var result = await sut.PullAsync("site-a", minUnspecified, afterId: null, batchSize: 256, CancellationToken.None);
|
||||||
|
|
||||||
Assert.Equal(1, invoker.CallCount);
|
Assert.Equal(1, invoker.CallCount);
|
||||||
Assert.Equal("http://site-a:8083", invoker.Endpoint);
|
Assert.Equal("http://site-a:8083", invoker.Endpoint);
|
||||||
|
|||||||
@@ -42,7 +42,7 @@ public class SiteCallAuditPurgeTests : TestKit
|
|||||||
private sealed class NoOpPullClient : IPullSiteCallsClient
|
private sealed class NoOpPullClient : IPullSiteCallsClient
|
||||||
{
|
{
|
||||||
public Task<PullSiteCallsResponse> PullAsync(
|
public Task<PullSiteCallsResponse> PullAsync(
|
||||||
string siteId, DateTime sinceUtc, int batchSize, CancellationToken ct) =>
|
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct) =>
|
||||||
Task.FromResult(new PullSiteCallsResponse(Array.Empty<SiteCall>(), MoreAvailable: false));
|
Task.FromResult(new PullSiteCallsResponse(Array.Empty<SiteCall>(), MoreAvailable: false));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+150
-4
@@ -74,7 +74,7 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
private sealed class ScriptedPullClient : IPullSiteCallsClient
|
private sealed class ScriptedPullClient : IPullSiteCallsClient
|
||||||
{
|
{
|
||||||
public List<(string SiteId, DateTime SinceUtc, int BatchSize)> Calls { get; } = new();
|
public List<(string SiteId, DateTime SinceUtc, string? AfterId, int BatchSize)> Calls { get; } = new();
|
||||||
private readonly Dictionary<string, Queue<PullSiteCallsResponse>> _scripted = new();
|
private readonly Dictionary<string, Queue<PullSiteCallsResponse>> _scripted = new();
|
||||||
private readonly Dictionary<string, Exception> _throwOnSite = new();
|
private readonly Dictionary<string, Exception> _throwOnSite = new();
|
||||||
|
|
||||||
@@ -91,9 +91,9 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
}
|
}
|
||||||
|
|
||||||
public Task<PullSiteCallsResponse> PullAsync(
|
public Task<PullSiteCallsResponse> PullAsync(
|
||||||
string siteId, DateTime sinceUtc, int batchSize, CancellationToken ct)
|
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
|
||||||
{
|
{
|
||||||
Calls.Add((siteId, sinceUtc, batchSize));
|
Calls.Add((siteId, sinceUtc, afterId, batchSize));
|
||||||
if (_throwOnSite.TryGetValue(siteId, out var ex))
|
if (_throwOnSite.TryGetValue(siteId, out var ex))
|
||||||
{
|
{
|
||||||
throw ex;
|
throw ex;
|
||||||
@@ -107,6 +107,67 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Keyset-honouring in-memory pull client (Task 16). Holds one static set of
|
||||||
|
/// rows and answers each pull as a real site would: returns the next
|
||||||
|
/// <c>batchSize</c> rows strictly greater than the composite
|
||||||
|
/// <c>(sinceUtc, afterId)</c> cursor under the deterministic
|
||||||
|
/// <c>(UpdatedAtUtc, TrackedOperationId ordinal)</c> ordering, with
|
||||||
|
/// <c>MoreAvailable=true</c> when unseen rows remain. This is the modern site
|
||||||
|
/// that DOES honour <c>after_id</c>, so a burst sharing one exact
|
||||||
|
/// <c>UpdatedAtUtc</c> drains fully via the id tiebreak — no single-timestamp
|
||||||
|
/// pin.
|
||||||
|
/// </summary>
|
||||||
|
private sealed class KeysetHonoringPullClient : IPullSiteCallsClient
|
||||||
|
{
|
||||||
|
private readonly IReadOnlyList<SiteCall> _rows;
|
||||||
|
public int CallCount { get; private set; }
|
||||||
|
|
||||||
|
public KeysetHonoringPullClient(IEnumerable<SiteCall> rows) =>
|
||||||
|
_rows = rows
|
||||||
|
.OrderBy(r => r.UpdatedAtUtc)
|
||||||
|
.ThenBy(r => r.TrackedOperationId.ToString(), StringComparer.Ordinal)
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
public Task<PullSiteCallsResponse> PullAsync(
|
||||||
|
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
|
||||||
|
{
|
||||||
|
CallCount++;
|
||||||
|
var greater = _rows
|
||||||
|
.Where(r =>
|
||||||
|
r.UpdatedAtUtc > sinceUtc ||
|
||||||
|
(r.UpdatedAtUtc == sinceUtc &&
|
||||||
|
string.CompareOrdinal(r.TrackedOperationId.ToString(), afterId) > 0))
|
||||||
|
.ToList();
|
||||||
|
var page = greater.Take(batchSize).ToList();
|
||||||
|
var more = greater.Count > page.Count;
|
||||||
|
return Task.FromResult(
|
||||||
|
new PullSiteCallsResponse(page, MoreAvailable: more));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Legacy pull client (Task 16) that IGNORES <c>afterId</c> and always
|
||||||
|
/// returns the same saturated page (<c>MoreAvailable=true</c>) — models a
|
||||||
|
/// pre-keyset site the central actor cannot drain past. The composite cursor
|
||||||
|
/// cannot advance, so the actor must latch and publish
|
||||||
|
/// <see cref="SiteCallReconciliationPinnedChanged"/> instead of spinning.
|
||||||
|
/// </summary>
|
||||||
|
private sealed class LegacyIgnoresAfterIdPullClient : IPullSiteCallsClient
|
||||||
|
{
|
||||||
|
private readonly IReadOnlyList<SiteCall> _rows;
|
||||||
|
public int CallCount { get; private set; }
|
||||||
|
|
||||||
|
public LegacyIgnoresAfterIdPullClient(IReadOnlyList<SiteCall> rows) => _rows = rows;
|
||||||
|
|
||||||
|
public Task<PullSiteCallsResponse> PullAsync(
|
||||||
|
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
|
||||||
|
{
|
||||||
|
CallCount++;
|
||||||
|
return Task.FromResult(new PullSiteCallsResponse(_rows, MoreAvailable: true));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Pull client that ALWAYS returns the same saturated response
|
/// Pull client that ALWAYS returns the same saturated response
|
||||||
/// (<c>MoreAvailable=true</c>) regardless of the <c>since</c> cursor —
|
/// (<c>MoreAvailable=true</c>) regardless of the <c>since</c> cursor —
|
||||||
@@ -124,7 +185,7 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
public SaturatedPinPullClient(IReadOnlyList<SiteCall> rows) => _rows = rows;
|
public SaturatedPinPullClient(IReadOnlyList<SiteCall> rows) => _rows = rows;
|
||||||
|
|
||||||
public Task<PullSiteCallsResponse> PullAsync(
|
public Task<PullSiteCallsResponse> PullAsync(
|
||||||
string siteId, DateTime sinceUtc, int batchSize, CancellationToken ct)
|
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
|
||||||
{
|
{
|
||||||
CallCount++;
|
CallCount++;
|
||||||
return Task.FromResult(new PullSiteCallsResponse(_rows, MoreAvailable: true));
|
return Task.FromResult(new PullSiteCallsResponse(_rows, MoreAvailable: true));
|
||||||
@@ -436,4 +497,89 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
$"a single-timestamp saturation pin must break the within-tick drain, not spin to the "
|
$"a single-timestamp saturation pin must break the within-tick drain, not spin to the "
|
||||||
+ $"page ceiling; got {client.CallCount} pulls (an unbounded within-tick loop would be 50+)");
|
+ $"page ceiling; got {client.CallCount} pulls (an unbounded within-tick loop would be 50+)");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------
|
||||||
|
// 7. Task 16: a keyset-honouring site drains a single-timestamp burst
|
||||||
|
// LARGER than the batch size in ONE tick — the composite (UpdatedAtUtc,
|
||||||
|
// TrackedOperationId) cursor advances by id even when every row shares
|
||||||
|
// one exact timestamp, so the tail no longer pins forever.
|
||||||
|
// ---------------------------------------------------------------------
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void ReconciliationTick_KeysetCursor_DrainsSingleTimestampBurstBeyondBatchSize()
|
||||||
|
{
|
||||||
|
var siteId = "siteA";
|
||||||
|
// 2 × batchSize rows all sharing ONE exact UpdatedAtUtc: the pre-keyset
|
||||||
|
// inclusive-timestamp cursor could never advance past this window.
|
||||||
|
var ts = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);
|
||||||
|
var rows = Enumerable.Range(0, 4)
|
||||||
|
.Select(_ => NewRow(TrackedOperationId.New(), siteId, updatedAtUtc: ts))
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
var sites = new StaticEnumerator(new SiteEntry(siteId, "http://siteA:8083"));
|
||||||
|
var client = new KeysetHonoringPullClient(rows);
|
||||||
|
var repo = new RecordingRepo();
|
||||||
|
|
||||||
|
// batchSize = 2 → the 4-row burst needs the id tiebreak to page through.
|
||||||
|
// Slow tick so the full drain must be the within-tick continuation loop,
|
||||||
|
// not multiple ticks.
|
||||||
|
var options = new SiteCallAuditOptions
|
||||||
|
{
|
||||||
|
ReconciliationIntervalOverride = TimeSpan.FromSeconds(2),
|
||||||
|
ReconciliationBatchSize = 2,
|
||||||
|
};
|
||||||
|
|
||||||
|
CreateActor(sites, client, repo, options);
|
||||||
|
|
||||||
|
AwaitAssert(
|
||||||
|
() =>
|
||||||
|
{
|
||||||
|
foreach (var r in rows)
|
||||||
|
{
|
||||||
|
Assert.True(repo.Upserted.ContainsKey(r.TrackedOperationId),
|
||||||
|
"every row in the single-timestamp burst must reconcile via the "
|
||||||
|
+ "composite keyset cursor within one tick");
|
||||||
|
}
|
||||||
|
},
|
||||||
|
duration: TimeSpan.FromSeconds(3),
|
||||||
|
interval: TimeSpan.FromMilliseconds(50));
|
||||||
|
|
||||||
|
// Two pages (2 + 2) drained the burst; well under the 50-page ceiling.
|
||||||
|
Assert.True(client.CallCount is >= 2 and < 10,
|
||||||
|
$"expected the burst to drain in a couple of keyset pages, got {client.CallCount}");
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------
|
||||||
|
// 8. Task 16: a LEGACY site that ignores after_id can never be drained, so
|
||||||
|
// the actor latches and publishes SiteCallReconciliationPinnedChanged
|
||||||
|
// instead of silently logging (the dead-end is now health-observable).
|
||||||
|
// ---------------------------------------------------------------------
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public void ReconciliationTick_LegacySiteIgnoresAfterId_PublishesPinnedChanged()
|
||||||
|
{
|
||||||
|
var siteId = "siteLegacy";
|
||||||
|
var ts = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);
|
||||||
|
var r1 = NewRow(TrackedOperationId.New(), siteId, updatedAtUtc: ts);
|
||||||
|
var r2 = NewRow(TrackedOperationId.New(), siteId, updatedAtUtc: ts);
|
||||||
|
|
||||||
|
var sites = new StaticEnumerator(new SiteEntry(siteId, "http://siteLegacy:8083"));
|
||||||
|
var client = new LegacyIgnoresAfterIdPullClient(new[] { r1, r2 });
|
||||||
|
var repo = new RecordingRepo();
|
||||||
|
|
||||||
|
// Subscribe BEFORE the actor starts ticking so the transition isn't missed.
|
||||||
|
Sys.EventStream.Subscribe(TestActor, typeof(SiteCallReconciliationPinnedChanged));
|
||||||
|
|
||||||
|
var options = new SiteCallAuditOptions
|
||||||
|
{
|
||||||
|
ReconciliationIntervalOverride = TimeSpan.FromMilliseconds(100),
|
||||||
|
ReconciliationBatchSize = 2,
|
||||||
|
};
|
||||||
|
|
||||||
|
CreateActor(sites, client, repo, options);
|
||||||
|
|
||||||
|
var evt = ExpectMsg<SiteCallReconciliationPinnedChanged>(TimeSpan.FromSeconds(5));
|
||||||
|
Assert.Equal(siteId, evt.SiteId);
|
||||||
|
Assert.True(evt.Pinned, "a legacy site that ignores after_id must publish Pinned=true");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user