fix(site-call-audit): reconciliation pull serves only un-acked tracking rows (node-local ack ledger), batched page upsert, prune removed sites' cursors (review 2026-10-21 06/PERF-2 + lane H follow-up)
This commit is contained in:
@@ -135,8 +135,41 @@ Pinned)` on the EventStream (transition-only, mirroring
|
||||
health-observable condition rather than a silent log line, and the latch clears
|
||||
with `Pinned=false` once a later tick makes progress.
|
||||
|
||||
**The pull serves only what central has not acknowledged.** The site's
|
||||
`ReadChangedSinceAsync` skips every tracking row whose *current* version
|
||||
(`UpdatedAtUtc`) central already accepted through the cached-telemetry push, so in
|
||||
steady state a reconciliation pull carries only genuinely lost telemetry rather than
|
||||
a second copy of every lifecycle step — and a central failover, which resets the
|
||||
in-memory cursors to `DateTime.MinValue`, no longer replays the whole tracking
|
||||
retention window of every site. When central acks an `IngestCachedTelemetry`
|
||||
packet (it commits the packet's `SiteCalls` upsert with its audit row), the site's
|
||||
`SiteAuditTelemetryActor` records `(TrackedOperationId, snapshot UpdatedAtUtc)` in
|
||||
the **node-local** `OperationTrackingCentralAck` table — deliberately a separate,
|
||||
unregistered table, never a column on the LocalDb-replicated `OperationTracking`
|
||||
table (same posture as `mirrored_secrets`): only the node that pushed saw central
|
||||
accept it. The filter matches the version exactly, so any later change (even one
|
||||
stamped by a clock that stepped back) is served again; acks are monotonic per row
|
||||
and pruned with the tracking rows they describe. **Failover:** a site node that
|
||||
has just become active holds no ack rows, so its pull serves every row once
|
||||
(bounded by central's cursor); central's idempotent monotonic upsert absorbs the
|
||||
duplicates and the ledger rebuilds from that node's own pushes. A lost ack only
|
||||
costs one extra pull of that row.
|
||||
|
||||
**Each pulled page is one batched upsert.** `ISiteCallAuditRepository.UpsertManyAsync`
|
||||
runs the same two per-row statements as `UpsertAsync` (insert-if-absent, then the
|
||||
ownership-checked monotonic update) for up to 128 rows per command — a 500-row page
|
||||
is 4 round trips instead of 500. A chunk that faults (an insert race, or an
|
||||
ownership violation) is re-applied row by row through `UpsertAsync`, which is inert
|
||||
for rows already applied; the rest of the page is still applied and the first
|
||||
row failure is rethrown, so the site's cursor stays put exactly as before.
|
||||
|
||||
**Removed sites are pruned.** Each tick drops the cursor and pinned latch of any
|
||||
site the enumerator no longer returns (a pinned one publishes its `Pinned=false`
|
||||
transition first), so the dictionaries do not grow with site churn; a site that
|
||||
reappears restarts from `DateTime.MinValue`.
|
||||
|
||||
**The drain runs off the mailbox.** A reconciliation pass is unbounded work —
|
||||
every site, up to a page ceiling of network pulls each, one upsert per row — so
|
||||
every site, up to a page ceiling of network pulls each, one batched upsert per page — so
|
||||
it runs as a background task with a `PipeTo`-delivered completion message and a
|
||||
single-flight guard, not as an actor message handler. A handler occupies the
|
||||
actor for its whole duration, so a post-outage catch-up used to park telemetry
|
||||
|
||||
@@ -271,6 +271,9 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
var batch = new CachedTelemetryBatch();
|
||||
var wireBytes = AuditWireBudget.InitialBytes;
|
||||
var emittedEventIds = new List<Guid>(pending.Count);
|
||||
// EventId → the tracking-row version its packet carried, so an accepted
|
||||
// packet can be recorded as a central ack (review 2026-10-21 06/PERF-2).
|
||||
var packetVersions = new Dictionary<Guid, (TrackedOperationId Id, DateTime UpdatedAtUtc)>(pending.Count);
|
||||
// Rows whose operational half can never be built. They are marked
|
||||
// PushAbandoned so they LEAVE this queue — see AbandonUnresolvableAsync
|
||||
// for why that is safe and why leaving them Pending is not.
|
||||
@@ -338,6 +341,7 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
var packet = BuildCachedPacket(auditRow, snapshot);
|
||||
if (!AuditWireBudget.TryAppend(batch.Packets, packet, ref wireBytes, auditRow.EventId.ToString())) break;
|
||||
emittedEventIds.Add(auditRow.EventId);
|
||||
packetVersions[auditRow.EventId] = (snapshot.Id, snapshot.UpdatedAtUtc);
|
||||
}
|
||||
|
||||
if (lookupFailures > 0)
|
||||
@@ -389,6 +393,7 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
{
|
||||
await _queue.MarkForwardedAsync(acceptedIds, ct)
|
||||
.ConfigureAwait(false);
|
||||
await RecordCentralAcksAsync(acceptedIds, packetVersions, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await RecordRefusalsAsync(
|
||||
@@ -413,6 +418,51 @@ public class SiteAuditTelemetryActor : ReceiveActor
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tells the tracking store which tracking-row versions central has just accepted, so
|
||||
/// the SiteCalls reconciliation pull stops re-serving them (review 2026-10-21
|
||||
/// 06/PERF-2). Accepted means central committed the packet's <c>SiteCalls</c> upsert
|
||||
/// in the same transaction as its audit row, i.e. central now holds exactly what a pull
|
||||
/// of that version would send. Several packets of one operation in a batch collapse to
|
||||
/// the newest version. Best-effort: the store only enqueues, and a failure here just
|
||||
/// leaves the rows eligible for one more pull.
|
||||
/// </summary>
|
||||
private async Task RecordCentralAcksAsync(
|
||||
IReadOnlyList<Guid> acceptedIds,
|
||||
Dictionary<Guid, (TrackedOperationId Id, DateTime UpdatedAtUtc)> packetVersions,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var versions = new Dictionary<TrackedOperationId, DateTime>();
|
||||
foreach (var eventId in acceptedIds)
|
||||
{
|
||||
if (!packetVersions.TryGetValue(eventId, out var v))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!versions.TryGetValue(v.Id, out var existing) || v.UpdatedAtUtc > existing)
|
||||
{
|
||||
versions[v.Id] = v.UpdatedAtUtc;
|
||||
}
|
||||
}
|
||||
|
||||
if (versions.Count == 0 || _trackingStore is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await _trackingStore.RecordCentralAcksAsync(versions, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
_logger.LogDebug(ex,
|
||||
"Recording {Count} central ack(s) in the tracking store failed; those rows stay eligible for the reconciliation pull.",
|
||||
versions.Count);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Moves cached rows whose operational half can never be built out of the
|
||||
/// cached-drain queue, as <see cref="SiteAuditForwardStates.PushAbandoned"/>.
|
||||
|
||||
@@ -42,6 +42,27 @@ public interface ISiteCallAuditRepository
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Applies <see cref="UpsertAsync"/>'s exact per-row semantics (insert-if-absent, then
|
||||
/// the monotonic rank-guarded update, ownership-checked) to a whole page of rows in as
|
||||
/// few round trips as the implementation can manage — the reconciliation pull's write
|
||||
/// path (review 2026-10-21 06/PERF-2). Rows are applied in order; a row that faults
|
||||
/// (e.g. an ownership violation) surfaces as the same exception <see cref="UpsertAsync"/>
|
||||
/// would throw — the SQL Server implementation applies the rest of the page first. The
|
||||
/// default implementation simply upserts row by row and stops at the first fault.
|
||||
/// </summary>
|
||||
/// <param name="siteCalls">The rows to insert or monotonically update.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
async Task UpsertManyAsync(IReadOnlyList<SiteCall> siteCalls, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(siteCalls);
|
||||
foreach (var siteCall in siteCalls)
|
||||
{
|
||||
await UpsertAsync(siteCall, ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the row for the given id, or <c>null</c> if none exists.
|
||||
/// </summary>
|
||||
|
||||
@@ -150,6 +150,15 @@ public interface IOperationTrackingStore
|
||||
/// deterministic — <c>UpdatedAtUtc ASC, TrackedOperationId ASC</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Only rows central has not already acknowledged.</b> A row whose
|
||||
/// current version (<c>UpdatedAtUtc</c>) central acknowledged through the
|
||||
/// cached-telemetry push — recorded via <see cref="RecordCentralAcksAsync"/> — is
|
||||
/// skipped: central already holds exactly what this read would send. The pull
|
||||
/// therefore carries only lost telemetry, not a replay of every lifecycle step
|
||||
/// (review 2026-10-21 06/PERF-2). The ack ledger is node-local, so a node that
|
||||
/// has just become active serves every row once.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <see cref="SiteCallOperational.SourceSite"/> is left as the empty string:
|
||||
/// the site id is not a tracking-store column, and the central client re-stamps
|
||||
/// it from the <c>siteId</c> it dialed (the only authority that knows which
|
||||
@@ -174,4 +183,24 @@ public interface IOperationTrackingStore
|
||||
int batchSize,
|
||||
string? afterId = null,
|
||||
CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Records that central accepted the cached-telemetry push of each operation at the
|
||||
/// given tracking-row version (the snapshot's <c>UpdatedAtUtc</c> the push carried),
|
||||
/// so <see cref="ReadChangedSinceAsync"/> stops serving that version to the
|
||||
/// reconciliation pull. Monotonic per operation (an older version never replaces a
|
||||
/// newer one) and best-effort: a lost ack only means the row is pulled once more,
|
||||
/// which central's idempotent upsert absorbs.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The ledger lives in a NODE-LOCAL side table, never a column on the replicated
|
||||
/// tracking table. The default implementation records nothing (every row stays
|
||||
/// eligible for the pull — the pre-ack behaviour).
|
||||
/// </remarks>
|
||||
/// <param name="ackedVersions">Operation id → the <c>UpdatedAtUtc</c> of the snapshot central accepted.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that completes once the acks are queued.</returns>
|
||||
Task RecordCentralAcksAsync(
|
||||
IReadOnlyDictionary<TrackedOperationId, DateTime> ackedVersions,
|
||||
CancellationToken ct = default) => Task.CompletedTask;
|
||||
}
|
||||
|
||||
+78
@@ -169,6 +169,84 @@ public class SiteCallAuditRepository : ISiteCallAuditRepository
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Rows per batched upsert command. Each row binds 15 parameters, so 128 rows stay
|
||||
/// under SQL Server's 2 100-parameter limit (1 920) with headroom.
|
||||
/// </summary>
|
||||
internal const int UpsertManyChunkSize = 128;
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task UpsertManyAsync(IReadOnlyList<SiteCall> siteCalls, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(siteCalls);
|
||||
|
||||
// The reconciliation pull's write path (review 2026-10-21 06/PERF-2): one command
|
||||
// per chunk of UpsertManyChunkSize rows instead of one round trip per row — a
|
||||
// 500-row page is 4 round trips, not 500. Each row runs the SAME two statements
|
||||
// UpsertAsync runs (InsertIfAbsentSql then MonotonicUpdateSql), with its parameters
|
||||
// suffixed per row, so the insert-first / monotonic / ownership semantics are
|
||||
// byte-identical rather than re-derived set-based.
|
||||
//
|
||||
// Any SqlException from a chunk — an insert race's 2601/2627 (statement-level,
|
||||
// the batch runs on) or an ownership THROW 51001 (batch-terminating) — re-runs that
|
||||
// chunk row by row through UpsertAsync. Re-applying rows the batch already applied
|
||||
// is inert (equal rank + equal UpdatedAtUtc changes nothing) and UpsertAsync handles
|
||||
// the race exactly as before. A genuinely bad row does not stop the rest of the
|
||||
// page: the remaining rows are still applied and the FIRST such failure is
|
||||
// rethrown at the end, so the caller sees the same exception the old per-row
|
||||
// loop surfaced (and keeps its cursor), minus the collateral of the rows behind it.
|
||||
System.Runtime.ExceptionServices.ExceptionDispatchInfo? firstFailure = null;
|
||||
for (var offset = 0; offset < siteCalls.Count; offset += UpsertManyChunkSize)
|
||||
{
|
||||
var chunk = siteCalls.Skip(offset).Take(UpsertManyChunkSize).ToList();
|
||||
var sql = new System.Text.StringBuilder();
|
||||
var parameters = new List<object>(chunk.Count * 15);
|
||||
for (var i = 0; i < chunk.Count; i++)
|
||||
{
|
||||
var siteCall = chunk[i] ?? throw new ArgumentException("A null SiteCall row was supplied.", nameof(siteCalls));
|
||||
var rank = GetRankOrThrow(siteCall.Status);
|
||||
var suffix = "_" + i.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
||||
sql.AppendLine(SuffixParameters(InsertIfAbsentSql, suffix));
|
||||
sql.AppendLine(SuffixParameters(MonotonicUpdateSql, suffix));
|
||||
foreach (SqlParameter p in BuildUpsertParameters(siteCall, siteCall.TrackedOperationId.Value.ToString("D"), rank))
|
||||
{
|
||||
p.ParameterName += suffix;
|
||||
parameters.Add(p);
|
||||
}
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await _context.Database.ExecuteSqlRawAsync(sql.ToString(), parameters, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (SqlException ex)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
ex,
|
||||
"SiteCallAuditRepository.UpsertManyAsync chunk of {Count} rows faulted (error {SqlErrorNumber}); re-applying it row by row.",
|
||||
chunk.Count,
|
||||
ex.Number);
|
||||
foreach (var siteCall in chunk)
|
||||
{
|
||||
try
|
||||
{
|
||||
await UpsertAsync(siteCall, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception rowFailure) when (rowFailure is not OperationCanceledException)
|
||||
{
|
||||
firstFailure ??= System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(rowFailure);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
firstFailure?.Throw();
|
||||
}
|
||||
|
||||
/// <summary>Renames every <c>@Name</c> parameter reference in <paramref name="sql"/> to <c>@Name{suffix}</c>.</summary>
|
||||
private static string SuffixParameters(string sql, string suffix) =>
|
||||
System.Text.RegularExpressions.Regex.Replace(sql, @"@(\w+)", "@$1" + suffix);
|
||||
|
||||
// Leg 1 of the upsert: create the row when it does not exist yet. Runs FIRST
|
||||
// so a concurrent first-write loser still has a row to update (see UpsertAsync).
|
||||
private const string InsertIfAbsentSql = @"
|
||||
|
||||
@@ -499,6 +499,8 @@ public class SiteCallAuditActor : ReceiveActor
|
||||
return;
|
||||
}
|
||||
|
||||
PruneRemovedSites(sites);
|
||||
|
||||
if (sites.Count == 0)
|
||||
{
|
||||
return;
|
||||
@@ -516,6 +518,30 @@ public class SiteCallAuditActor : ReceiveActor
|
||||
await ReconcileSitesAsync(sites, client, repository).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Drops the per-site cursor and pinned latch of every site the enumerator no longer
|
||||
/// returns (a deleted or renamed site), so the dictionaries do not grow with site
|
||||
/// churn for the singleton's lifetime. A site that was pinned publishes its
|
||||
/// <c>Pinned=false</c> transition first, so a health subscriber does not keep showing
|
||||
/// a pinned site that no longer exists. Runs on the single-flight pass that owns the
|
||||
/// dictionaries. A site that later reappears simply starts again from
|
||||
/// <c>DateTime.MinValue</c>, which the idempotent upsert absorbs.
|
||||
/// </summary>
|
||||
private void PruneRemovedSites(IReadOnlyList<SiteEntry> sites)
|
||||
{
|
||||
var live = new HashSet<string>(sites.Select(s => s.SiteId), StringComparer.Ordinal);
|
||||
foreach (var siteId in _reconciliationPinned.Keys.Where(k => !live.Contains(k)).ToList())
|
||||
{
|
||||
SetReconciliationPinned(siteId, pinned: false);
|
||||
_reconciliationPinned.Remove(siteId);
|
||||
}
|
||||
|
||||
foreach (var siteId in _reconciliationCursors.Keys.Where(k => !live.Contains(k)).ToList())
|
||||
{
|
||||
_reconciliationCursors.Remove(siteId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reconciles every site in the tick against a single resolved repository,
|
||||
/// isolating per-site faults so one bad site never sinks the rest of the
|
||||
@@ -635,15 +661,19 @@ public class SiteCallAuditActor : ReceiveActor
|
||||
var maxUpdated = since;
|
||||
var maxAfterId = afterId;
|
||||
var nowUtc = DateTime.UtcNow;
|
||||
|
||||
// IngestedAtUtc is the "central ingested (or last refreshed) this row" stamp —
|
||||
// owned by the central actor, exactly as OnUpsertAsync does for the telemetry
|
||||
// path. The whole page is written through ONE batched monotonic upsert (a few
|
||||
// round trips per page instead of one per row — review 2026-10-21 06/PERF-2);
|
||||
// a row already present from a prior push is a silent no-op. A fault propagates
|
||||
// to the per-site catch BEFORE the cursor below moves, so the page is re-pulled.
|
||||
await repository
|
||||
.UpsertManyAsync(response.SiteCalls.Select(row => row with { IngestedAtUtc = nowUtc }).ToList())
|
||||
.ConfigureAwait(false);
|
||||
|
||||
foreach (var row in response.SiteCalls)
|
||||
{
|
||||
// IngestedAtUtc is the "central ingested (or last refreshed) this
|
||||
// row" stamp — owned by the central actor, exactly as OnUpsertAsync
|
||||
// does for the telemetry path. Monotonic UpsertAsync makes a row
|
||||
// already present (from a prior push) a silent no-op.
|
||||
var siteCall = row with { IngestedAtUtc = nowUtc };
|
||||
await repository.UpsertAsync(siteCall).ConfigureAwait(false);
|
||||
|
||||
// Advance the composite max: greater timestamp wins; on a tie the
|
||||
// greater TrackedOperationId (ordinal, matching the site's SQLite
|
||||
// BINARY text collation) wins.
|
||||
|
||||
@@ -141,7 +141,34 @@ public class OperationTrackingStore : IOperationTrackingStore, IAsyncDisposable,
|
||||
// which is strictly before this constructor can receive it. It stays because it
|
||||
// keeps a directly-constructed store (tests, tooling) self-sufficient, and the
|
||||
// DDL is idempotent.
|
||||
private void InitializeSchema() => OperationTrackingSchema.Apply(_writeConnection);
|
||||
private void InitializeSchema()
|
||||
{
|
||||
OperationTrackingSchema.Apply(_writeConnection);
|
||||
using var cmd = _writeConnection.CreateCommand();
|
||||
cmd.CommandText = CentralAckSchemaSql;
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Node-local ledger of the tracking-row version central last acknowledged per
|
||||
/// operation (review 2026-10-21 06/PERF-2). Deliberately a SEPARATE table, never a
|
||||
/// column on <c>OperationTracking</c>: that table is LocalDb-replicated and its
|
||||
/// capture triggers are built from its column list, while acks are per-node fact —
|
||||
/// only the node that pushed the telemetry saw central accept it. Like
|
||||
/// <c>mirrored_secrets</c>, it is not in <c>SiteLocalDbSetup.ReplicatedTables</c>.
|
||||
/// <para>
|
||||
/// Failover: the standby holds no ack rows, so after it takes over its
|
||||
/// reconciliation read serves every row once (bounded by central's cursor), and
|
||||
/// central's idempotent monotonic upsert absorbs the duplicates. Pushes from the new
|
||||
/// active node rebuild the ledger from then on.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private const string CentralAckSchemaSql = """
|
||||
CREATE TABLE IF NOT EXISTS OperationTrackingCentralAck (
|
||||
TrackedOperationId TEXT NOT NULL PRIMARY KEY,
|
||||
AckedUpdatedAtUtc TEXT NOT NULL
|
||||
);
|
||||
""";
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task RecordEnqueueAsync(
|
||||
@@ -271,6 +298,51 @@ public class OperationTrackingStore : IOperationTrackingStore, IAsyncDisposable,
|
||||
}, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc/>
|
||||
public async Task RecordCentralAcksAsync(
|
||||
IReadOnlyDictionary<TrackedOperationId, DateTime> ackedVersions,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(ackedVersions);
|
||||
if (ackedVersions.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// Rendered exactly as the write path stamps UpdatedAtUtc ("o", UTC 'Z'), so the
|
||||
// read filter's text equality compares like with like.
|
||||
var acks = ackedVersions
|
||||
.Select(a => (Id: a.Key.ToString(), Version: DateTime
|
||||
.SpecifyKind(a.Value, DateTimeKind.Utc)
|
||||
.ToString("o", CultureInfo.InvariantCulture)))
|
||||
.ToList();
|
||||
|
||||
// Rides the single-writer queue like a lifecycle write (enqueue and return) —
|
||||
// the ack is best-effort, and FIFO keeps it behind the writes it describes.
|
||||
await EnqueueAsync($"central-ack x{acks.Count}", (conn, tx) =>
|
||||
{
|
||||
using var cmd = conn.CreateCommand();
|
||||
cmd.Transaction = tx;
|
||||
// Monotonic: an older ack never rolls a newer one back. ISO-8601 "o" text
|
||||
// orders lexicographically.
|
||||
cmd.CommandText = """
|
||||
INSERT INTO OperationTrackingCentralAck (TrackedOperationId, AckedUpdatedAtUtc)
|
||||
VALUES ($id, $version)
|
||||
ON CONFLICT (TrackedOperationId) DO UPDATE
|
||||
SET AckedUpdatedAtUtc = excluded.AckedUpdatedAtUtc
|
||||
WHERE excluded.AckedUpdatedAtUtc > AckedUpdatedAtUtc;
|
||||
""";
|
||||
var idParam = cmd.Parameters.Add("$id", SqliteType.Text);
|
||||
var versionParam = cmd.Parameters.Add("$version", SqliteType.Text);
|
||||
foreach (var (id, version) in acks)
|
||||
{
|
||||
idParam.Value = id;
|
||||
versionParam.Value = version;
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
}, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// ── Queue plumbing ─────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
@@ -494,10 +566,17 @@ public class OperationTrackingStore : IOperationTrackingStore, IAsyncDisposable,
|
||||
cmd.Transaction = tx;
|
||||
// Non-terminal rows (TerminalAtUtc IS NULL) are kept regardless of
|
||||
// age — the operation is still in flight.
|
||||
// The node-local ack ledger is pruned in the same transaction so it never
|
||||
// outlives the rows it describes (it also drops acks for rows the peer's
|
||||
// purge removed through replication).
|
||||
cmd.CommandText = """
|
||||
DELETE FROM OperationTracking
|
||||
WHERE TerminalAtUtc IS NOT NULL
|
||||
AND TerminalAtUtc < $threshold;
|
||||
DELETE FROM OperationTrackingCentralAck
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM OperationTracking t
|
||||
WHERE t.TrackedOperationId = OperationTrackingCentralAck.TrackedOperationId);
|
||||
""";
|
||||
cmd.Parameters.AddWithValue("$threshold", threshold);
|
||||
cmd.ExecuteNonQuery();
|
||||
@@ -543,12 +622,23 @@ public class OperationTrackingStore : IOperationTrackingStore, IAsyncDisposable,
|
||||
var predicate = useKeyset
|
||||
? "UpdatedAtUtc > $since OR (UpdatedAtUtc = $since AND TrackedOperationId > $afterId)"
|
||||
: "UpdatedAtUtc >= $since";
|
||||
// Rows whose CURRENT version central already acknowledged through the
|
||||
// cached-telemetry push are skipped (review 2026-10-21 06/PERF-2): central holds
|
||||
// exactly what this read would send, so serving them only replays every
|
||||
// lifecycle step a second time — and, after a central failover reset the
|
||||
// cursor, the whole retention window. Equality (not >=) on the version is the
|
||||
// safe direction: any later change, even one stamped by a clock that stepped
|
||||
// backwards, no longer matches and is served.
|
||||
cmd.CommandText = $"""
|
||||
SELECT TrackedOperationId, Kind, TargetSummary, Status,
|
||||
RetryCount, LastError, HttpStatus,
|
||||
CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, SourceNode
|
||||
FROM OperationTracking
|
||||
WHERE {predicate}
|
||||
WHERE ({predicate})
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM OperationTrackingCentralAck a
|
||||
WHERE a.TrackedOperationId = OperationTracking.TrackedOperationId
|
||||
AND a.AckedUpdatedAtUtc = OperationTracking.UpdatedAtUtc)
|
||||
ORDER BY UpdatedAtUtc ASC, TrackedOperationId ASC
|
||||
LIMIT $batchSize;
|
||||
""";
|
||||
|
||||
+48
@@ -416,6 +416,54 @@ public class SiteAuditTelemetryActorTests : TestKit
|
||||
Assert.Equal(expectedIds, emittedEventIds);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the tracking-row versions central ACCEPTED are recorded as central acks
|
||||
/// in the tracking store — and only those — so the SiteCalls reconciliation pull stops
|
||||
/// re-serving them (review 2026-10-21 06/PERF-2).
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CachedDrain_AcceptedPackets_AreRecordedAsCentralAcks()
|
||||
{
|
||||
var acceptedRow = NewCachedEvent(AuditKind.CachedSubmit);
|
||||
var refusedRow = NewCachedEvent(AuditKind.CachedSubmit);
|
||||
_queue.ReadPendingAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
|
||||
_queue.ReadPendingCachedTelemetryAsync(Arg.Any<int>(), Arg.Any<CancellationToken>())
|
||||
.Returns(
|
||||
Task.FromResult<IReadOnlyList<AuditEvent>>(new[] { acceptedRow, refusedRow }),
|
||||
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>()));
|
||||
|
||||
var acceptedTid = new TrackedOperationId(acceptedRow.CorrelationId!.Value);
|
||||
var refusedTid = new TrackedOperationId(refusedRow.CorrelationId!.Value);
|
||||
var acceptedSnapshot = NewSnapshot(acceptedTid) with
|
||||
{
|
||||
UpdatedAtUtc = new DateTime(2026, 5, 20, 10, 0, 7, DateTimeKind.Utc),
|
||||
};
|
||||
_trackingStore.GetStatusAsync(acceptedTid, Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult<TrackingStatusSnapshot?>(acceptedSnapshot));
|
||||
_trackingStore.GetStatusAsync(refusedTid, Arg.Any<CancellationToken>())
|
||||
.Returns(Task.FromResult<TrackingStatusSnapshot?>(NewSnapshot(refusedTid)));
|
||||
|
||||
_client.IngestCachedTelemetryAsync(Arg.Any<CachedTelemetryBatch>(), Arg.Any<CancellationToken>())
|
||||
.Returns(_ =>
|
||||
{
|
||||
var ack = new IngestAck();
|
||||
ack.AcceptedEventIds.Add(acceptedRow.EventId.ToString());
|
||||
return Task.FromResult(ack);
|
||||
});
|
||||
|
||||
CreateActorWithCachedDrain();
|
||||
|
||||
await AwaitAssertAsync(async () =>
|
||||
{
|
||||
await _trackingStore.Received(1).RecordCentralAcksAsync(
|
||||
Arg.Is<IReadOnlyDictionary<TrackedOperationId, DateTime>>(d =>
|
||||
d.Count == 1 && d[acceptedTid] == acceptedSnapshot.UpdatedAtUtc),
|
||||
Arg.Any<CancellationToken>());
|
||||
}, TimeSpan.FromSeconds(5));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a cached row with no tracking snapshot past the grace period is abandoned while a valid row still flows.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
|
||||
+114
@@ -929,6 +929,120 @@ public class SiteCallAuditRepositoryTests : IClassFixture<MsSqlMigrationFixture>
|
||||
Assert.DoesNotContain(perNode, n => n.SourceNode is null);
|
||||
}
|
||||
|
||||
// --- UpsertManyAsync (review 2026-10-21 06/PERF-2) ----------------------
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a batched upsert applies exactly the per-row monotonic semantics: fresh
|
||||
/// rows insert, a higher rank advances, a lower rank is a no-op, and a page larger than
|
||||
/// one command chunk is fully applied — in a handful of round trips, not one per row.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[SkippableFact]
|
||||
public async Task UpsertManyAsync_AppliesMonotonicSemantics_InFewRoundTrips()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
var site = NewSiteId();
|
||||
var now = DateTime.UtcNow;
|
||||
var advance = TrackedOperationId.New();
|
||||
var regress = TrackedOperationId.New();
|
||||
await using (var seed = CreateContext())
|
||||
{
|
||||
var seedRepo = new SiteCallAuditRepository(seed);
|
||||
await seedRepo.UpsertAsync(NewRow(advance, site, status: "Submitted", createdAtUtc: now));
|
||||
await seedRepo.UpsertAsync(NewRow(regress, site, status: "Delivered", createdAtUtc: now,
|
||||
terminal: true, terminalAtUtc: now));
|
||||
}
|
||||
|
||||
var fresh = Enumerable.Range(0, 300).Select(_ => TrackedOperationId.New()).ToList();
|
||||
var page = fresh.Select(id => NewRow(id, site, status: "Attempted", retryCount: 1, createdAtUtc: now))
|
||||
.Append(NewRow(advance, site, status: "Delivered", createdAtUtc: now, terminal: true, terminalAtUtc: now))
|
||||
.Append(NewRow(regress, site, status: "Attempted", retryCount: 9, createdAtUtc: now))
|
||||
.ToList();
|
||||
|
||||
var counter = new NonQueryCounter();
|
||||
await using (var context = CreateContext(counter))
|
||||
{
|
||||
await new SiteCallAuditRepository(context).UpsertManyAsync(page);
|
||||
}
|
||||
|
||||
Assert.True(counter.Count <= 5, $"UpsertManyAsync issued {counter.Count} commands for {page.Count} rows");
|
||||
await using var read = CreateContext();
|
||||
var stored = await read.Set<SiteCall>().Where(s => s.SourceSite == site).ToListAsync();
|
||||
Assert.Equal(302, stored.Count);
|
||||
Assert.All(stored.Where(s => fresh.Contains(s.TrackedOperationId)), s => Assert.Equal("Attempted", s.Status));
|
||||
Assert.Equal("Delivered", stored.Single(s => s.TrackedOperationId == advance).Status);
|
||||
var kept = stored.Single(s => s.TrackedOperationId == regress);
|
||||
Assert.Equal("Delivered", kept.Status);
|
||||
Assert.Equal(0, kept.RetryCount);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a row another site owns still fails the batch with the ownership error,
|
||||
/// while every legitimate row of the page is applied (the chunk falls back to per-row).
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[SkippableFact]
|
||||
public async Task UpsertManyAsync_OwnershipViolation_StillThrows_AfterApplyingTheRestOfThePage()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
var owner = NewSiteId();
|
||||
var attacker = NewSiteId();
|
||||
var owned = TrackedOperationId.New();
|
||||
await using (var seed = CreateContext())
|
||||
{
|
||||
await new SiteCallAuditRepository(seed).UpsertAsync(NewRow(owned, owner, status: "Submitted"));
|
||||
}
|
||||
|
||||
var good1 = TrackedOperationId.New();
|
||||
var good2 = TrackedOperationId.New();
|
||||
var page = new List<SiteCall>
|
||||
{
|
||||
NewRow(good1, attacker, status: "Submitted"),
|
||||
NewRow(owned, attacker, status: "Delivered", terminal: true),
|
||||
NewRow(good2, attacker, status: "Submitted"),
|
||||
};
|
||||
|
||||
await using (var context = CreateContext())
|
||||
{
|
||||
var failure = await Assert.ThrowsAsync<Microsoft.Data.SqlClient.SqlException>(
|
||||
() => new SiteCallAuditRepository(context).UpsertManyAsync(page));
|
||||
Assert.Equal(51001, failure.Number);
|
||||
}
|
||||
|
||||
await using var read = CreateContext();
|
||||
var repo = new SiteCallAuditRepository(read);
|
||||
Assert.NotNull(await repo.GetAsync(good1));
|
||||
Assert.NotNull(await repo.GetAsync(good2));
|
||||
var stored = await repo.GetAsync(owned);
|
||||
Assert.Equal(owner, stored!.SourceSite);
|
||||
Assert.Equal("Submitted", stored.Status);
|
||||
}
|
||||
|
||||
private ScadaBridgeDbContext CreateContext(DbCommandInterceptor interceptor)
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(_fixture.ConnectionString)
|
||||
.AddInterceptors(interceptor)
|
||||
.Options;
|
||||
return new ScadaBridgeDbContext(options);
|
||||
}
|
||||
|
||||
/// <summary>Counts the non-query commands a repository call issues.</summary>
|
||||
private sealed class NonQueryCounter : DbCommandInterceptor
|
||||
{
|
||||
/// <summary>The number of non-query commands issued.</summary>
|
||||
public int Count { get; private set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public override ValueTask<InterceptionResult<int>> NonQueryExecutingAsync(
|
||||
DbCommand command, CommandEventData eventData, InterceptionResult<int> result,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
Count++;
|
||||
return base.NonQueryExecutingAsync(command, eventData, result, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
// --- helpers ------------------------------------------------------------
|
||||
|
||||
private ScadaBridgeDbContext CreateContext()
|
||||
|
||||
@@ -231,6 +231,9 @@ public class SiteCallAuditReconciliationTests : TestKit
|
||||
/// <summary>Gets the number of upsert calls received.</summary>
|
||||
public int UpsertCallCount { get; private set; }
|
||||
|
||||
/// <summary>Gets the row count of every batched (page) upsert received.</summary>
|
||||
public List<int> UpsertManyPageSizes { get; } = new();
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default)
|
||||
{
|
||||
@@ -239,6 +242,23 @@ public class SiteCallAuditReconciliationTests : TestKit
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task UpsertManyAsync(IReadOnlyList<SiteCall> siteCalls, CancellationToken ct = default)
|
||||
{
|
||||
lock (UpsertManyPageSizes)
|
||||
{
|
||||
UpsertManyPageSizes.Add(siteCalls.Count);
|
||||
}
|
||||
|
||||
foreach (var siteCall in siteCalls)
|
||||
{
|
||||
UpsertCallCount++;
|
||||
Upserted[siteCall.TrackedOperationId] = siteCall;
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default) =>
|
||||
Task.FromResult(Upserted.TryGetValue(id, out var row) ? row : null);
|
||||
@@ -706,6 +726,148 @@ public class SiteCallAuditReconciliationTests : TestKit
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Review 2026-10-21 06/PERF-2: a pulled page is written through ONE batched
|
||||
// upsert, not one repository call per row.
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
/// <summary>Verifies that a reconciliation page is written through a single batched upsert.</summary>
|
||||
[Fact]
|
||||
public void ReconciliationTick_WritesEachPageThroughOneBatchedUpsert()
|
||||
{
|
||||
var siteId = "siteBatch";
|
||||
var rows = Enumerable.Range(0, 3)
|
||||
.Select(i => NewRow(TrackedOperationId.New(), siteId, updatedAtUtc: DateTime.UtcNow.AddSeconds(i)))
|
||||
.ToArray();
|
||||
var sites = new StaticEnumerator(new SiteEntry(siteId, "http://siteBatch:8083"));
|
||||
var client = new ScriptedPullClient().Script(siteId,
|
||||
new PullSiteCallsResponse(rows, MoreAvailable: false));
|
||||
var repo = new RecordingRepo();
|
||||
|
||||
var actor = CreateActor(sites, client, repo, FastTickOptions());
|
||||
try
|
||||
{
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
Assert.Equal(3, repo.Upserted.Count);
|
||||
lock (repo.UpsertManyPageSizes)
|
||||
{
|
||||
Assert.Contains(3, repo.UpsertManyPageSizes);
|
||||
}
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(3),
|
||||
interval: TimeSpan.FromMilliseconds(50));
|
||||
}
|
||||
finally
|
||||
{
|
||||
StopTicker(actor);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Enumerator whose site list a test can swap between ticks.</summary>
|
||||
private sealed class MutableEnumerator : ISiteEnumerator
|
||||
{
|
||||
private volatile IReadOnlyList<SiteEntry> _sites;
|
||||
private int _calls;
|
||||
|
||||
/// <summary>Creates the enumerator with an initial site list.</summary>
|
||||
/// <param name="sites">The initial sites.</param>
|
||||
public MutableEnumerator(params SiteEntry[] sites) => _sites = sites;
|
||||
|
||||
/// <summary>Gets the number of enumerations performed so far.</summary>
|
||||
public int Calls => Volatile.Read(ref _calls);
|
||||
|
||||
/// <summary>Replaces the site list returned from the next enumeration on.</summary>
|
||||
/// <param name="sites">The new sites.</param>
|
||||
public void Set(params SiteEntry[] sites) => _sites = sites;
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyList<SiteEntry>> EnumerateAsync(CancellationToken ct = default)
|
||||
{
|
||||
Interlocked.Increment(ref _calls);
|
||||
return Task.FromResult(_sites);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a site the enumerator stops returning has its cursor and pinned latch
|
||||
/// pruned: the pinned site publishes Pinned=false, and when it reappears its pull restarts
|
||||
/// from <see cref="DateTime.MinValue"/> (lane H follow-up to review 2026-10-21).
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void ReconciliationTick_RemovedSite_PrunesCursorAndPinnedLatch()
|
||||
{
|
||||
var siteId = "siteGone";
|
||||
var ts = new DateTime(2026, 5, 20, 10, 0, 0, DateTimeKind.Utc);
|
||||
var rows = new[]
|
||||
{
|
||||
NewRow(TrackedOperationId.New(), siteId, updatedAtUtc: ts),
|
||||
NewRow(TrackedOperationId.New(), siteId, updatedAtUtc: ts),
|
||||
};
|
||||
var entry = new SiteEntry(siteId, "http://siteGone:8083");
|
||||
var sites = new MutableEnumerator(entry);
|
||||
var client = new LegacyIgnoresAfterIdPullClient(rows);
|
||||
var recorder = new RecordingPullClient(client);
|
||||
var repo = new RecordingRepo();
|
||||
// A dedicated probe, not TestActor: the re-added legacy site pins again, and that
|
||||
// event must not land in StopTicker's ExpectTerminated.
|
||||
var pinnedProbe = CreateTestProbe();
|
||||
Sys.EventStream.Subscribe(pinnedProbe.Ref, typeof(SiteCallReconciliationPinnedChanged));
|
||||
|
||||
var actor = CreateActor(sites, recorder, repo, new SiteCallAuditOptions
|
||||
{
|
||||
ReconciliationIntervalOverride = TimeSpan.FromMilliseconds(100),
|
||||
ReconciliationBatchSize = 2,
|
||||
});
|
||||
try
|
||||
{
|
||||
Assert.True(pinnedProbe.ExpectMsg<SiteCallReconciliationPinnedChanged>(TimeSpan.FromSeconds(5)).Pinned);
|
||||
|
||||
sites.Set();
|
||||
var unpinned = pinnedProbe.ExpectMsg<SiteCallReconciliationPinnedChanged>(TimeSpan.FromSeconds(5));
|
||||
Assert.Equal(siteId, unpinned.SiteId);
|
||||
Assert.False(unpinned.Pinned);
|
||||
|
||||
var callsBeforeReturn = recorder.Calls.Count;
|
||||
sites.Set(entry);
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
var later = recorder.Calls.Skip(callsBeforeReturn).ToList();
|
||||
Assert.NotEmpty(later);
|
||||
Assert.Equal(DateTime.MinValue, later[0].SinceUtc);
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(3),
|
||||
interval: TimeSpan.FromMilliseconds(50));
|
||||
}
|
||||
finally
|
||||
{
|
||||
Sys.EventStream.Unsubscribe(pinnedProbe.Ref);
|
||||
StopTicker(actor);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Wraps a pull client and records the cursor of every pull, thread-safely.</summary>
|
||||
private sealed class RecordingPullClient(IPullSiteCallsClient inner) : IPullSiteCallsClient
|
||||
{
|
||||
private readonly List<(DateTime SinceUtc, string? AfterId)> _calls = new();
|
||||
|
||||
/// <summary>Gets a snapshot of every pull cursor recorded so far.</summary>
|
||||
public IReadOnlyList<(DateTime SinceUtc, string? AfterId)> Calls
|
||||
{
|
||||
get { lock (_calls) { return _calls.ToList(); } }
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<PullSiteCallsResponse> PullAsync(
|
||||
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
|
||||
{
|
||||
lock (_calls) { _calls.Add((sinceUtc, afterId)); }
|
||||
return inner.PullAsync(siteId, sinceUtc, afterId, batchSize, ct);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 9. The reconciliation drain runs OFF the mailbox, so ingest,
|
||||
// query and KPI messages are answered while a long post-outage
|
||||
|
||||
@@ -116,6 +116,10 @@ internal sealed class SerializingSiteCallAuditRepository : ISiteCallAuditReposit
|
||||
public Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default) =>
|
||||
RunAsync(() => _inner.UpsertAsync(siteCall, ct));
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task UpsertManyAsync(IReadOnlyList<SiteCall> siteCalls, CancellationToken ct = default) =>
|
||||
RunAsync(() => _inner.UpsertManyAsync(siteCalls, ct));
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default) =>
|
||||
RunAsync(() => _inner.GetAsync(id, ct));
|
||||
|
||||
+120
@@ -1057,4 +1057,124 @@ public class OperationTrackingStoreTests : IDisposable
|
||||
Assert.Equal(System.Data.ConnectionState.Closed, connection.State);
|
||||
}
|
||||
|
||||
|
||||
// ── Central-ack filter for the reconciliation pull (review 2026-10-21 06/PERF-2) ──
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a row whose current version central has acknowledged through the
|
||||
/// cached-telemetry push is no longer served by the reconciliation read, while
|
||||
/// unacknowledged rows still are.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ReadChangedSince_SkipsRowsCentralAckedAtTheirCurrentVersion()
|
||||
{
|
||||
var (store, _) = CreateStore(nameof(ReadChangedSince_SkipsRowsCentralAckedAtTheirCurrentVersion));
|
||||
await using var _store = store;
|
||||
var acked = TrackedOperationId.New();
|
||||
var unacked = TrackedOperationId.New();
|
||||
await store.RecordEnqueueAsync(acked, nameof(AuditKind.ApiCallCached), "A", null, null, "node-a");
|
||||
await store.RecordEnqueueAsync(unacked, nameof(AuditKind.ApiCallCached), "B", null, null, "node-a");
|
||||
var snapshot = await store.GetStatusAsync(acked);
|
||||
|
||||
await store.RecordCentralAcksAsync(new Dictionary<TrackedOperationId, DateTime>
|
||||
{
|
||||
[acked] = snapshot!.UpdatedAtUtc,
|
||||
});
|
||||
|
||||
var rows = await store.ReadChangedSinceAsync(DateTime.MinValue, batchSize: 100);
|
||||
Assert.Equal(new[] { unacked }, rows.Select(r => r.TrackedOperationId).ToArray());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a row changed AFTER central acknowledged it is served again — the ack
|
||||
/// covers one version, not the operation.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ReadChangedSince_ServesRowChangedAfterItsAck()
|
||||
{
|
||||
var (store, _) = CreateStore(nameof(ReadChangedSince_ServesRowChangedAfterItsAck));
|
||||
await using var _store = store;
|
||||
var id = TrackedOperationId.New();
|
||||
await store.RecordEnqueueAsync(id, nameof(AuditKind.ApiCallCached), "A", null, null, "node-a");
|
||||
var submitted = await store.GetStatusAsync(id);
|
||||
await store.RecordCentralAcksAsync(new Dictionary<TrackedOperationId, DateTime>
|
||||
{
|
||||
[id] = submitted!.UpdatedAtUtc,
|
||||
});
|
||||
await Task.Delay(5); // a strictly later UpdatedAtUtc stamp
|
||||
await store.RecordTerminalAsync(id, nameof(AuditStatus.Delivered), null, 200);
|
||||
|
||||
var rows = await store.ReadChangedSinceAsync(DateTime.MinValue, batchSize: 100);
|
||||
var row = Assert.Single(rows);
|
||||
Assert.Equal(nameof(AuditStatus.Delivered), row.Status);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that an older ack never rolls back a newer one (acks are monotonic per row),
|
||||
/// and that the ack ledger is pruned with the tracking rows it describes.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CentralAcks_AreMonotonic_AndPrunedWithTheirTrackingRow()
|
||||
{
|
||||
var (store, dataSource) = CreateStore(nameof(CentralAcks_AreMonotonic_AndPrunedWithTheirTrackingRow));
|
||||
await using var _store = store;
|
||||
var id = TrackedOperationId.New();
|
||||
await store.RecordEnqueueAsync(id, nameof(AuditKind.ApiCallCached), "A", null, null, "node-a");
|
||||
await store.RecordTerminalAsync(id, nameof(AuditStatus.Delivered), null, 200);
|
||||
var terminal = await store.GetStatusAsync(id);
|
||||
|
||||
await store.RecordCentralAcksAsync(new Dictionary<TrackedOperationId, DateTime> { [id] = terminal!.UpdatedAtUtc });
|
||||
await store.RecordCentralAcksAsync(new Dictionary<TrackedOperationId, DateTime> { [id] = terminal.CreatedAtUtc.AddSeconds(-1) });
|
||||
Assert.Empty(await store.ReadChangedSinceAsync(DateTime.MinValue, batchSize: 100));
|
||||
|
||||
// Age the terminal row past retention and purge: the ack row goes with it.
|
||||
await store.FlushAsync();
|
||||
using (var connection = OpenVerifierConnection(dataSource))
|
||||
using (var cmd = connection.CreateCommand())
|
||||
{
|
||||
cmd.CommandText = "UPDATE OperationTracking SET TerminalAtUtc = $old;";
|
||||
cmd.Parameters.AddWithValue("$old", DateTime.UtcNow.AddDays(-30).ToString("o"));
|
||||
cmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
await store.PurgeTerminalAsync(DateTime.UtcNow.AddDays(-7));
|
||||
|
||||
using (var connection = OpenVerifierConnection(dataSource))
|
||||
using (var cmd = connection.CreateCommand())
|
||||
{
|
||||
cmd.CommandText = "SELECT COUNT(*) FROM OperationTrackingCentralAck;";
|
||||
Assert.Equal(0L, (long)cmd.ExecuteScalar()!);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies the ack ledger is its own table, not a column on the LocalDb-replicated
|
||||
/// <c>OperationTracking</c> table (whose capture triggers are built from its column list).
|
||||
/// Being node-local, a standby that takes over has no ack rows and serves every row once.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CentralAckLedger_IsADistinctNodeLocalTable_NotAColumnOnTheReplicatedTable()
|
||||
{
|
||||
var (store, dataSource) = CreateStore(nameof(CentralAckLedger_IsADistinctNodeLocalTable_NotAColumnOnTheReplicatedTable));
|
||||
await using var _store = store;
|
||||
await store.FlushAsync();
|
||||
|
||||
using var connection = OpenVerifierConnection(dataSource);
|
||||
using var cmd = connection.CreateCommand();
|
||||
cmd.CommandText = "PRAGMA table_info(OperationTracking);";
|
||||
var columns = new List<string>();
|
||||
using (var reader = cmd.ExecuteReader())
|
||||
{
|
||||
while (reader.Read()) columns.Add(reader.GetString(1));
|
||||
}
|
||||
|
||||
Assert.DoesNotContain(columns, c => c.StartsWith("Acked", StringComparison.Ordinal)
|
||||
|| c.Contains("CentralAck", StringComparison.Ordinal));
|
||||
cmd.CommandText = "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'OperationTrackingCentralAck';";
|
||||
Assert.Equal(1L, (long)cmd.ExecuteScalar()!);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user