fix(central): review findings — no client-side audit truncation, insert-first upsert, QI-safe scripts, honest operator replies
Six adversarial-review findings in the central SQL/ingest layer. F1 (AuditLogRepository.InsertChunkAsync) — the set-based ingest declared each string parameter at its COLUMN width (Actor/Target 256, Action 64, Outcome 16, Category 32, SourceNode 64), so SqlClient truncated an over-long value at bind time and committed the mutilated row — silent, in an append-only store, with no PayloadTruncated flag — while the per-row and reconciliation paths sent the same value in full and let the server reject it with 2628. Bind at the value's own length instead; explicit SqlDbType is kept (it fixes the VALUES constructor's derived column types and datetime2 precision). Design: reject everywhere, truncate nowhere — matching today's per-row behaviour. F2 (SiteCallAuditRepository.UpsertAsync) — the single-statement upsert ran the monotonic UPDATE first and INSERTed only if nothing matched. Two writers racing the first packet of one TrackedOperationId (the cached dual-write and the reconciliation pull carry DIFFERENT lifecycle states) both matched nothing, and the loser then skipped its INSERT or swallowed a 2627 — dropping its Status/RetryCount/HttpStatus/TerminalAtUtc. Legs swapped to `IF NOT EXISTS … INSERT; UPDATE <monotonic>` — still one round trip, and the loser's UPDATE now lands on the winner's row. The duplicate-key catch re-runs the monotonic UPDATE for the same reason. Moved to raw SQL with explicitly-typed parameters so the intricate rank predicate exists in exactly one place (an untyped DateTime would bind as `datetime` and round the freshness tiebreaker). F3 (docs/plans/sql/*.sql) — filtered-index DDL failed with error 1934 under the documented `docker exec … sqlcmd` path, which defaults QUOTED_IDENTIFIER OFF; once IX_Notifications_Delivered exists, QI-OFF DML on Notifications fails too. All four scripts now open with `SET QUOTED_IDENTIFIER ON; SET ANSI_NULLS ON; GO` (own batch, so it is in force when the next batch parses), and the migration convention in Component-ConfigurationDatabase.md documents `sqlcmd -I`. Verified live: the pre-fix script fails 1934 without -I, the fixed one applies. F4 (SiteCallAuditActor) — the off-mailbox reconciliation/purge passes reuse the injected repository, so tests drove one DbContext from the pass and a mailbox handler concurrently. Serialized at the CALL via a private SerializedRepository wrapper applied only by the test constructors, rather than running the pass on-mailbox: production keeps its PipeTo shape untouched, and the existing "a blocked drain does not stall ingest/query/KPI" regression tests stay meaningful (they would have been invalidated by suspending the mailbox). F5 (AuditLogIngestActor) — when the batch failed because the 20 s IngestBudget expired, the per-row fallback reused the same expired token: N instant failures, N counter bumps, zero accepted. The fallback now gets a fresh 5 s budget (inside the 30 s outer Ask), and a blown budget bumps the failure counter ONCE for the batch instead of once per row. F6 (NotificationOutboxRepository.UpdateAsync) — ExecuteUpdate's row count was discarded, so an operator Retry/Discard of a notification the retention purge had already deleted reported success (the pre-ExecuteUpdate code threw DbUpdateConcurrencyException). UpdateAsync now returns whether a row matched; the operator one-shots answer "notification not found" and emit no audit row for the action that did not happen, while the dispatcher logs a warning (its delivery already happened; nothing to retry). GetByIdAsync switched to AsNoTracking since the write is out-of-band. Tests: 5 new SQL-backed regressions (over-long Target rejected on both paths + boundary round-trip; concurrent first-write and already-created-by-another-writer upserts; vanished-row UpdateAsync), a token-identity pin on the ingest fallback, a repository-concurrency detector for the SiteCallAudit passes, and vanished-row operator-path tests. The F1/F2/F4 regressions were each confirmed failing against the pre-fix code. Suites: ConfigurationDatabase 369, AuditLog 378, SiteCallAudit 66, NotificationOutbox 152 — all green, solution builds with 0 warnings.
This commit is contained in:
@@ -603,7 +603,7 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
|
||||
notification.Status = NotificationStatus.Parked;
|
||||
notification.LastError = missingAdapterError;
|
||||
notification.LastAttemptAt = now;
|
||||
await outboxRepository.UpdateAsync(notification, cancellationToken);
|
||||
await WarnIfVanishedAsync(outboxRepository, notification, cancellationToken);
|
||||
await EmitAttemptAuditAsync(
|
||||
notification,
|
||||
now,
|
||||
@@ -654,7 +654,7 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
|
||||
break;
|
||||
}
|
||||
|
||||
await outboxRepository.UpdateAsync(notification, cancellationToken);
|
||||
await WarnIfVanishedAsync(outboxRepository, notification, cancellationToken);
|
||||
|
||||
// Emit the per-attempt Attempted row exactly once regardless of the
|
||||
// outcome (B2). The error message comes from the outcome, not from
|
||||
@@ -680,6 +680,35 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Persists the dispatcher's delivery-state write and logs when the row has
|
||||
/// VANISHED underneath it — <see cref="INotificationOutboxRepository.UpdateAsync"/>
|
||||
/// returning false means the retention purge deleted the notification between
|
||||
/// the claim and the write.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The dispatcher deliberately does not treat this as an error: the delivery
|
||||
/// itself already happened (or failed) and there is no row left to record the
|
||||
/// outcome on, so there is nothing to retry or roll back. The audit rows are
|
||||
/// still emitted — they are the durable record. The operator one-shots
|
||||
/// (retry/discard) take the opposite stance and answer "not found", because a
|
||||
/// human is waiting on that answer.
|
||||
/// </remarks>
|
||||
private async Task WarnIfVanishedAsync(
|
||||
INotificationOutboxRepository repository,
|
||||
Notification notification,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var persisted = await repository.UpdateAsync(notification, cancellationToken);
|
||||
if (!persisted)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Notification {NotificationId} disappeared before its delivery state could be written (status {Status}); the row was most likely purged mid-flight.",
|
||||
notification.NotificationId,
|
||||
notification.Status);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// True for <see cref="NotificationStatus.Delivered"/>,
|
||||
/// <see cref="NotificationStatus.Parked"/>, or
|
||||
@@ -1098,7 +1127,16 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
|
||||
notification.RetryCount = 0;
|
||||
notification.NextAttemptAt = null;
|
||||
notification.LastError = null;
|
||||
await repository.UpdateAsync(notification);
|
||||
|
||||
// Zero rows updated means the row was purged between the read above and
|
||||
// this write. Answer the operator honestly instead of reporting a
|
||||
// re-queue that never happened — and emit no un-park audit row, because
|
||||
// there is nothing to attribute it to.
|
||||
if (!await repository.UpdateAsync(notification))
|
||||
{
|
||||
return new RetryNotificationResponse(
|
||||
request.CorrelationId, Success: false, ErrorMessage: "notification not found");
|
||||
}
|
||||
|
||||
// Operator re-queued a parked notification. Emit a Submitted NotifyDeliver
|
||||
// row attributing the un-park to the operator — otherwise the lifecycle
|
||||
@@ -1170,7 +1208,15 @@ public class NotificationOutboxActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
|
||||
notification.Status = NotificationStatus.Discarded;
|
||||
await repository.UpdateAsync(notification);
|
||||
|
||||
// Same vanished-row honesty as the retry path: a purge between the read
|
||||
// and the write leaves nothing to discard, so report not-found rather
|
||||
// than success (and skip the terminal audit row below).
|
||||
if (!await repository.UpdateAsync(notification))
|
||||
{
|
||||
return new DiscardNotificationResponse(
|
||||
request.CorrelationId, Success: false, ErrorMessage: "notification not found");
|
||||
}
|
||||
|
||||
// A manual discard is the OTHER code path that produces
|
||||
// a terminal NotificationStatus transition (alongside the dispatcher).
|
||||
|
||||
Reference in New Issue
Block a user