fix(notifications): list sort index + CAS operator Retry/Discard (P5-3, P5-4)

P5-3: add IX_Notifications_Created_NotificationId (CreatedAt DESC,
NotificationId DESC) — the Central UI Notification Outbox list page sorts on
exactly that pair, but no existing index leads unconditionally on CreatedAt,
so an unfiltered/lightly filtered page fell back to a full clustered-index
scan + sort on every COUNT + page request. Mirrors the Medium wave's
IX_SiteCalls_Created_TrackedOperationId migration + SQL twin pattern
faithfully, including the handwritten docs/plans/sql script.

P5-4: operator Retry/Discard was read-check-write keyed on NotificationId
only — a detached PipeTo task interleaving with the dispatcher's own write
(or a second concurrent operator action) could land between the read and
the write and get silently clobbered by the unconditional UpdateAsync,
stamping a Delivered row Discarded with stale fields. Adds
INotificationOutboxRepository.TryUpdateFromParkedAsync, a single-statement
compare-and-swap (WHERE Status == Parked) mirroring
SecuredWriteRepository.TryMarkApprovedAsync's CAS shape. NotificationOutboxActor's
RetryAsync/DiscardAsync now use the CAS write and, on a lost race, re-read
to report "notification already progressed to <Status>" instead of silently
overwriting whatever is there now.

Proof of failure: reverting only the actor's CAS usage while keeping the
repository addition reproduces the silent-overwrite bug — 7 tests fail,
including two new deterministic (non-sleep) race-proof tests that mock the
exact GetByIdAsync -> lost-CAS -> re-read sequence the finding describes.
P5-3 is perf-only; verified via migration/SQL-twin parity instead (no
runtime proof-of-failure is meaningful for an index).
This commit is contained in:
Joseph Doherty
2026-08-19 16:20:18 -04:00
parent 11675285af
commit 1cea01d924
12 changed files with 2607 additions and 32 deletions
@@ -83,5 +83,23 @@ public class NotificationOutboxConfiguration : IEntityTypeConfiguration<Notifica
builder.HasIndex(n => n.DeliveredAt)
.HasDatabaseName("IX_Notifications_Delivered")
.HasFilter("[Status] = 'Delivered'");
// Created_NotificationId backs the Central UI Notification Outbox list
// page's sort key — NotificationOutboxRepository.QueryAsync orders
// (CreatedAt DESC, NotificationId DESC) (NotificationId breaks CreatedAt
// ties for deterministic OFFSET paging). None of the indexes above lead
// unconditionally on CreatedAt: Status_NextAttemptAt requires an equality
// on Status first, SourceSiteId_CreatedAt requires an equality on
// SourceSiteId first, and IX_Notifications_Delivered is filtered to
// Status='Delivered' only. So an unfiltered (or lightly filtered) page
// fell back to a full clustered-index scan + sort on every COUNT + page
// request (arch-review 2026-08-19 P5-3, register row-35 class). With both
// key columns descending, a forward scan of this index already delivers
// rows in the requested order. Exact mirror of
// SiteCallEntityTypeConfiguration's IX_SiteCalls_Created_TrackedOperationId
// (2026-08-19, arch-review P5-2).
builder.HasIndex(n => new { n.CreatedAt, n.NotificationId })
.IsDescending(true, true)
.HasDatabaseName("IX_Notifications_Created_NotificationId");
}
}
@@ -0,0 +1,46 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
{
/// <summary>
/// Adds <c>IX_Notifications_Created_NotificationId (CreatedAt DESC,
/// NotificationId DESC)</c> — arch-review 2026-08-19 P5-3, register
/// row-35 class.
/// </summary>
/// <remarks>
/// <see cref="ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories.NotificationOutboxRepository.QueryAsync"/>
/// orders <c>(CreatedAt DESC, NotificationId DESC)</c> for the Central UI
/// Notification Outbox list page. None of the existing Notifications indexes
/// lead unconditionally on <c>CreatedAt</c> — <c>IX_Notifications_Status_NextAttemptAt</c>
/// requires an equality on <c>Status</c> first, <c>IX_Notifications_SourceSiteId_CreatedAt</c>
/// requires an equality on <c>SourceSiteId</c> first, and
/// <c>IX_Notifications_Delivered</c> is filtered to <c>Status = 'Delivered'</c>
/// only — so an unfiltered (or lightly filtered) list page fell back to a full
/// clustered-index scan + sort on every COUNT + page request. With both key
/// columns descending, a forward scan of this index already delivers rows in
/// the requested order. See <c>docs/plans/sql/AddNotificationsListSortIndex.sql</c>
/// for the handwritten production-parity script.
/// </remarks>
public partial class AddNotificationsListSortIndex : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateIndex(
name: "IX_Notifications_Created_NotificationId",
table: "Notifications",
columns: new[] { "CreatedAt", "NotificationId" },
descending: new bool[0]);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_Notifications_Created_NotificationId",
table: "Notifications");
}
}
}
@@ -913,6 +913,10 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
.HasDatabaseName("IX_Notifications_Delivered")
.HasFilter("[Status] = 'Delivered'");
b.HasIndex("CreatedAt", "NotificationId")
.IsDescending()
.HasDatabaseName("IX_Notifications_Created_NotificationId");
b.HasIndex("SourceSiteId", "CreatedAt");
b.HasIndex("Status", "NextAttemptAt");
@@ -205,6 +205,48 @@ VALUES
return rowsAffected > 0;
}
/// <inheritdoc />
public async Task<bool> TryUpdateFromParkedAsync(Notification n, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(n);
// Same seven mutable delivery-state columns as UpdateAsync, but the WHERE
// clause below ALSO requires the row's current Status to still be Parked
// -- a single-statement compare-and-swap (arch-review 2026-08-19 P5-4).
// Two round trips (GetByIdAsync then this write) with a status check in
// between is not a correctness guard by itself: a detached PipeTo task
// can race the dispatcher's own write (or a second concurrent operator
// action) in that window. Mirrors SecuredWriteRepository.TryMarkApprovedAsync's
// conditional ExecuteUpdate/ExecuteSql pattern, expressed via LINQ
// ExecuteUpdateAsync (like UpdateAsync above) so this stays portable
// across the SQL Server production provider and the SQLite provider the
// repository tests run against.
var status = n.Status;
var retryCount = n.RetryCount;
var lastError = n.LastError;
var resolvedTargets = n.ResolvedTargets;
var lastAttemptAt = n.LastAttemptAt;
var nextAttemptAt = n.NextAttemptAt;
var deliveredAt = n.DeliveredAt;
var notificationId = n.NotificationId;
var rowsAffected = await _context.Notifications
.Where(row => row.NotificationId == notificationId && row.Status == NotificationStatus.Parked)
.ExecuteUpdateAsync(
setters => setters
.SetProperty(row => row.Status, status)
.SetProperty(row => row.RetryCount, retryCount)
.SetProperty(row => row.LastError, lastError)
.SetProperty(row => row.ResolvedTargets, resolvedTargets)
.SetProperty(row => row.LastAttemptAt, lastAttemptAt)
.SetProperty(row => row.NextAttemptAt, nextAttemptAt)
.SetProperty(row => row.DeliveredAt, deliveredAt),
cancellationToken);
return rowsAffected > 0;
}
/// <inheritdoc />
public async Task<Notification?> GetByIdAsync(string notificationId, CancellationToken cancellationToken = default)
{