dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(notifications): list sort index + CAS operator Retry/Discard (P5-3, P5-4)

Browse Source 
P5-3: add IX_Notifications_Created_NotificationId (CreatedAt DESC,
NotificationId DESC) — the Central UI Notification Outbox list page sorts on
exactly that pair, but no existing index leads unconditionally on CreatedAt,
so an unfiltered/lightly filtered page fell back to a full clustered-index
scan + sort on every COUNT + page request. Mirrors the Medium wave's
IX_SiteCalls_Created_TrackedOperationId migration + SQL twin pattern
faithfully, including the handwritten docs/plans/sql script.

P5-4: operator Retry/Discard was read-check-write keyed on NotificationId
only — a detached PipeTo task interleaving with the dispatcher's own write
(or a second concurrent operator action) could land between the read and
the write and get silently clobbered by the unconditional UpdateAsync,
stamping a Delivered row Discarded with stale fields. Adds
INotificationOutboxRepository.TryUpdateFromParkedAsync, a single-statement
compare-and-swap (WHERE Status == Parked) mirroring
SecuredWriteRepository.TryMarkApprovedAsync's CAS shape. NotificationOutboxActor's
RetryAsync/DiscardAsync now use the CAS write and, on a lost race, re-read
to report "notification already progressed to <Status>" instead of silently
overwriting whatever is there now.

Proof of failure: reverting only the actor's CAS usage while keeping the
repository addition reproduces the silent-overwrite bug — 7 tests fail,
including two new deterministic (non-sleep) race-proof tests that mock the
exact GetByIdAsync -> lost-CAS -> re-read sequence the finding describes.
P5-3 is perf-only; verified via migration/SQL-twin parity instead (no
runtime proof-of-failure is meaningful for an index).
This commit is contained in:
Joseph Doherty
2026-08-19 16:20:18 -04:00
parent 11675285af
commit 1cea01d924
12 changed files with 2607 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests/NotificationOutboxActorQueryTests.cs
+84 -12
View File
@@ -258,7 +258,7 @@ public class NotificationOutboxActorQueryTests : TestKit
Assert.True(response.Success);
Assert.Null(response.ErrorMessage);
_repository.Received(1).UpdateAsync(
_repository.Received(1).TryUpdateFromParkedAsync(
Arg.Is<Notification>(n =>
n.NotificationId == row.NotificationId &&
n.Status == NotificationStatus.Pending &&
@@ -280,7 +280,7 @@ public class NotificationOutboxActorQueryTests : TestKit
var response = ExpectMsg<RetryNotificationResponse>();
Assert.False(response.Success);
Assert.NotNull(response.ErrorMessage);
_repository.DidNotReceive().UpdateAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>());
_repository.DidNotReceive().TryUpdateFromParkedAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>());
}
[Fact]
@@ -300,17 +300,19 @@ public class NotificationOutboxActorQueryTests : TestKit
/// <summary>
/// The row is read successfully but the retention purge deletes it before the
/// write lands, so the targeted UPDATE matches nothing. The operator must be
/// told the notification is gone — reporting a re-queue that never happened
/// is what the ExecuteUpdate rewrite silently introduced (the predecessor
/// threw DbUpdateConcurrencyException here).
/// write lands, so the CAS write matches nothing. The re-read that follows a
/// lost CAS (arch-review 2026-08-19 P5-4) finds nothing either, and the
/// operator must be told the notification is gone — reporting a re-queue
/// that never happened is what a blind ExecuteUpdate would silently do (the
/// pre-ExecuteUpdate predecessor threw DbUpdateConcurrencyException here).
/// </summary>
[Fact]
public void Retry_RowPurgedBeforeWrite_RepliesNotFound()
{
var row = MakeNotification(status: NotificationStatus.Parked, retryCount: 10, lastError: "gave up");
_repository.GetByIdAsync(row.NotificationId, Arg.Any<CancellationToken>()).Returns(row);
_repository.UpdateAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>()).Returns(false);
_repository.GetByIdAsync(row.NotificationId, Arg.Any<CancellationToken>())
.Returns(row, (Notification?)null);
_repository.TryUpdateFromParkedAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>()).Returns(false);
var actor = CreateActor();
actor.Tell(new RetryNotificationRequest("corr-vanished-retry", row.NotificationId), TestActor);
@@ -327,8 +329,9 @@ public class NotificationOutboxActorQueryTests : TestKit
public void Discard_RowPurgedBeforeWrite_RepliesNotFound()
{
var row = MakeNotification(status: NotificationStatus.Parked);
_repository.GetByIdAsync(row.NotificationId, Arg.Any<CancellationToken>()).Returns(row);
_repository.UpdateAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>()).Returns(false);
_repository.GetByIdAsync(row.NotificationId, Arg.Any<CancellationToken>())
.Returns(row, (Notification?)null);
_repository.TryUpdateFromParkedAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>()).Returns(false);
var actor = CreateActor();
actor.Tell(new DiscardNotificationRequest("corr-vanished-discard", row.NotificationId), TestActor);
@@ -340,6 +343,75 @@ public class NotificationOutboxActorQueryTests : TestKit
Assert.Contains("not found", response.ErrorMessage);
}
/// <summary>
/// PROOF OF FAILURE for arch-review 2026-08-19 P5-4. Models the race the
/// finding describes: the operator's read sees Parked, but by the time the
/// write lands the dispatcher has already delivered the row (a detached
/// PipeTo task interleaving with the dispatcher's own write). The mechanism
/// is proven deterministically via a mocked call sequence, not a wall-clock
/// sleep (per the "no new sleep-then-assert" constraint) — the first
/// <see cref="INotificationOutboxRepository.GetByIdAsync"/> call (the
/// operator's read) returns the Parked row, the CAS write reports a lost
/// race, and the SECOND <c>GetByIdAsync</c> call (the fix's re-read used to
/// build an honest reply) returns the row now Delivered.
/// <para>
/// Against the pre-fix actor — which calls the unconditional
/// <c>UpdateAsync</c> exactly once and never re-reads — this test fails:
/// the single-read code path only ever consumes the first queued value
/// (Parked), the in-memory check passes, <c>UpdateAsync</c> is stubbed
/// <c>Healthy()</c> (returns true), and the response comes back
/// <c>Success = true</c> — i.e. the Delivered row gets silently clobbered
/// to Discarded. Post-fix, the CAS is lost, the actor re-reads and reports
/// "already progressed to Delivered" instead of overwriting it.
/// </para>
/// </summary>
[Fact]
public void Discard_ConcurrentlyDelivered_RepliesAlreadyProgressed_DoesNotOverwrite()
{
var parkedRow = MakeNotification(status: NotificationStatus.Parked, retryCount: 5, lastError: "gave up");
var deliveredRow = MakeNotification(status: NotificationStatus.Delivered);
_repository.GetByIdAsync(parkedRow.NotificationId, Arg.Any<CancellationToken>())
.Returns(parkedRow, deliveredRow);
_repository.TryUpdateFromParkedAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>()).Returns(false);
var actor = CreateActor();
actor.Tell(new DiscardNotificationRequest("corr-race-discard", parkedRow.NotificationId), TestActor);
var response = ExpectMsg<DiscardNotificationResponse>();
Assert.Equal("corr-race-discard", response.CorrelationId);
Assert.False(response.Success);
Assert.NotNull(response.ErrorMessage);
Assert.Contains("already progressed", response.ErrorMessage);
Assert.Contains("Delivered", response.ErrorMessage);
// The critical invariant: the old unconditional UpdateAsync must NEVER
// be reached by the operator path — that call is what silently
// clobbered the Delivered row pre-fix.
_repository.DidNotReceive().UpdateAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>());
}
/// <summary>Retry half of <see cref="Discard_ConcurrentlyDelivered_RepliesAlreadyProgressed_DoesNotOverwrite"/>.</summary>
[Fact]
public void Retry_ConcurrentlyDelivered_RepliesAlreadyProgressed_DoesNotOverwrite()
{
var parkedRow = MakeNotification(status: NotificationStatus.Parked, retryCount: 5, lastError: "gave up");
var deliveredRow = MakeNotification(status: NotificationStatus.Delivered);
_repository.GetByIdAsync(parkedRow.NotificationId, Arg.Any<CancellationToken>())
.Returns(parkedRow, deliveredRow);
_repository.TryUpdateFromParkedAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>()).Returns(false);
var actor = CreateActor();
actor.Tell(new RetryNotificationRequest("corr-race-retry", parkedRow.NotificationId), TestActor);
var response = ExpectMsg<RetryNotificationResponse>();
Assert.Equal("corr-race-retry", response.CorrelationId);
Assert.False(response.Success);
Assert.NotNull(response.ErrorMessage);
Assert.Contains("already progressed", response.ErrorMessage);
Assert.Contains("Delivered", response.ErrorMessage);
_repository.DidNotReceive().UpdateAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>());
}
[Fact]
public void Discard_ParkedNotification_MarksDiscarded_AndSucceeds()
{
@@ -353,7 +425,7 @@ public class NotificationOutboxActorQueryTests : TestKit
Assert.Equal("corr-8", response.CorrelationId);
Assert.True(response.Success);
_repository.Received(1).UpdateAsync(
_repository.Received(1).TryUpdateFromParkedAsync(
Arg.Is<Notification>(n => n.Status == NotificationStatus.Discarded),
Arg.Any<CancellationToken>());
}
@@ -370,7 +442,7 @@ public class NotificationOutboxActorQueryTests : TestKit
var response = ExpectMsg<DiscardNotificationResponse>();
Assert.False(response.Success);
Assert.NotNull(response.ErrorMessage);
_repository.DidNotReceive().UpdateAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>());
_repository.DidNotReceive().TryUpdateFromParkedAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>());
}
[Fact]