Merge branch 'worktree-agent-adf34e265d2dcae96' into arch-review-remediation

tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/AuditLogIngestActorTests.cs

@@ -322,6 +322,47 @@ public class AuditLogIngestActorTests : TestKit, IClassFixture<MsSqlMigrationFix
         Assert.Equal(2, counter.Count);
     }
 
+    /// <summary>
+    /// The per-row fallback must run on its OWN cancellation token, never the
+    /// batch's. Sharing it meant that when the batch failed BECAUSE the 20 s
+    /// ingest budget expired, every fallback insert was handed an
+    /// already-cancelled token: N instant failures, N counter bumps, zero rows
+    /// accepted — the fallback's entire purpose (land the good rows) defeated at
+    /// the exact moment it was needed.
+    /// </summary>
+    /// <remarks>
+    /// Pinned by token IDENTITY rather than by expiring a real budget: the budget
+    /// is a fixed 20 s and a test that waited for it would cost 20 s of wall clock
+    /// to assert something the token comparison establishes outright. Two
+    /// <see cref="CancellationToken"/>s are equal iff they come from the same
+    /// source, so "not equal" IS "a fresh CTS", and a fresh CTS cannot be
+    /// pre-cancelled by the batch.
+    /// </remarks>
+    [Fact]
+    public async Task Receive_WhenBatchFails_PerRowFallbackUsesAFreshToken()
+    {
+        var repository = new TokenRecordingRepository();
+        var actor = CreateActor(repository);
+
+        var events = Enumerable.Range(0, 3).Select(_ => NewEvent(NewSiteId())).ToList();
+        actor.Tell(new IngestAuditEventsCommand(events), TestActor);
+
+        var reply = ExpectMsg<IngestAuditEventsReply>(TimeSpan.FromSeconds(10));
+
+        // The fallback landed every row.
+        Assert.Equal(3, reply.AcceptedEventIds.Count);
+
+        Assert.NotNull(repository.BatchToken);
+        Assert.Equal(3, repository.RowTokens.Count);
+        Assert.All(repository.RowTokens, rowToken =>
+        {
+            Assert.NotEqual(repository.BatchToken!.Value, rowToken);
+            Assert.False(rowToken.IsCancellationRequested);
+        });
+
+        await Task.CompletedTask;
+    }
+
     /// <summary>Counts how many times the guard's catch surfaced a write failure.</summary>
     private sealed class CountingFailureCounter : ICentralAuditWriteFailureCounter
     {
@@ -329,6 +370,62 @@ public class AuditLogIngestActorTests : TestKit, IClassFixture<MsSqlMigrationFix
         public void Increment() => Count++;
     }
 
+    /// <summary>
+    /// Fails the set-based insert (as an expired budget would) and records the
+    /// token handed to each write so the fallback's token can be compared with
+    /// the batch's.
+    /// </summary>
+    private sealed class TokenRecordingRepository : IAuditLogRepository
+    {
+        public CancellationToken? BatchToken { get; private set; }
+
+        public List<CancellationToken> RowTokens { get; } = new();
+
+        public Task<int> InsertManyIfNotExistsAsync(
+            IReadOnlyList<AuditEvent> events, TimeSpan? commandTimeout = null, CancellationToken ct = default)
+        {
+            BatchToken = ct;
+            throw new OperationCanceledException("simulated ingest-budget expiry", ct);
+        }
+
+        public Task InsertIfNotExistsAsync(AuditEvent evt, CancellationToken ct = default)
+        {
+            RowTokens.Add(ct);
+            return Task.CompletedTask;
+        }
+
+        public Task<IReadOnlyList<AuditEvent>> QueryAsync(
+            AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default) =>
+            throw new NotSupportedException();
+
+        public Task<long> SwitchOutPartitionAsync(
+            DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
+            throw new NotSupportedException();
+
+        public Task<long> PurgeChannelOlderThanAsync(
+            string channel, DateTime threshold, int batchSize, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
+            throw new NotSupportedException();
+
+        public Task<long> BackfillSourceNodeAsync(
+            string sentinel, DateTime before, int batchSize, CancellationToken ct = default) =>
+            throw new NotSupportedException();
+
+        public Task<IReadOnlyList<DateTime>> GetPartitionBoundariesOlderThanAsync(
+            DateTime threshold, CancellationToken ct = default) =>
+            throw new NotSupportedException();
+
+        public Task<ZB.MOM.WW.ScadaBridge.Commons.Types.AuditLogKpiSnapshot> GetKpiSnapshotAsync(
+            TimeSpan window, DateTime? nowUtc = null, CancellationToken ct = default) =>
+            throw new NotSupportedException();
+
+        public Task<IReadOnlyList<ExecutionTreeNode>> GetExecutionTreeAsync(
+            Guid executionId, CancellationToken ct = default) =>
+            throw new NotSupportedException();
+
+        public Task<IReadOnlyList<string>> GetDistinctSourceNodesAsync(CancellationToken ct = default) =>
+            throw new NotSupportedException();
+    }
+
     /// <summary>
     /// Tiny test double that delegates to a real repository but throws on a
     /// specified EventId. Used to verify per-row failure isolation: one bad

tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Repositories/AuditLogRepositoryTests.cs

@@ -707,6 +707,85 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
         Assert.Equal(250, count);
     }
 
+    /// <summary>
+    /// An over-long value must be REJECTED by the server on every write path, not
+    /// clipped to fit by the client. The set-based path used to declare each
+    /// string parameter at its COLUMN width (Target/Actor 256, Action 64,
+    /// Outcome 16, Category 32, SourceNode 64), which makes SqlClient truncate the
+    /// value at bind time and commit the shortened row — in an APPEND-ONLY audit
+    /// store, with no PayloadTruncated flag to admit it happened — while the
+    /// per-row and reconciliation paths sent the same value in full and let the
+    /// server raise 2628/8152. Reject-everywhere is the contract; this test pins
+    /// both halves of it.
+    /// </summary>
+    [SkippableFact]
+    public async Task InsertManyIfNotExistsAsync_TargetLongerThanColumn_IsRejected_NotTruncated()
+    {
+        Skip.IfNot(_fixture.Available, _fixture.SkipReason);
+
+        var siteId = NewSiteId();
+
+        // dbo.AuditLog.Target is nvarchar(256).
+        var overlongTarget = new string('T', 300);
+        var evt = NewEvent(
+            siteId,
+            occurredAtUtc: new DateTime(2026, 5, 22, 12, 0, 0, DateTimeKind.Utc),
+            target: overlongTarget);
+
+        await using var batchContext = CreateContext();
+        var batchRepo = new AuditLogRepository(batchContext);
+
+        // 2628 = "String or binary data would be truncated in table …" (SQL 2019+),
+        // 8152 = its pre-2019 predecessor. Either proves the server saw the full value.
+        var batchEx = await Assert.ThrowsAsync<SqlException>(
+            () => batchRepo.InsertManyIfNotExistsAsync(new[] { evt }));
+        Assert.Contains(batchEx.Number, new[] { 2628, 8152 });
+
+        // The per-row path — which the batch path falls back to — rejects it too.
+        await using var rowContext = CreateContext();
+        var rowRepo = new AuditLogRepository(rowContext);
+        var rowEx = await Assert.ThrowsAsync<SqlException>(
+            () => rowRepo.InsertIfNotExistsAsync(evt));
+        Assert.Contains(rowEx.Number, new[] { 2628, 8152 });
+
+        // Nothing landed — in particular, no 256-character mutilated copy.
+        await using var readContext = CreateContext();
+        var rows = await readContext.Set<AuditLogRow>()
+            .Where(e => e.SourceSiteId == siteId)
+            .ToListAsync();
+
+        Assert.Empty(rows);
+    }
+
+    /// <summary>
+    /// The companion to the rejection test: a value that FITS must still round-trip
+    /// through the set-based path byte for byte, at the exact column boundary.
+    /// </summary>
+    [SkippableFact]
+    public async Task InsertManyIfNotExistsAsync_TargetAtColumnLimit_RoundTripsIntact()
+    {
+        Skip.IfNot(_fixture.Available, _fixture.SkipReason);
+
+        var siteId = NewSiteId();
+        var boundaryTarget = new string('T', 256);
+        var evt = NewEvent(
+            siteId,
+            occurredAtUtc: new DateTime(2026, 5, 22, 12, 30, 0, DateTimeKind.Utc),
+            target: boundaryTarget);
+
+        await using var context = CreateContext();
+        var repo = new AuditLogRepository(context);
+        Assert.Equal(1, await repo.InsertManyIfNotExistsAsync(new[] { evt }));
+
+        await using var readContext = CreateContext();
+        var loaded = await readContext.Set<AuditLogRow>()
+            .Where(e => e.SourceSiteId == siteId)
+            .ToListAsync();
+
+        Assert.Single(loaded);
+        Assert.Equal(boundaryTarget, loaded[0].Target);
+    }
+
     [SkippableFact]
     public async Task InsertManyIfNotExistsAsync_EmptyBatch_IsANoOp()
     {
@@ -1346,12 +1425,14 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
         string? errorMessage = null,
         Guid? executionId = null,
         Guid? parentExecutionId = null,
-        string? sourceNode = null) =>
+        string? sourceNode = null,
+        string? target = null) =>
         ScadaBridgeAuditEventFactory.Create(
             channel: channel,
             kind: kind,
             status: status,
             occurredAtUtc: occurredAtUtc,
+            target: target,
             sourceNode: sourceNode,
             sourceSiteId: siteId,
             executionId: executionId,

tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Repositories/SiteCallAuditRepositoryTests.cs

@@ -101,15 +101,12 @@ public class SiteCallAuditRepositoryTests : IClassFixture<MsSqlMigrationFixture>
     {
         Skip.IfNot(_fixture.Available, _fixture.SkipReason);
 
-        // WP2.2 collapsed the two-round-trip "insert-if-absent then monotonic
-        // update" into ONE batch that updates first and inserts only when nothing
-        // matched. That makes @@ROWCOUNT = 0 ambiguous: it means "no such row"
-        // AND "the monotonic guard rejected this packet". Guarding the insert on
-        // @@ROWCOUNT alone would therefore let every stale/regressive packet
-        // append a SECOND row for an id that already exists — silently forking
-        // the mirror. The re-check for the row's existence is what prevents that,
-        // and this pins it: after a rejected regressive upsert there is still
-        // exactly ONE row, still carrying the advanced state.
+        // A rejected packet must leave the mirror untouched — one row, still
+        // carrying the advanced state. The hazard this pins is the insert leg
+        // firing for a row that already exists and forking the mirror into two
+        // rows for one id: the insert is gated on IF NOT EXISTS alone (never on
+        // "the update matched nothing", which is ALSO what a monotonic rejection
+        // looks like), so a stale or regressive packet is a pure no-op.
         var id = TrackedOperationId.New();
         await using var context = CreateContext();
         var repo = new SiteCallAuditRepository(context);
@@ -137,6 +134,109 @@ public class SiteCallAuditRepositoryTests : IClassFixture<MsSqlMigrationFixture>
         Assert.Null(loaded[0].LastError);
     }
 
+    /// <summary>
+    /// The deterministic half of the concurrent-first-write regression: the OTHER
+    /// writer already committed the row, so this call's insert leg is skipped
+    /// entirely and only the monotonic UPDATE can carry its state in. Under the
+    /// UPDATE-first shape the loser of a first-write race ran its UPDATE BEFORE
+    /// any row existed and then skipped its INSERT, dropping the packet's
+    /// Status/RetryCount/HttpStatus/TerminalAtUtc on the floor.
+    /// </summary>
+    [SkippableFact]
+    public async Task UpsertAsync_RowAlreadyCreatedByAnotherWriter_StillAppliesThisPacketsState()
+    {
+        Skip.IfNot(_fixture.Available, _fixture.SkipReason);
+
+        var id = TrackedOperationId.New();
+        var t0 = new DateTime(2026, 5, 22, 8, 0, 0, DateTimeKind.Utc);
+
+        // Writer 1 (say the reconciliation pull) wins the insert with an early state.
+        await using var winnerContext = CreateContext();
+        await new SiteCallAuditRepository(winnerContext).UpsertAsync(
+            NewRow(id, status: "Submitted", createdAtUtc: t0, updatedAtUtc: t0));
+
+        // Writer 2 (the cached dual-write) carries a LATER lifecycle state for the
+        // same id, on its own connection.
+        await using var loserContext = CreateContext();
+        var loserRepo = new SiteCallAuditRepository(loserContext);
+        await loserRepo.UpsertAsync(NewRow(
+            id,
+            status: "Parked",
+            retryCount: 4,
+            lastError: "gave up",
+            httpStatus: 503,
+            createdAtUtc: t0,
+            updatedAtUtc: t0.AddSeconds(30),
+            terminal: true));
+
+        await using var readContext = CreateContext();
+        var loaded = await readContext.Set<SiteCall>()
+            .Where(s => s.TrackedOperationId == id)
+            .ToListAsync();
+
+        Assert.Single(loaded);
+        Assert.Equal("Parked", loaded[0].Status);
+        Assert.Equal(4, loaded[0].RetryCount);
+        Assert.Equal(503, loaded[0].HttpStatus);
+        Assert.Equal("gave up", loaded[0].LastError);
+        Assert.NotNull(loaded[0].TerminalAtUtc);
+    }
+
+    /// <summary>
+    /// The real-concurrency half: two writers racing the FIRST packet of the same
+    /// id, on separate connections, carrying DIFFERENT lifecycle states. However
+    /// the interleaving falls — either insert winning, or the loser eating a 2627
+    /// on its insert leg — the NEWER state must be the one on the row afterwards,
+    /// and exactly one row must exist.
+    /// </summary>
+    /// <remarks>
+    /// Repeated over several ids because the interleaving is not controllable; one
+    /// id would make the test a coin flip on whether it visits the racing path at
+    /// all. Every iteration must hold regardless of which way it raced, so the
+    /// test itself is not flaky — only its coverage of the rare branch is
+    /// probabilistic.
+    /// </remarks>
+    [SkippableFact]
+    public async Task UpsertAsync_TwoConcurrentFirstWrites_NewerStateSurvives()
+    {
+        Skip.IfNot(_fixture.Available, _fixture.SkipReason);
+
+        var t0 = new DateTime(2026, 5, 22, 9, 0, 0, DateTimeKind.Utc);
+
+        for (var i = 0; i < 12; i++)
+        {
+            var id = TrackedOperationId.New();
+
+            await using var contextA = CreateContext();
+            await using var contextB = CreateContext();
+            var repoA = new SiteCallAuditRepository(contextA);
+            var repoB = new SiteCallAuditRepository(contextB);
+
+            var early = NewRow(id, status: "Submitted", createdAtUtc: t0, updatedAtUtc: t0);
+            var late = NewRow(
+                id,
+                status: "Delivered",
+                retryCount: 2,
+                createdAtUtc: t0,
+                updatedAtUtc: t0.AddSeconds(10),
+                terminal: true);
+
+            await Task.WhenAll(
+                Task.Run(() => repoA.UpsertAsync(early)),
+                Task.Run(() => repoB.UpsertAsync(late)));
+
+            await using var readContext = CreateContext();
+            var loaded = await readContext.Set<SiteCall>()
+                .Where(s => s.TrackedOperationId == id)
+                .ToListAsync();
+
+            Assert.Single(loaded);
+            Assert.Equal("Delivered", loaded[0].Status);
+            Assert.Equal(2, loaded[0].RetryCount);
+            Assert.NotNull(loaded[0].TerminalAtUtc);
+        }
+    }
+
     [SkippableFact]
     public async Task UpsertAsync_SameStatus_EqualUpdatedAt_IsNoOp()
     {

tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/RepositoryCoverageTests.cs

@@ -537,7 +537,7 @@ public class NotificationOutboxRepositoryTests : IDisposable
         var loaded = await _repository.GetByIdAsync(id);
         loaded!.Status = NotificationStatus.Delivered;
         loaded.DeliveredAt = new DateTimeOffset(2026, 5, 19, 9, 0, 0, TimeSpan.Zero);
-        await _repository.UpdateAsync(loaded);
+        Assert.True(await _repository.UpdateAsync(loaded));
 
         _context.ChangeTracker.Clear();
         var reloaded = await _context.Notifications.FindAsync(id);
@@ -545,6 +545,33 @@ public class NotificationOutboxRepositoryTests : IDisposable
         Assert.Equal(new DateTimeOffset(2026, 5, 19, 9, 0, 0, TimeSpan.Zero), reloaded.DeliveredAt);
     }
 
+    /// <summary>
+    /// The write is a targeted UPDATE that bypasses the change tracker, so a row
+    /// deleted between the read and the write simply matches nothing. The row
+    /// count is the ONLY not-found signal available, and callers (the operator
+    /// retry/discard one-shots) depend on it to avoid reporting success against a
+    /// purged notification.
+    /// </summary>
+    [Fact]
+    public async Task UpdateAsync_RowPurgedBeforeWrite_ReturnsFalse()
+    {
+        var id = Guid.NewGuid().ToString();
+        _context.Notifications.Add(MakeNotification(id, NotificationStatus.Parked));
+        await _context.SaveChangesAsync();
+        _context.ChangeTracker.Clear();
+
+        var loaded = await _repository.GetByIdAsync(id);
+        Assert.NotNull(loaded);
+
+        // The daily retention purge removes the row while the operator's request
+        // is in flight.
+        await _context.Notifications.Where(n => n.NotificationId == id).ExecuteDeleteAsync();
+        _context.ChangeTracker.Clear();
+
+        loaded!.Status = NotificationStatus.Pending;
+        Assert.False(await _repository.UpdateAsync(loaded));
+    }
+
     [Fact]
     public async Task QueryAsync_AppliesFilters_OrdersByCreatedAtDescending_AndPaginates()
     {

tests/ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests/Ingest/NotificationIngestTypeStampingTests.cs

@@ -21,7 +21,7 @@ namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests.Ingest;
 public class NotificationIngestTypeStampingTests : TestKit
 {
     private readonly INotificationOutboxRepository _outboxRepository =
-        Substitute.For<INotificationOutboxRepository>();
+        OutboxRepositorySubstitute.Healthy();
 
     private readonly INotificationRepository _listRepository =
         Substitute.For<INotificationRepository>();

tests/ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests/NotificationOutboxActorAttemptEmissionTests.cs

@@ -26,7 +26,7 @@ namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests;
 public class NotificationOutboxActorAttemptEmissionTests : TestKit
 {
     private readonly INotificationOutboxRepository _outboxRepository =
-        Substitute.For<INotificationOutboxRepository>();
+        OutboxRepositorySubstitute.Healthy();
 
     private readonly INotificationRepository _notificationRepository =
         Substitute.For<INotificationRepository>();

tests/ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests/NotificationOutboxActorAuditInjectionTests.cs

@@ -21,7 +21,7 @@ public class NotificationOutboxActorAuditInjectionTests : TestKit
     private static IServiceProvider BuildEmptyProvider()
     {
         var services = new ServiceCollection();
-        services.AddScoped(_ => Substitute.For<INotificationOutboxRepository>());
+        services.AddScoped(_ => OutboxRepositorySubstitute.Healthy());
         services.AddScoped(_ => Substitute.For<INotificationRepository>());
         return services.BuildServiceProvider();
     }

tests/ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests/NotificationOutboxActorDispatchTests.cs

@@ -22,7 +22,7 @@ namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests;
 public class NotificationOutboxActorDispatchTests : TestKit
 {
     private readonly INotificationOutboxRepository _outboxRepository =
-        Substitute.For<INotificationOutboxRepository>();
+        OutboxRepositorySubstitute.Healthy();
 
     private readonly INotificationRepository _notificationRepository =
         Substitute.For<INotificationRepository>();
@@ -491,7 +491,7 @@ public class NotificationOutboxActorDispatchTests : TestKit
         // INotificationOutboxRepository registration with a private counting factory,
         // so we don't mutate the shared _outboxRepository field that other tests in
         // this class configure differently.
-        var outboxRepository = Substitute.For<INotificationOutboxRepository>();
+        var outboxRepository = OutboxRepositorySubstitute.Healthy();
         // De-race (S11): hand out a fresh due notification for the FIRST THREE claims, then
         // an empty batch forever. This caps the deliverable work — and therefore the
         // UpdateAsync count — at exactly three, no matter how many dispatch ticks the

tests/ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests/NotificationOutboxActorIngestTests.cs

@@ -21,7 +21,7 @@ namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests;
 public class NotificationOutboxActorIngestTests : TestKit
 {
     private readonly INotificationOutboxRepository _repository =
-        Substitute.For<INotificationOutboxRepository>();
+        OutboxRepositorySubstitute.Healthy();
 
     private IServiceProvider BuildServiceProvider()
     {

tests/ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests/NotificationOutboxActorPurgeTests.cs

@@ -21,7 +21,7 @@ namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests;
 public class NotificationOutboxActorPurgeTests : TestKit
 {
     private readonly INotificationOutboxRepository _outboxRepository =
-        Substitute.For<INotificationOutboxRepository>();
+        OutboxRepositorySubstitute.Healthy();
 
     private readonly INotificationRepository _notificationRepository =
         Substitute.For<INotificationRepository>();

tests/ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests/NotificationOutboxActorQueryTests.cs

@@ -22,7 +22,7 @@ namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests;
 public class NotificationOutboxActorQueryTests : TestKit
 {
     private readonly INotificationOutboxRepository _repository =
-        Substitute.For<INotificationOutboxRepository>();
+        OutboxRepositorySubstitute.Healthy();
 
     private IServiceProvider BuildServiceProvider()
     {
@@ -298,6 +298,48 @@ public class NotificationOutboxActorQueryTests : TestKit
         Assert.Contains("not found", response.ErrorMessage);
     }
 
+    /// <summary>
+    /// The row is read successfully but the retention purge deletes it before the
+    /// write lands, so the targeted UPDATE matches nothing. The operator must be
+    /// told the notification is gone — reporting a re-queue that never happened
+    /// is what the ExecuteUpdate rewrite silently introduced (the predecessor
+    /// threw DbUpdateConcurrencyException here).
+    /// </summary>
+    [Fact]
+    public void Retry_RowPurgedBeforeWrite_RepliesNotFound()
+    {
+        var row = MakeNotification(status: NotificationStatus.Parked, retryCount: 10, lastError: "gave up");
+        _repository.GetByIdAsync(row.NotificationId, Arg.Any<CancellationToken>()).Returns(row);
+        _repository.UpdateAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>()).Returns(false);
+        var actor = CreateActor();
+
+        actor.Tell(new RetryNotificationRequest("corr-vanished-retry", row.NotificationId), TestActor);
+
+        var response = ExpectMsg<RetryNotificationResponse>();
+        Assert.Equal("corr-vanished-retry", response.CorrelationId);
+        Assert.False(response.Success);
+        Assert.NotNull(response.ErrorMessage);
+        Assert.Contains("not found", response.ErrorMessage);
+    }
+
+    /// <summary>Discard half of <see cref="Retry_RowPurgedBeforeWrite_RepliesNotFound"/>.</summary>
+    [Fact]
+    public void Discard_RowPurgedBeforeWrite_RepliesNotFound()
+    {
+        var row = MakeNotification(status: NotificationStatus.Parked);
+        _repository.GetByIdAsync(row.NotificationId, Arg.Any<CancellationToken>()).Returns(row);
+        _repository.UpdateAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>()).Returns(false);
+        var actor = CreateActor();
+
+        actor.Tell(new DiscardNotificationRequest("corr-vanished-discard", row.NotificationId), TestActor);
+
+        var response = ExpectMsg<DiscardNotificationResponse>();
+        Assert.Equal("corr-vanished-discard", response.CorrelationId);
+        Assert.False(response.Success);
+        Assert.NotNull(response.ErrorMessage);
+        Assert.Contains("not found", response.ErrorMessage);
+    }
+
     [Fact]
     public void Discard_ParkedNotification_MarksDiscarded_AndSucceeds()
     {

tests/ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests/NotificationOutboxActorRetryEmissionTests.cs

@@ -25,7 +25,7 @@ namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests;
 public class NotificationOutboxActorRetryEmissionTests : TestKit
 {
     private readonly INotificationOutboxRepository _outboxRepository =
-        Substitute.For<INotificationOutboxRepository>();
+        OutboxRepositorySubstitute.Healthy();
 
     private readonly RecordingCentralAuditWriter _auditWriter = new();
 

tests/ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests/NotificationOutboxActorTerminalEmissionTests.cs

@@ -27,7 +27,7 @@ namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests;
 public class NotificationOutboxActorTerminalEmissionTests : TestKit
 {
     private readonly INotificationOutboxRepository _outboxRepository =
-        Substitute.For<INotificationOutboxRepository>();
+        OutboxRepositorySubstitute.Healthy();
 
     private readonly INotificationRepository _notificationRepository =
         Substitute.For<INotificationRepository>();

tests/ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests/OutboxRepositorySubstitute.cs

@@ -0,0 +1,31 @@
+using NSubstitute;
+using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
+
+namespace ZB.MOM.WW.ScadaBridge.NotificationOutbox.Tests;
+
+/// <summary>
+/// Factory for the outbox repository substitute used across these tests.
+/// </summary>
+/// <remarks>
+/// <see cref="INotificationOutboxRepository.UpdateAsync"/> returns whether the
+/// targeted UPDATE actually matched a row — <c>false</c> is the "this
+/// notification no longer exists" signal the operator retry/discard handlers
+/// turn into a not-found reply. NSubstitute's default for <c>Task&lt;bool&gt;</c>
+/// is <c>false</c>, i.e. the VANISHED-row answer, so an unconfigured substitute
+/// would silently put every test on the failure path. Tests that want a healthy
+/// store take one from here; the vanished-row tests configure
+/// <c>Returns(false)</c> for themselves.
+/// </remarks>
+internal static class OutboxRepositorySubstitute
+{
+    /// <summary>Creates a substitute whose writes report that the row was found.</summary>
+    public static INotificationOutboxRepository Healthy()
+    {
+        var repository = Substitute.For<INotificationOutboxRepository>();
+        repository
+            .UpdateAsync(Arg.Any<Notification>(), Arg.Any<CancellationToken>())
+            .Returns(true);
+        return repository;
+    }
+}

tests/ZB.MOM.WW.ScadaBridge.SiteCallAudit.Tests/SiteCallAuditReconciliationTests.cs

@@ -674,6 +674,164 @@ public class SiteCallAuditReconciliationTests : TestKit
         client.Release();
     }
 
+    /// <summary>
+    /// An injected repository is ONE instance — typically wrapping one
+    /// <c>DbContext</c> — shared by the mailbox handlers and the off-mailbox
+    /// reconciliation/purge passes, and <c>DbContext</c> forbids concurrent
+    /// operations. Every call the actor makes through an injected repository must
+    /// therefore be serialized, so a drain's upserts never overlap an ingest
+    /// upsert arriving on the mailbox. (Production is unaffected: each message and
+    /// each pass resolves its own scope, hence its own context.)
+    /// </summary>
+    [Fact]
+    public async Task InjectedRepository_IsNeverCalledConcurrently_ByDrainAndMailbox()
+    {
+        var siteId = "siteBusy";
+        var sites = new StaticEnumerator(new SiteEntry(siteId, "http://siteBusy:8083"));
+
+        // One page of rows for the drain to upsert, each call held open long
+        // enough that a concurrent mailbox upsert would land inside it.
+        var pulled = Enumerable.Range(0, 6)
+            .Select(_ => NewRow(TrackedOperationId.New(), siteId))
+            .ToArray();
+        var client = new OneBatchThenEmptyPullClient(pulled);
+        var repo = new ConcurrencyDetectingRepo(TimeSpan.FromMilliseconds(40));
+
+        var actor = CreateActor(sites, client, repo, FastTickOptions());
+
+        // Once the pull has been served the drain is upserting; flood the mailbox
+        // with ingest commands so the two writers overlap in wall-clock time.
+        await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
+
+        var asks = Enumerable.Range(0, 6)
+            .Select(_ => actor.Ask<UpsertSiteCallReply>(
+                new UpsertSiteCallCommand(NewRow(TrackedOperationId.New(), siteId)),
+                TimeSpan.FromSeconds(10)))
+            .ToArray();
+
+        await Task.WhenAll(asks);
+        Assert.All(asks, ask => Assert.True(ask.Result.Accepted));
+
+        // The drain's own upserts must have run in the same window.
+        AwaitAssert(
+            () => Assert.True(repo.UpsertCount >= 12, $"expected both writers to have run; saw {repo.UpsertCount}"),
+            TimeSpan.FromSeconds(5));
+
+        Assert.Equal(1, repo.MaxObservedConcurrency);
+    }
+
+    /// <summary>
+    /// Serves one page of rows on the first pull and nothing afterwards, so the
+    /// drain has real upsert work to do and then settles.
+    /// </summary>
+    private sealed class OneBatchThenEmptyPullClient : IPullSiteCallsClient
+    {
+        private readonly SiteCall[] _rows;
+        private readonly TaskCompletionSource _entered =
+            new(TaskCreationOptions.RunContinuationsAsynchronously);
+        private int _callCount;
+
+        public OneBatchThenEmptyPullClient(SiteCall[] rows) => _rows = rows;
+
+        public Task Entered => _entered.Task;
+
+        public Task<PullSiteCallsResponse> PullAsync(
+            string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
+        {
+            var first = Interlocked.Increment(ref _callCount) == 1;
+            _entered.TrySetResult();
+            return Task.FromResult(new PullSiteCallsResponse(
+                first ? _rows : Array.Empty<SiteCall>(), MoreAvailable: false));
+        }
+    }
+
+    /// <summary>
+    /// Records the peak number of overlapping repository calls. Each call is held
+    /// open briefly so an overlap, if the actor allows one, is observed rather
+    /// than missed by timing luck.
+    /// </summary>
+    private sealed class ConcurrencyDetectingRepo : ISiteCallAuditRepository
+    {
+        private readonly TimeSpan _hold;
+        private int _inFlight;
+        private int _maxObserved;
+        private int _upsertCount;
+
+        public ConcurrencyDetectingRepo(TimeSpan hold) => _hold = hold;
+
+        public int MaxObservedConcurrency => Volatile.Read(ref _maxObserved);
+
+        public int UpsertCount => Volatile.Read(ref _upsertCount);
+
+        public async Task UpsertAsync(SiteCall siteCall, CancellationToken ct = default)
+        {
+            Interlocked.Increment(ref _upsertCount);
+            await TrackAsync().ConfigureAwait(false);
+        }
+
+        public async Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default)
+        {
+            await TrackAsync().ConfigureAwait(false);
+            return null;
+        }
+
+        public async Task<IReadOnlyList<SiteCall>> QueryAsync(
+            SiteCallQueryFilter filter, SiteCallPaging paging, CancellationToken ct = default)
+        {
+            await TrackAsync().ConfigureAwait(false);
+            return Array.Empty<SiteCall>();
+        }
+
+        public async Task<int> PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default)
+        {
+            await TrackAsync().ConfigureAwait(false);
+            return 0;
+        }
+
+        public async Task<SiteCallKpiSnapshot> ComputeKpisAsync(
+            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default)
+        {
+            await TrackAsync().ConfigureAwait(false);
+            return new SiteCallKpiSnapshot(0, 0, 0, 0, null, 0);
+        }
+
+        public async Task<IReadOnlyList<SiteCallSiteKpiSnapshot>> ComputePerSiteKpisAsync(
+            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default)
+        {
+            await TrackAsync().ConfigureAwait(false);
+            return Array.Empty<SiteCallSiteKpiSnapshot>();
+        }
+
+        public async Task<IReadOnlyList<SiteCallNodeKpiSnapshot>> ComputePerNodeKpisAsync(
+            DateTime stuckCutoff, DateTime intervalSince, CancellationToken ct = default)
+        {
+            await TrackAsync().ConfigureAwait(false);
+            return Array.Empty<SiteCallNodeKpiSnapshot>();
+        }
+
+        private async Task TrackAsync()
+        {
+            var current = Interlocked.Increment(ref _inFlight);
+
+            // Monotonic max without a lock.
+            int seen;
+            while (current > (seen = Volatile.Read(ref _maxObserved))
+                   && Interlocked.CompareExchange(ref _maxObserved, current, seen) != seen)
+            {
+                // Another thread moved the max; re-read and retry.
+            }
+
+            try
+            {
+                await Task.Delay(_hold).ConfigureAwait(false);
+            }
+            finally
+            {
+                Interlocked.Decrement(ref _inFlight);
+            }
+        }
+    }
+
     /// <summary>
     /// <see cref="BlockingPullClient"/> that also counts invocations, so a test
     /// can prove no second pass started while the first was blocked.