fix(concurrency): close 8 race / thread-safety findings across CD, DCL, SR

CD-015: rewrite NotificationOutboxRepository.InsertIfNotExistsAsync as raw-SQL IF NOT EXISTS … INSERT with SqlException 2601/2627 catch, ending the at-least-once livelock on the site→central notification handoff. DCL-018/019/020/021/022: add _subscribesInFlight guard so concurrent same-tag subscribes don't orphan an adapter handle; delete the latent dead _subscriptionHandles dictionary; stop double-counting _totalSubscribed when an unresolved tag is promoted via another instance; release adapter handles on mid-flight unsubscribe; gate the tag-resolution retry timer with IsTimerActive so subscribe bursts don't reset it into starvation. SR-020: add _terminatingActorsByName shadow so a third deploy arriving during a pending redeploy doesn't crash on InvalidActorNameException — displaced senders get a Failed/superseded response and the latest command wins on Terminated. SR-024: split OperationTrackingStore reads from writes (fresh SqliteConnection per GetStatusAsync) so long writes don't block status queries; rewrite Dispose to drop the sync-over-async bridge that could deadlock on a non-reentrant SyncContext; Interlocked.Exchange makes the dispose-once flag race-safe across both paths.
2026-05-28 05:20:13 -04:00
parent 5d2386cc9d
commit f936f55f51
15 changed files with 1152 additions and 170 deletions
@@ -0,0 +1,132 @@
+using Microsoft.EntityFrameworkCore;
+using ScadaLink.Commons.Entities.Notifications;
+using ScadaLink.Commons.Types.Enums;
+using ScadaLink.ConfigurationDatabase;
+using ScadaLink.ConfigurationDatabase.Repositories;
+using ScadaLink.ConfigurationDatabase.Tests.Migrations;
+using Xunit;
+
+namespace ScadaLink.ConfigurationDatabase.Tests.Repositories;
+
+/// <summary>
+/// CD-015 race-fix integration tests for
+/// <see cref="NotificationOutboxRepository.InsertIfNotExistsAsync"/>. The method
+/// is raw-SQL (<c>IF NOT EXISTS … INSERT</c>) matching the AuditLog and SiteCalls
+/// idempotent-insert pattern; it must execute against a real SQL Server schema,
+/// so this class uses <see cref="MsSqlMigrationFixture"/> rather than the SQLite
+/// in-memory provider used by <see cref="RepositoryCoverageTests"/>.
+/// </summary>
+public class NotificationOutboxRepositoryIntegrationTests : IClassFixture<MsSqlMigrationFixture>
+{
+    private readonly MsSqlMigrationFixture _fixture;
+
+    public NotificationOutboxRepositoryIntegrationTests(MsSqlMigrationFixture fixture)
+    {
+        _fixture = fixture;
+    }
+
+    [SkippableFact]
+    public async Task InsertIfNotExistsAsync_FreshId_InsertsAndReturnsTrue()
+    {
+        Skip.IfNot(_fixture.Available, _fixture.SkipReason);
+
+        var id = Guid.NewGuid().ToString();
+        await using var context = CreateContext();
+        var repo = new NotificationOutboxRepository(context);
+
+        var inserted = await repo.InsertIfNotExistsAsync(MakeNotification(id));
+
+        Assert.True(inserted);
+
+        await using var readContext = CreateContext();
+        var loaded = await readContext.Notifications.FindAsync(id);
+        Assert.NotNull(loaded);
+        Assert.Equal("Subject", loaded!.Subject);
+    }
+
+    [SkippableFact]
+    public async Task InsertIfNotExistsAsync_DuplicateId_ReturnsFalseAndLeavesExistingRow()
+    {
+        Skip.IfNot(_fixture.Available, _fixture.SkipReason);
+
+        var id = Guid.NewGuid().ToString();
+        await using (var context = CreateContext())
+        {
+            var repo = new NotificationOutboxRepository(context);
+            await repo.InsertIfNotExistsAsync(MakeNotification(id, subject: "Original"));
+        }
+
+        await using (var context = CreateContext())
+        {
+            var repo = new NotificationOutboxRepository(context);
+            var inserted = await repo.InsertIfNotExistsAsync(MakeNotification(id, subject: "Changed"));
+            Assert.False(inserted);
+        }
+
+        await using var readContext = CreateContext();
+        var loaded = await readContext.Notifications.FindAsync(id);
+        Assert.NotNull(loaded);
+        Assert.Equal("Original", loaded!.Subject);
+    }
+
+    [SkippableFact]
+    public async Task InsertIfNotExistsAsync_ConcurrentInserts_SameId_OnlyOneRow()
+    {
+        Skip.IfNot(_fixture.Available, _fixture.SkipReason);
+
+        // CD-015 race coverage. The IF NOT EXISTS … INSERT pattern has a
+        // check-then-act window: two concurrent sessions can both pass the
+        // EXISTS check and both attempt the INSERT — the loser surfaces as a
+        // SqlException with Number 2601/2627. The site→central handoff is
+        // documented at-least-once with insert-if-not-exists, so this collision
+        // IS the expected contention mode. The race losers MUST be swallowed
+        // (not bubbled) so the site doesn't retry the same NotificationId
+        // forever. Final row count must be exactly 1; no exceptions thrown.
+        var id = Guid.NewGuid().ToString();
+
+        await Parallel.ForEachAsync(
+            Enumerable.Range(0, 50),
+            new ParallelOptions { MaxDegreeOfParallelism = 50 },
+            async (_, ct) =>
+            {
+                await using var context = CreateContext();
+                var repo = new NotificationOutboxRepository(context);
+                await repo.InsertIfNotExistsAsync(MakeNotification(id), ct);
+            });
+
+        await using var readContext = CreateContext();
+        var count = await readContext.Notifications
+            .Where(n => n.NotificationId == id)
+            .CountAsync();
+        Assert.Equal(1, count);
+    }
+
+    // --- helpers ------------------------------------------------------------
+
+    private ScadaLinkDbContext CreateContext()
+    {
+        var options = new DbContextOptionsBuilder<ScadaLinkDbContext>()
+            .UseSqlServer(_fixture.ConnectionString)
+            .Options;
+        return new ScadaLinkDbContext(options);
+    }
+
+    private static Notification MakeNotification(
+        string id,
+        NotificationStatus status = NotificationStatus.Pending,
+        string subject = "Subject")
+    {
+        return new Notification(
+            id,
+            NotificationType.Email,
+            "Ops List",
+            subject,
+            "Body",
+            "site-cd015")
+        {
+            Status = status,
+            CreatedAt = new DateTimeOffset(2026, 5, 20, 10, 0, 0, TimeSpan.Zero),
+            SiteEnqueuedAt = new DateTimeOffset(2026, 5, 20, 9, 59, 0, TimeSpan.Zero),
+        };
+    }
+}