fix: outbox KPI per-status legs, composite event-log indexes, AuditLog ingest FORCESEEK (#122, #123, #124)

#122: the Notification Outbox KPI snapshots ORed every status in one aggregation, which scanned the clustered index once the table held ~1 M Parked rows (0.7-2.6 s; Health dashboard up to 5 s). Each snapshot now runs three status-equality legs that seek their own index (~0.12 s); a cold Health load on the rig is 0.12-0.52 s. #123: site_events event_type/severity indexes are now composites with timestamp, so a filtered page is read in order and stops at the limit (severity=Error over 1 GB: 2,287 -> 15 ms cold, 115 -> 1.6 ms warm). Costs 454 -> 530 B/row, ~14 % fewer rows under the same cap. The schema apply drops the old indexes in place; the cap size model samples timestamp three times and re-scores inside its band with unchanged constants. #124: the set-based AuditLog insert's NOT EXISTS probe scanned the EventId index and concurrent writers deadlocked on S/IX page locks. WITH (FORCESEEK): 4 writers 775 rows/s + 76 deadlocks in 30 s -> 5,214 rows/s + 0. New concurrent-writer regression test fails 4/4 without the hint.
2026-11-10 16:46:42 -05:00
parent 2b80a6defa
commit 424db1555d
12 changed files with 386 additions and 158 deletions
@@ -635,6 +635,65 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
            rows.Select(r => r.EventId).OrderBy(g => g).ToArray());
    }

+    /// <summary>
+    /// Verifies that concurrent batch inserts do not deadlock each other (#124). Before the
+    /// probe was forced to seek, four writers of 256-row packets lost most packets to error
+    /// 1205 within seconds: the anti-semi-join scanned the EventId index and the writers
+    /// deadlocked on each other's S/IX page locks.
+    /// </summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [SkippableFact]
+    public async Task InsertManyIfNotExistsAsync_ConcurrentWriters_DoNotDeadlock()
+    {
+        Skip.IfNot(_fixture.Available, _fixture.SkipReason);
+
+        const int writers = 4, packetsPerWriter = 15, packetSize = 256, seedPackets = 40;
+        var siteId = NewSiteId();
+        var baseTime = new DateTime(2026, 5, 22, 8, 0, 0, DateTimeKind.Utc);
+
+        // Every packet spans the SAME time window, as live telemetry from several sites
+        // does, so the writers' rows interleave on the same clustered pages. The seed
+        // grows the index first: the deadlock rate rose with table size (bench, 4
+        // writers: 3 victims in 65 packets on an empty table, 17 in 52 at 5 k rows).
+        List<IReadOnlyList<AuditEvent>> Packets(int count, int offset) =>
+            Enumerable.Range(offset, count)
+                .Select(p => (IReadOnlyList<AuditEvent>)Enumerable.Range(0, packetSize)
+                    .Select(i => NewEvent(siteId, occurredAtUtc: baseTime.AddMilliseconds(i).AddTicks(p)))
+                    .ToList())
+                .ToList();
+
+        foreach (var seed in Packets(seedPackets, 0))
+        {
+            await using var context = CreateContext();
+            await new AuditLogRepository(context).InsertManyIfNotExistsAsync(seed);
+        }
+
+        var packets = Packets(writers * packetsPerWriter, seedPackets);
+
+        var deadlocks = 0;
+        await Task.WhenAll(Enumerable.Range(0, writers).Select(w => Task.Run(async () =>
+        {
+            for (var p = w; p < packets.Count; p += writers)
+            {
+                try
+                {
+                    await using var context = CreateContext();
+                    await new AuditLogRepository(context).InsertManyIfNotExistsAsync(packets[p]);
+                }
+                catch (SqlException ex) when (ex.Number == 1205)
+                {
+                    Interlocked.Increment(ref deadlocks);
+                }
+            }
+        })));
+
+        Assert.Equal(0, deadlocks);
+
+        await using var readContext = CreateContext();
+        var count = await readContext.Set<AuditLogRow>().CountAsync(e => e.SourceSiteId == siteId);
+        Assert.Equal((seedPackets + (writers * packetsPerWriter)) * packetSize, count);
+    }
+
    /// <summary>Verifies that duplicate event ids within one batch collapse to a single row.</summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [SkippableFact]