feat(audit): flip AuditLog clustered key to (OccurredAtUtc, EventId) (ScadaBridge#111)

Variant B2 of the #78 benchmark: migration FlipAuditLogClusteredKey drops the
twelve non-clustered indexes, re-adds PK_AuditLog as (OccurredAtUtc, EventId)
on ps_AuditLog_Month, adds the aligned unique UX_AuditLog_EventId_Occurred
(EventId, OccurredAtUtc) for the idempotency probe, and rebuilds the twelve
indexes unchanged. Guarded (re-runnable) Up, inverse-rebuild Down; production
scripts docs/plans/sql/FlipAuditLogClusteredKey{,.rollback}.sql.

The partition-switch purge staging table now reads the live clustered key
order from sys.index_columns, so it switches either shape (old binaries
hard-code the old order: ship with the binaries, both centrals stopped).

Evidence: 300k rows, page 1 90,532 -> 34 logical reads (ordered backward
clustered scan under Top), keyset page 90,532 -> 46, probe 14 -> 14;
FlipAuditLogClusteredKeyMigrationTests pins indexes, grants, Down/Up, the
staging key (mutation: hard-coded old key fails SWITCH) and the read ratio.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV

src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Configurations/AuditLogEntityTypeConfiguration.cs

@@ -182,18 +182,28 @@ public class AuditLogEntityTypeConfiguration : IEntityTypeConfiguration<AuditLog
 
         // ── Keys + indexes ───────────────────────────────────────────────────
 
-        // Composite PK includes OccurredAtUtc — required by the monthly partition scheme
-        // (ps_AuditLog_Month) so the clustered key is partition-aligned. It is ALSO the
-        // only uniqueness enforcement the ingest path needs: EventId is a GUID minted
-        // once at the emitting site and never re-stamped, so a given EventId always
-        // arrives with the same OccurredAtUtc and can only ever land in one partition.
-        // Uniqueness of the pair is therefore uniqueness of EventId in practice, and the
-        // idempotency probe (WHERE EventId = @id) still seeks the clustered key's leading
-        // column. The predecessor non-aligned UX_AuditLog_EventId on [PRIMARY] was
-        // dropped by AlignAuditLogEventIdUniqueness — it existed only to give
-        // single-column uniqueness and its non-alignment forced an offline drop/rebuild
-        // around every partition-switch purge.
-        builder.HasKey(e => new { e.EventId, e.OccurredAtUtc });
+        // Clustered PK (OccurredAtUtc, EventId) — FlipAuditLogClusteredKey (ScadaBridge#111,
+        // variant B2 of the 2026-11-09 benchmark). Time-leading so the partitioned table can
+        // serve the Audit Log page's ORDER BY OccurredAtUtc DESC, EventId DESC TOP-N and its
+        // keyset cursor as an ordered backward range seek instead of reading the whole table
+        // (739k reads -> ~50 at 3M rows). OccurredAtUtc is also the partitioning column of
+        // ps_AuditLog_Month, so the key stays partition-aligned. Before the flip the key was
+        // (EventId, OccurredAtUtc); the pair - and therefore the uniqueness it enforces - is
+        // unchanged, only the column order (and so the physical row order) moved.
+        //
+        // EF does not manage the physical clustering of dbo.AuditLog (raw-SQL migrations own
+        // it); the key order here only keeps the model, snapshot and migration in agreement.
+        builder.HasKey(e => new { e.OccurredAtUtc, e.EventId });
+
+        // EventId-leading unique index (B2): keeps the ingest idempotency probe
+        // (WHERE EventId = @id) a seek - one shallow seek per partition, as before the flip -
+        // and keeps a real uniqueness constraint led by EventId. Partition-aligned (raw SQL
+        // ON ps_AuditLog_Month) so SWITCH PARTITION has nothing to object to. EventId is a GUID
+        // minted once at the emitting site together with OccurredAtUtc and never re-stamped,
+        // so pair-uniqueness is EventId-uniqueness in practice (see AlignAuditLogEventIdUniqueness).
+        builder.HasIndex(e => new { e.EventId, e.OccurredAtUtc })
+            .IsUnique()
+            .HasDatabaseName("UX_AuditLog_EventId_Occurred");
 
         // Index names are locked for reconciliation/migration discoverability. The
         // column SETS migrate to the canonical/computed shape (alog.md §4 semantics

src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Migrations/20261110033732_FlipAuditLogClusteredKey.cs

@@ -0,0 +1,130 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
+{
+    /// <summary>
+    /// Flips <c>dbo.AuditLog</c>'s clustered key from <c>(EventId, OccurredAtUtc)</c> to
+    /// <c>(OccurredAtUtc, EventId)</c> and adds the partition-aligned unique index
+    /// <c>UX_AuditLog_EventId_Occurred (EventId, OccurredAtUtc)</c> — variant B2 of the
+    /// 2026-11-09 clustered-key benchmark (ScadaBridge#78, #111;
+    /// <c>docs/plans/2026-11-09-auditlog-clustered-key-benchmark.md</c>).
+    /// </summary>
+    /// <remarks>
+    /// <para>
+    /// <b>Why.</b> With random-GUID EventIds leading the clustered key, the Audit Log page's
+    /// default query (<c>ORDER BY OccurredAtUtc DESC, EventId DESC</c>, newest 50) and its
+    /// keyset cursor could not be served as an ordered seek on the partitioned table and read
+    /// the whole table (739k logical reads at 3M rows); with the time-leading key the same
+    /// shapes read ~50 pages. The EventId-leading unique index keeps the ingest idempotency
+    /// probe (<c>WHERE EventId = @id</c>) a seek and keeps a uniqueness constraint led by
+    /// EventId. Ingest cost is unchanged (benchmark: same rows/s, same fragmentation).
+    /// </para>
+    /// <para>
+    /// <b>Shape of the rebuild.</b> Every non-clustered index carries the clustered key as its
+    /// row locator, so they are dropped first and recreated last (each is built once, not twice).
+    /// The clustered PK is dropped (the heap stays on <c>ps_AuditLog_Month</c>) and re-added in
+    /// the new order on the same scheme. All twelve non-clustered indexes are recreated with
+    /// their exact prior definitions, partition-aligned. This is an OFFLINE whole-table data
+    /// move: plan a write window and size it from the real table first (#111). Site buffers
+    /// retain unforwarded audit rows while central ingest is stopped.
+    /// </para>
+    /// <para>
+    /// <b>Grants.</b> An index-only rebuild keeps every table- and column-level permission on
+    /// <c>dbo.AuditLog</c> (the writer's DENY UPDATE/DELETE, the chainer's column UPDATE on
+    /// <c>ChainSeq, RowHash</c>, the purger's SELECT/DELETE) — pinned by
+    /// <c>FlipAuditLogClusteredKeyMigrationTests</c>.
+    /// </para>
+    /// <para>
+    /// <b>Mixed version.</b> Every query issued by old and new binaries names columns, never the
+    /// key, so both read and write either shape. The ONE key-sensitive statement is the
+    /// partition-switch purge's staging table, whose clustered key must match the table's: new
+    /// binaries read the live key order from <c>sys.index_columns</c> and build staging to match
+    /// (either shape), but OLD binaries hard-code <c>(EventId, OccurredAtUtc)</c>, so on a flipped
+    /// table their monthly switch-out fails (msg 4913/4947, rolled back, nothing purged, reported
+    /// as a purge failure and retried next tick). Ship this migration with the binaries.
+    /// </para>
+    /// <para>
+    /// <b>Re-runnable and reversible.</b> Up is a no-op when the clustered key already leads with
+    /// <c>OccurredAtUtc</c>; Down is the inverse rebuild (same cost) and a no-op when it already
+    /// leads with <c>EventId</c>.
+    /// </para>
+    /// </remarks>
+    public partial class FlipAuditLogClusteredKey : Migration
+    {
+        /// <summary>
+        /// The twelve non-clustered indexes, exactly as their creating migrations defined them
+        /// (AddAuditLogTable / CollapseAuditLogToCanonical / AddAuditLogAndNotificationCoveringIndexes /
+        /// ReviewRemediationIndexes / AddAuditLogHashChain), all on <c>ps_AuditLog_Month</c>.
+        /// </summary>
+        internal const string NonClusteredIndexDdl = @"
+CREATE NONCLUSTERED INDEX IX_AuditLog_OccurredAtUtc ON dbo.AuditLog (OccurredAtUtc DESC) INCLUDE (Status) ON ps_AuditLog_Month(OccurredAtUtc);
+CREATE NONCLUSTERED INDEX IX_AuditLog_Site_Occurred ON dbo.AuditLog (SourceSiteId ASC, OccurredAtUtc DESC) ON ps_AuditLog_Month(OccurredAtUtc);
+CREATE NONCLUSTERED INDEX IX_AuditLog_CorrelationId ON dbo.AuditLog (CorrelationId) WHERE CorrelationId IS NOT NULL ON ps_AuditLog_Month(OccurredAtUtc);
+CREATE NONCLUSTERED INDEX IX_AuditLog_Channel_Status_Occurred ON dbo.AuditLog (Category ASC, Status ASC, OccurredAtUtc DESC) ON ps_AuditLog_Month(OccurredAtUtc);
+CREATE NONCLUSTERED INDEX IX_AuditLog_Target_Occurred ON dbo.AuditLog (Target ASC, OccurredAtUtc DESC) WHERE Target IS NOT NULL ON ps_AuditLog_Month(OccurredAtUtc);
+CREATE NONCLUSTERED INDEX IX_AuditLog_Execution ON dbo.AuditLog (ExecutionId) ON ps_AuditLog_Month(OccurredAtUtc);
+CREATE NONCLUSTERED INDEX IX_AuditLog_ParentExecution ON dbo.AuditLog (ParentExecutionId) ON ps_AuditLog_Month(OccurredAtUtc);
+CREATE NONCLUSTERED INDEX IX_AuditLog_Node_Occurred ON dbo.AuditLog (SourceNode, OccurredAtUtc) ON ps_AuditLog_Month(OccurredAtUtc);
+CREATE NONCLUSTERED INDEX IX_AuditLog_Kind_Occurred ON dbo.AuditLog (Kind ASC, OccurredAtUtc DESC) ON ps_AuditLog_Month(OccurredAtUtc);
+CREATE NONCLUSTERED INDEX IX_AuditLog_Actor_Occurred ON dbo.AuditLog (Actor ASC, OccurredAtUtc DESC) ON ps_AuditLog_Month(OccurredAtUtc);
+CREATE NONCLUSTERED INDEX IX_AuditLog_Unchained ON dbo.AuditLog (OccurredAtUtc, EventId) WHERE ChainSeq IS NULL ON ps_AuditLog_Month(OccurredAtUtc);
+CREATE NONCLUSTERED INDEX IX_AuditLog_ChainSeq ON dbo.AuditLog (ChainSeq) WHERE ChainSeq IS NOT NULL ON ps_AuditLog_Month(OccurredAtUtc);";
+
+        /// <summary>Drops every non-clustered index on dbo.AuditLog (the twelve above plus the B2 unique index), each existence-guarded.</summary>
+        internal const string DropNonClusteredIndexesSql = @"
+DECLARE @dropSql nvarchar(max) = N'';
+SELECT @dropSql = @dropSql + N'DROP INDEX ' + QUOTENAME(i.name) + N' ON dbo.AuditLog;' + CHAR(10)
+FROM sys.indexes i
+WHERE i.object_id = OBJECT_ID('dbo.AuditLog') AND i.type = 2 AND i.name IN (
+    N'IX_AuditLog_OccurredAtUtc', N'IX_AuditLog_Site_Occurred', N'IX_AuditLog_CorrelationId',
+    N'IX_AuditLog_Channel_Status_Occurred', N'IX_AuditLog_Target_Occurred', N'IX_AuditLog_Execution',
+    N'IX_AuditLog_ParentExecution', N'IX_AuditLog_Node_Occurred', N'IX_AuditLog_Kind_Occurred',
+    N'IX_AuditLog_Actor_Occurred', N'IX_AuditLog_Unchained', N'IX_AuditLog_ChainSeq',
+    N'UX_AuditLog_EventId_Occurred');
+EXEC sp_executesql @dropSql;";
+
+        // Leading key column of the clustered index, or NULL when the table has none.
+        private const string LeadingKeyColumnSql = @"
+(SELECT c.name FROM sys.index_columns ic
+   JOIN sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id
+  WHERE ic.object_id = OBJECT_ID('dbo.AuditLog') AND ic.index_id = 1 AND ic.key_ordinal = 1)";
+
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            // Raw SQL, not the scaffolded DropPrimaryKey/AddPrimaryKey: EF's AddPrimaryKey cannot
+            // place the key ON ps_AuditLog_Month, and the twelve non-clustered indexes must be
+            // dropped first and rebuilt last. The whole body is one guarded batch so a re-run on
+            // an already-flipped table changes nothing.
+            migrationBuilder.Sql($@"
+IF {LeadingKeyColumnSql} = N'EventId'
+BEGIN
+{DropNonClusteredIndexesSql}
+    ALTER TABLE dbo.AuditLog DROP CONSTRAINT PK_AuditLog;
+    ALTER TABLE dbo.AuditLog ADD CONSTRAINT PK_AuditLog PRIMARY KEY CLUSTERED (OccurredAtUtc, EventId)
+        ON ps_AuditLog_Month(OccurredAtUtc);
+    CREATE UNIQUE NONCLUSTERED INDEX UX_AuditLog_EventId_Occurred ON dbo.AuditLog (EventId, OccurredAtUtc)
+        ON ps_AuditLog_Month(OccurredAtUtc);
+{NonClusteredIndexDdl}
+END");
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            // The inverse rebuild: back to (EventId, OccurredAtUtc), the B2 unique index removed
+            // (the clustered PK is again the only EventId-leading structure, as before).
+            migrationBuilder.Sql($@"
+IF {LeadingKeyColumnSql} = N'OccurredAtUtc'
+BEGIN
+{DropNonClusteredIndexesSql}
+    ALTER TABLE dbo.AuditLog DROP CONSTRAINT PK_AuditLog;
+    ALTER TABLE dbo.AuditLog ADD CONSTRAINT PK_AuditLog PRIMARY KEY CLUSTERED (EventId, OccurredAtUtc)
+        ON ps_AuditLog_Month(OccurredAtUtc);
+{NonClusteredIndexDdl}
+END");
+        }
+    }
+}

src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Migrations/ScadaBridgeDbContextModelSnapshot.cs

@@ -1979,12 +1979,12 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
 
             modelBuilder.Entity("ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Entities.AuditLogRow", b =>
                 {
-                    b.Property<Guid>("EventId")
-                        .HasColumnType("uniqueidentifier");
-
                     b.Property<DateTime>("OccurredAtUtc")
                         .HasColumnType("datetime2");
 
+                    b.Property<Guid>("EventId")
+                        .HasColumnType("uniqueidentifier");
+
                     b.Property<string>("Action")
                         .IsRequired()
                         .HasMaxLength(64)
@@ -2069,7 +2069,7 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
                         .HasMaxLength(256)
                         .HasColumnType("nvarchar(256)");
 
-                    b.HasKey("EventId", "OccurredAtUtc");
+                    b.HasKey("OccurredAtUtc", "EventId");
 
                     b.HasIndex("ChainSeq")
                         .HasDatabaseName("IX_AuditLog_ChainSeq")
@@ -2095,6 +2095,10 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
                         .IsDescending(false, true)
                         .HasDatabaseName("IX_AuditLog_Actor_Occurred");
 
+                    b.HasIndex("EventId", "OccurredAtUtc")
+                        .IsUnique()
+                        .HasDatabaseName("UX_AuditLog_EventId_Occurred");
+
                     b.HasIndex("Kind", "OccurredAtUtc")
                         .IsDescending(false, true)
                         .HasDatabaseName("IX_AuditLog_Kind_Occurred");

src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Repositories/AuditLogRepository.cs

@@ -23,7 +23,8 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
 public class AuditLogRepository : IAuditLogRepository
 {
     // SQL Server error numbers for duplicate-key violations on the
-    // partition-aligned clustered PK_AuditLog (EventId, OccurredAtUtc).
+    // partition-aligned clustered PK_AuditLog (OccurredAtUtc, EventId) or the unique
+    // UX_AuditLog_EventId_Occurred (EventId, OccurredAtUtc) — ScadaBridge#111.
     // 2601 is a unique-index violation; 2627 is a primary-key/unique-constraint
     // violation. The IF NOT EXISTS … INSERT pattern has a check-then-act race
     // window — two sessions can both pass the EXISTS check and then both attempt
@@ -504,8 +505,9 @@ VALUES
         // guarded and the staging table is GUID-suffixed, so a re-run is idempotent.
         //
         // ALIGNED UNIQUENESS: there is no longer an index drop/rebuild around
-        // the SWITCH. EventId uniqueness is now enforced solely by the clustered
-        // PK_AuditLog (EventId, OccurredAtUtc), which is partition-aligned on
+        // the SWITCH. EventId uniqueness is enforced by the clustered PK_AuditLog
+        // (OccurredAtUtc, EventId since FlipAuditLogClusteredKey, ScadaBridge#111) and the
+        // unique UX_AuditLog_EventId_Occurred, both partition-aligned on
         // ps_AuditLog_Month(OccurredAtUtc) — so ALTER TABLE … SWITCH PARTITION has no
         // non-aligned index to object to. The former dance dropped UX_AuditLog_EventId,
         // switched, then rebuilt it OFFLINE inside the same transaction: a whole-table
@@ -595,10 +597,21 @@ VALUES
                 ParentExecutionId  AS CAST(JSON_VALUE(DetailsJson,'$.parentExecutionId') AS uniqueidentifier) PERSISTED,
                 IngestedAtUtc      AS CAST(SWITCHOFFSET(CAST(JSON_VALUE(DetailsJson,'$.ingestedAtUtc') AS datetimeoffset), 0) AS datetime2(7)),
                 ChainSeq           bigint           NULL,
-                RowHash            binary(32)       NULL,
-                CONSTRAINT PK_{stagingTableName} PRIMARY KEY CLUSTERED (EventId, OccurredAtUtc)
+                RowHash            binary(32)       NULL
             ) ON [PRIMARY];
 
+            -- 2b. The staging clustered key must match dbo.AuditLog's or SWITCH refuses
+            --     (msg 4913/4947). FlipAuditLogClusteredKey (ScadaBridge#111) changed it from
+            --     (EventId, OccurredAtUtc) to (OccurredAtUtc, EventId), so read the live key
+            --     order instead of assuming one: this binary purges either shape, before or
+            --     after that migration (and after its Down).
+            IF (SELECT c.name FROM sys.index_columns ic
+                  JOIN sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id
+                 WHERE ic.object_id = OBJECT_ID('dbo.AuditLog') AND ic.index_id = 1 AND ic.key_ordinal = 1) = N'OccurredAtUtc'
+                ALTER TABLE dbo.[{stagingTableName}] ADD CONSTRAINT PK_{stagingTableName} PRIMARY KEY CLUSTERED (OccurredAtUtc, EventId);
+            ELSE
+                ALTER TABLE dbo.[{stagingTableName}] ADD CONSTRAINT PK_{stagingTableName} PRIMARY KEY CLUSTERED (EventId, OccurredAtUtc);
+
             -- 3. Switch the partition out. $partition.pf_AuditLog_Month returns
             --    the partition number that contains the supplied boundary value;
             --    SWITCH PARTITION N moves that partition's pages to the staging