feat(audit): flip AuditLog clustered key to (OccurredAtUtc, EventId) (ScadaBridge#111)
Variant B2 of the #78 benchmark: migration FlipAuditLogClusteredKey drops the twelve non-clustered indexes, re-adds PK_AuditLog as (OccurredAtUtc, EventId) on ps_AuditLog_Month, adds the aligned unique UX_AuditLog_EventId_Occurred (EventId, OccurredAtUtc) for the idempotency probe, and rebuilds the twelve indexes unchanged. Guarded (re-runnable) Up, inverse-rebuild Down; production scripts docs/plans/sql/FlipAuditLogClusteredKey{,.rollback}.sql. The partition-switch purge staging table now reads the live clustered key order from sys.index_columns, so it switches either shape (old binaries hard-code the old order: ship with the binaries, both centrals stopped). Evidence: 300k rows, page 1 90,532 -> 34 logical reads (ordered backward clustered scan under Top), keyset page 90,532 -> 46, probe 14 -> 14; FlipAuditLogClusteredKeyMigrationTests pins indexes, grants, Down/Up, the staging key (mutation: hard-coded old key fails SWITCH) and the read ratio. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
@@ -118,11 +118,14 @@ public interface IAuditLogRepository
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Partition-aligned uniqueness — no index drop.</b> EventId uniqueness is
|
||||
/// enforced by the clustered <c>PK_AuditLog (EventId, OccurredAtUtc)</c>,
|
||||
/// which is aligned on <c>ps_AuditLog_Month(OccurredAtUtc)</c>, so
|
||||
/// enforced by the clustered <c>PK_AuditLog (OccurredAtUtc, EventId)</c> and the
|
||||
/// unique <c>UX_AuditLog_EventId_Occurred (EventId, OccurredAtUtc)</c> (ScadaBridge#111;
|
||||
/// before that migration the PK alone, keyed <c>(EventId, OccurredAtUtc)</c>), all
|
||||
/// aligned on <c>ps_AuditLog_Month(OccurredAtUtc)</c>, so
|
||||
/// <c>ALTER TABLE … SWITCH PARTITION</c> has no non-aligned unique index to
|
||||
/// object to. The implementation creates a staging table with byte-identical
|
||||
/// schema, switches the partition's data into staging, and drops staging
|
||||
/// schema and the same clustered key order as the live table (read at run time, so
|
||||
/// either key shape purges), switches the partition's data into staging, and drops staging
|
||||
/// (discarding the rows). Nothing is dropped from the live table, so there is
|
||||
/// no window in which the idempotency enforcement is absent and no offline
|
||||
/// index rebuild inside the switch transaction.
|
||||
|
||||
+22
-12
@@ -182,18 +182,28 @@ public class AuditLogEntityTypeConfiguration : IEntityTypeConfiguration<AuditLog
|
||||
|
||||
// ── Keys + indexes ───────────────────────────────────────────────────
|
||||
|
||||
// Composite PK includes OccurredAtUtc — required by the monthly partition scheme
|
||||
// (ps_AuditLog_Month) so the clustered key is partition-aligned. It is ALSO the
|
||||
// only uniqueness enforcement the ingest path needs: EventId is a GUID minted
|
||||
// once at the emitting site and never re-stamped, so a given EventId always
|
||||
// arrives with the same OccurredAtUtc and can only ever land in one partition.
|
||||
// Uniqueness of the pair is therefore uniqueness of EventId in practice, and the
|
||||
// idempotency probe (WHERE EventId = @id) still seeks the clustered key's leading
|
||||
// column. The predecessor non-aligned UX_AuditLog_EventId on [PRIMARY] was
|
||||
// dropped by AlignAuditLogEventIdUniqueness — it existed only to give
|
||||
// single-column uniqueness and its non-alignment forced an offline drop/rebuild
|
||||
// around every partition-switch purge.
|
||||
builder.HasKey(e => new { e.EventId, e.OccurredAtUtc });
|
||||
// Clustered PK (OccurredAtUtc, EventId) — FlipAuditLogClusteredKey (ScadaBridge#111,
|
||||
// variant B2 of the 2026-11-09 benchmark). Time-leading so the partitioned table can
|
||||
// serve the Audit Log page's ORDER BY OccurredAtUtc DESC, EventId DESC TOP-N and its
|
||||
// keyset cursor as an ordered backward range seek instead of reading the whole table
|
||||
// (739k reads -> ~50 at 3M rows). OccurredAtUtc is also the partitioning column of
|
||||
// ps_AuditLog_Month, so the key stays partition-aligned. Before the flip the key was
|
||||
// (EventId, OccurredAtUtc); the pair - and therefore the uniqueness it enforces - is
|
||||
// unchanged, only the column order (and so the physical row order) moved.
|
||||
//
|
||||
// EF does not manage the physical clustering of dbo.AuditLog (raw-SQL migrations own
|
||||
// it); the key order here only keeps the model, snapshot and migration in agreement.
|
||||
builder.HasKey(e => new { e.OccurredAtUtc, e.EventId });
|
||||
|
||||
// EventId-leading unique index (B2): keeps the ingest idempotency probe
|
||||
// (WHERE EventId = @id) a seek - one shallow seek per partition, as before the flip -
|
||||
// and keeps a real uniqueness constraint led by EventId. Partition-aligned (raw SQL
|
||||
// ON ps_AuditLog_Month) so SWITCH PARTITION has nothing to object to. EventId is a GUID
|
||||
// minted once at the emitting site together with OccurredAtUtc and never re-stamped,
|
||||
// so pair-uniqueness is EventId-uniqueness in practice (see AlignAuditLogEventIdUniqueness).
|
||||
builder.HasIndex(e => new { e.EventId, e.OccurredAtUtc })
|
||||
.IsUnique()
|
||||
.HasDatabaseName("UX_AuditLog_EventId_Occurred");
|
||||
|
||||
// Index names are locked for reconciliation/migration discoverability. The
|
||||
// column SETS migrate to the canonical/computed shape (alog.md §4 semantics
|
||||
|
||||
+2385
File diff suppressed because it is too large
Load Diff
+130
@@ -0,0 +1,130 @@
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
|
||||
{
|
||||
/// <summary>
|
||||
/// Flips <c>dbo.AuditLog</c>'s clustered key from <c>(EventId, OccurredAtUtc)</c> to
|
||||
/// <c>(OccurredAtUtc, EventId)</c> and adds the partition-aligned unique index
|
||||
/// <c>UX_AuditLog_EventId_Occurred (EventId, OccurredAtUtc)</c> — variant B2 of the
|
||||
/// 2026-11-09 clustered-key benchmark (ScadaBridge#78, #111;
|
||||
/// <c>docs/plans/2026-11-09-auditlog-clustered-key-benchmark.md</c>).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Why.</b> With random-GUID EventIds leading the clustered key, the Audit Log page's
|
||||
/// default query (<c>ORDER BY OccurredAtUtc DESC, EventId DESC</c>, newest 50) and its
|
||||
/// keyset cursor could not be served as an ordered seek on the partitioned table and read
|
||||
/// the whole table (739k logical reads at 3M rows); with the time-leading key the same
|
||||
/// shapes read ~50 pages. The EventId-leading unique index keeps the ingest idempotency
|
||||
/// probe (<c>WHERE EventId = @id</c>) a seek and keeps a uniqueness constraint led by
|
||||
/// EventId. Ingest cost is unchanged (benchmark: same rows/s, same fragmentation).
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Shape of the rebuild.</b> Every non-clustered index carries the clustered key as its
|
||||
/// row locator, so they are dropped first and recreated last (each is built once, not twice).
|
||||
/// The clustered PK is dropped (the heap stays on <c>ps_AuditLog_Month</c>) and re-added in
|
||||
/// the new order on the same scheme. All twelve non-clustered indexes are recreated with
|
||||
/// their exact prior definitions, partition-aligned. This is an OFFLINE whole-table data
|
||||
/// move: plan a write window and size it from the real table first (#111). Site buffers
|
||||
/// retain unforwarded audit rows while central ingest is stopped.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Grants.</b> An index-only rebuild keeps every table- and column-level permission on
|
||||
/// <c>dbo.AuditLog</c> (the writer's DENY UPDATE/DELETE, the chainer's column UPDATE on
|
||||
/// <c>ChainSeq, RowHash</c>, the purger's SELECT/DELETE) — pinned by
|
||||
/// <c>FlipAuditLogClusteredKeyMigrationTests</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Mixed version.</b> Every query issued by old and new binaries names columns, never the
|
||||
/// key, so both read and write either shape. The ONE key-sensitive statement is the
|
||||
/// partition-switch purge's staging table, whose clustered key must match the table's: new
|
||||
/// binaries read the live key order from <c>sys.index_columns</c> and build staging to match
|
||||
/// (either shape), but OLD binaries hard-code <c>(EventId, OccurredAtUtc)</c>, so on a flipped
|
||||
/// table their monthly switch-out fails (msg 4913/4947, rolled back, nothing purged, reported
|
||||
/// as a purge failure and retried next tick). Ship this migration with the binaries.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Re-runnable and reversible.</b> Up is a no-op when the clustered key already leads with
|
||||
/// <c>OccurredAtUtc</c>; Down is the inverse rebuild (same cost) and a no-op when it already
|
||||
/// leads with <c>EventId</c>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public partial class FlipAuditLogClusteredKey : Migration
|
||||
{
|
||||
/// <summary>
|
||||
/// The twelve non-clustered indexes, exactly as their creating migrations defined them
|
||||
/// (AddAuditLogTable / CollapseAuditLogToCanonical / AddAuditLogAndNotificationCoveringIndexes /
|
||||
/// ReviewRemediationIndexes / AddAuditLogHashChain), all on <c>ps_AuditLog_Month</c>.
|
||||
/// </summary>
|
||||
internal const string NonClusteredIndexDdl = @"
|
||||
CREATE NONCLUSTERED INDEX IX_AuditLog_OccurredAtUtc ON dbo.AuditLog (OccurredAtUtc DESC) INCLUDE (Status) ON ps_AuditLog_Month(OccurredAtUtc);
|
||||
CREATE NONCLUSTERED INDEX IX_AuditLog_Site_Occurred ON dbo.AuditLog (SourceSiteId ASC, OccurredAtUtc DESC) ON ps_AuditLog_Month(OccurredAtUtc);
|
||||
CREATE NONCLUSTERED INDEX IX_AuditLog_CorrelationId ON dbo.AuditLog (CorrelationId) WHERE CorrelationId IS NOT NULL ON ps_AuditLog_Month(OccurredAtUtc);
|
||||
CREATE NONCLUSTERED INDEX IX_AuditLog_Channel_Status_Occurred ON dbo.AuditLog (Category ASC, Status ASC, OccurredAtUtc DESC) ON ps_AuditLog_Month(OccurredAtUtc);
|
||||
CREATE NONCLUSTERED INDEX IX_AuditLog_Target_Occurred ON dbo.AuditLog (Target ASC, OccurredAtUtc DESC) WHERE Target IS NOT NULL ON ps_AuditLog_Month(OccurredAtUtc);
|
||||
CREATE NONCLUSTERED INDEX IX_AuditLog_Execution ON dbo.AuditLog (ExecutionId) ON ps_AuditLog_Month(OccurredAtUtc);
|
||||
CREATE NONCLUSTERED INDEX IX_AuditLog_ParentExecution ON dbo.AuditLog (ParentExecutionId) ON ps_AuditLog_Month(OccurredAtUtc);
|
||||
CREATE NONCLUSTERED INDEX IX_AuditLog_Node_Occurred ON dbo.AuditLog (SourceNode, OccurredAtUtc) ON ps_AuditLog_Month(OccurredAtUtc);
|
||||
CREATE NONCLUSTERED INDEX IX_AuditLog_Kind_Occurred ON dbo.AuditLog (Kind ASC, OccurredAtUtc DESC) ON ps_AuditLog_Month(OccurredAtUtc);
|
||||
CREATE NONCLUSTERED INDEX IX_AuditLog_Actor_Occurred ON dbo.AuditLog (Actor ASC, OccurredAtUtc DESC) ON ps_AuditLog_Month(OccurredAtUtc);
|
||||
CREATE NONCLUSTERED INDEX IX_AuditLog_Unchained ON dbo.AuditLog (OccurredAtUtc, EventId) WHERE ChainSeq IS NULL ON ps_AuditLog_Month(OccurredAtUtc);
|
||||
CREATE NONCLUSTERED INDEX IX_AuditLog_ChainSeq ON dbo.AuditLog (ChainSeq) WHERE ChainSeq IS NOT NULL ON ps_AuditLog_Month(OccurredAtUtc);";
|
||||
|
||||
/// <summary>Drops every non-clustered index on dbo.AuditLog (the twelve above plus the B2 unique index), each existence-guarded.</summary>
|
||||
internal const string DropNonClusteredIndexesSql = @"
|
||||
DECLARE @dropSql nvarchar(max) = N'';
|
||||
SELECT @dropSql = @dropSql + N'DROP INDEX ' + QUOTENAME(i.name) + N' ON dbo.AuditLog;' + CHAR(10)
|
||||
FROM sys.indexes i
|
||||
WHERE i.object_id = OBJECT_ID('dbo.AuditLog') AND i.type = 2 AND i.name IN (
|
||||
N'IX_AuditLog_OccurredAtUtc', N'IX_AuditLog_Site_Occurred', N'IX_AuditLog_CorrelationId',
|
||||
N'IX_AuditLog_Channel_Status_Occurred', N'IX_AuditLog_Target_Occurred', N'IX_AuditLog_Execution',
|
||||
N'IX_AuditLog_ParentExecution', N'IX_AuditLog_Node_Occurred', N'IX_AuditLog_Kind_Occurred',
|
||||
N'IX_AuditLog_Actor_Occurred', N'IX_AuditLog_Unchained', N'IX_AuditLog_ChainSeq',
|
||||
N'UX_AuditLog_EventId_Occurred');
|
||||
EXEC sp_executesql @dropSql;";
|
||||
|
||||
// Leading key column of the clustered index, or NULL when the table has none.
|
||||
private const string LeadingKeyColumnSql = @"
|
||||
(SELECT c.name FROM sys.index_columns ic
|
||||
JOIN sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id
|
||||
WHERE ic.object_id = OBJECT_ID('dbo.AuditLog') AND ic.index_id = 1 AND ic.key_ordinal = 1)";
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
// Raw SQL, not the scaffolded DropPrimaryKey/AddPrimaryKey: EF's AddPrimaryKey cannot
|
||||
// place the key ON ps_AuditLog_Month, and the twelve non-clustered indexes must be
|
||||
// dropped first and rebuilt last. The whole body is one guarded batch so a re-run on
|
||||
// an already-flipped table changes nothing.
|
||||
migrationBuilder.Sql($@"
|
||||
IF {LeadingKeyColumnSql} = N'EventId'
|
||||
BEGIN
|
||||
{DropNonClusteredIndexesSql}
|
||||
ALTER TABLE dbo.AuditLog DROP CONSTRAINT PK_AuditLog;
|
||||
ALTER TABLE dbo.AuditLog ADD CONSTRAINT PK_AuditLog PRIMARY KEY CLUSTERED (OccurredAtUtc, EventId)
|
||||
ON ps_AuditLog_Month(OccurredAtUtc);
|
||||
CREATE UNIQUE NONCLUSTERED INDEX UX_AuditLog_EventId_Occurred ON dbo.AuditLog (EventId, OccurredAtUtc)
|
||||
ON ps_AuditLog_Month(OccurredAtUtc);
|
||||
{NonClusteredIndexDdl}
|
||||
END");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
// The inverse rebuild: back to (EventId, OccurredAtUtc), the B2 unique index removed
|
||||
// (the clustered PK is again the only EventId-leading structure, as before).
|
||||
migrationBuilder.Sql($@"
|
||||
IF {LeadingKeyColumnSql} = N'OccurredAtUtc'
|
||||
BEGIN
|
||||
{DropNonClusteredIndexesSql}
|
||||
ALTER TABLE dbo.AuditLog DROP CONSTRAINT PK_AuditLog;
|
||||
ALTER TABLE dbo.AuditLog ADD CONSTRAINT PK_AuditLog PRIMARY KEY CLUSTERED (EventId, OccurredAtUtc)
|
||||
ON ps_AuditLog_Month(OccurredAtUtc);
|
||||
{NonClusteredIndexDdl}
|
||||
END");
|
||||
}
|
||||
}
|
||||
}
|
||||
+8
-4
@@ -1979,12 +1979,12 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
|
||||
|
||||
modelBuilder.Entity("ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Entities.AuditLogRow", b =>
|
||||
{
|
||||
b.Property<Guid>("EventId")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<DateTime>("OccurredAtUtc")
|
||||
.HasColumnType("datetime2");
|
||||
|
||||
b.Property<Guid>("EventId")
|
||||
.HasColumnType("uniqueidentifier");
|
||||
|
||||
b.Property<string>("Action")
|
||||
.IsRequired()
|
||||
.HasMaxLength(64)
|
||||
@@ -2069,7 +2069,7 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
|
||||
.HasMaxLength(256)
|
||||
.HasColumnType("nvarchar(256)");
|
||||
|
||||
b.HasKey("EventId", "OccurredAtUtc");
|
||||
b.HasKey("OccurredAtUtc", "EventId");
|
||||
|
||||
b.HasIndex("ChainSeq")
|
||||
.HasDatabaseName("IX_AuditLog_ChainSeq")
|
||||
@@ -2095,6 +2095,10 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
|
||||
.IsDescending(false, true)
|
||||
.HasDatabaseName("IX_AuditLog_Actor_Occurred");
|
||||
|
||||
b.HasIndex("EventId", "OccurredAtUtc")
|
||||
.IsUnique()
|
||||
.HasDatabaseName("UX_AuditLog_EventId_Occurred");
|
||||
|
||||
b.HasIndex("Kind", "OccurredAtUtc")
|
||||
.IsDescending(false, true)
|
||||
.HasDatabaseName("IX_AuditLog_Kind_Occurred");
|
||||
|
||||
@@ -23,7 +23,8 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
public class AuditLogRepository : IAuditLogRepository
|
||||
{
|
||||
// SQL Server error numbers for duplicate-key violations on the
|
||||
// partition-aligned clustered PK_AuditLog (EventId, OccurredAtUtc).
|
||||
// partition-aligned clustered PK_AuditLog (OccurredAtUtc, EventId) or the unique
|
||||
// UX_AuditLog_EventId_Occurred (EventId, OccurredAtUtc) — ScadaBridge#111.
|
||||
// 2601 is a unique-index violation; 2627 is a primary-key/unique-constraint
|
||||
// violation. The IF NOT EXISTS … INSERT pattern has a check-then-act race
|
||||
// window — two sessions can both pass the EXISTS check and then both attempt
|
||||
@@ -504,8 +505,9 @@ VALUES
|
||||
// guarded and the staging table is GUID-suffixed, so a re-run is idempotent.
|
||||
//
|
||||
// ALIGNED UNIQUENESS: there is no longer an index drop/rebuild around
|
||||
// the SWITCH. EventId uniqueness is now enforced solely by the clustered
|
||||
// PK_AuditLog (EventId, OccurredAtUtc), which is partition-aligned on
|
||||
// the SWITCH. EventId uniqueness is enforced by the clustered PK_AuditLog
|
||||
// (OccurredAtUtc, EventId since FlipAuditLogClusteredKey, ScadaBridge#111) and the
|
||||
// unique UX_AuditLog_EventId_Occurred, both partition-aligned on
|
||||
// ps_AuditLog_Month(OccurredAtUtc) — so ALTER TABLE … SWITCH PARTITION has no
|
||||
// non-aligned index to object to. The former dance dropped UX_AuditLog_EventId,
|
||||
// switched, then rebuilt it OFFLINE inside the same transaction: a whole-table
|
||||
@@ -595,10 +597,21 @@ VALUES
|
||||
ParentExecutionId AS CAST(JSON_VALUE(DetailsJson,'$.parentExecutionId') AS uniqueidentifier) PERSISTED,
|
||||
IngestedAtUtc AS CAST(SWITCHOFFSET(CAST(JSON_VALUE(DetailsJson,'$.ingestedAtUtc') AS datetimeoffset), 0) AS datetime2(7)),
|
||||
ChainSeq bigint NULL,
|
||||
RowHash binary(32) NULL,
|
||||
CONSTRAINT PK_{stagingTableName} PRIMARY KEY CLUSTERED (EventId, OccurredAtUtc)
|
||||
RowHash binary(32) NULL
|
||||
) ON [PRIMARY];
|
||||
|
||||
-- 2b. The staging clustered key must match dbo.AuditLog's or SWITCH refuses
|
||||
-- (msg 4913/4947). FlipAuditLogClusteredKey (ScadaBridge#111) changed it from
|
||||
-- (EventId, OccurredAtUtc) to (OccurredAtUtc, EventId), so read the live key
|
||||
-- order instead of assuming one: this binary purges either shape, before or
|
||||
-- after that migration (and after its Down).
|
||||
IF (SELECT c.name FROM sys.index_columns ic
|
||||
JOIN sys.columns c ON c.object_id = ic.object_id AND c.column_id = ic.column_id
|
||||
WHERE ic.object_id = OBJECT_ID('dbo.AuditLog') AND ic.index_id = 1 AND ic.key_ordinal = 1) = N'OccurredAtUtc'
|
||||
ALTER TABLE dbo.[{stagingTableName}] ADD CONSTRAINT PK_{stagingTableName} PRIMARY KEY CLUSTERED (OccurredAtUtc, EventId);
|
||||
ELSE
|
||||
ALTER TABLE dbo.[{stagingTableName}] ADD CONSTRAINT PK_{stagingTableName} PRIMARY KEY CLUSTERED (EventId, OccurredAtUtc);
|
||||
|
||||
-- 3. Switch the partition out. $partition.pf_AuditLog_Month returns
|
||||
-- the partition number that contains the supplied boundary value;
|
||||
-- SWITCH PARTITION N moves that partition's pages to the staging
|
||||
|
||||
Reference in New Issue
Block a user