dohertj2/ScadaBridge
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(audit)!: ScadaBridge C5 — collapse central dbo.AuditLog to 10 canonical cols + persisted computed cols; CollapseAuditLogToCanonical migration; repo writes canonical directly (Task 2.5)

Browse Source
This commit is contained in:
Joseph Doherty
2026-06-02 14:06:46 -04:00
parent 1737d15f04
commit 68a6bd1720
12 changed files with 2592 additions and 440 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Migrations/20260602174346_CollapseAuditLogToCanonical.Designer.cs
Generated
+1724
View File
File diff suppressed because it is too large Load Diff
src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Migrations/20260602174346_CollapseAuditLogToCanonical.cs
+244
View File
@@ -0,0 +1,244 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
{
/// <summary>
/// C5 of the ScadaBridge audit re-architecture (Task 2.5): collapses the central
/// <c>dbo.AuditLog</c> table from the transitional 24 typed columns to the 10
/// canonical <c>ZB.MOM.WW.Audit.AuditEvent</c> columns plus six read-only,
/// server-side <b>persisted computed columns</b> derived from <c>DetailsJson</c>
/// (<c>JSON_VALUE</c> … <c>PERSISTED</c>) that back the indexed reporting queries.
/// </summary>
/// <remarks>
/// <para>
/// <b>Why new-table + copy (not in-place ALTER).</b> An in-place collapse is
/// infeasible: the partition-aligned indexes are keyed on columns that are being
/// dropped; the purge path (<c>SwitchOutPartitionAsync</c>) hard-codes a
/// byte-identical staging column list; and dropping <c>nvarchar(max)</c> columns
/// per partition is expensive. Instead this migration builds a fresh
/// <c>dbo.AuditLog_v2</c> on the SAME preserved partition scheme
/// (<c>ps_AuditLog_Month(OccurredAtUtc)</c>), copies every row with a one-way
/// projection of the old typed columns into canonical columns + <c>DetailsJson</c>,
/// drops the old table, renames v2 into place, and re-grants the append-only roles.
/// </para>
/// <para>
/// <b>Down() is a documented ONE-WAY.</b> The projection of ~17 typed columns into
/// a single JSON bag is lossy to reverse byte-for-byte (e.g. the codec's
/// null-omission + key order, the Action/Category/Outcome derivation), so the
/// reverse is NOT implemented. <see cref="Down"/> throws
/// <see cref="System.NotSupportedException"/> with guidance to restore from backup.
/// </para>
/// <para>
/// <b>Partition / SWITCH / computed-column interaction.</b> The new table and its
/// non-clustered indexes are created directly on <c>ps_AuditLog_Month</c> so the
/// partition-switch purge keeps working; the non-aligned <c>UX_AuditLog_EventId</c>
/// stays on <c>[PRIMARY]</c> exactly as before. The persisted computed columns are
/// part of the table's storage, so the staging table used by
/// <c>SwitchOutPartitionAsync</c> MUST declare them with identical expressions —
/// see that method (kept in sync with the v2 DDL below).
/// </para>
/// </remarks>
public partial class CollapseAuditLogToCanonical : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
// 1) Create dbo.AuditLog_v2 on the PRESERVED partition scheme. Ten
// canonical columns first (ordinal-stable), then the six persisted
// computed columns. The clustered PK is composite {EventId,
// OccurredAtUtc} (partition-aligned). The computed-column expressions
// here are the single source of truth that AuditLogEntityTypeConfiguration
// and SwitchOutPartitionAsync's staging table must mirror exactly.
migrationBuilder.Sql(@"
CREATE TABLE dbo.AuditLog_v2 (
EventId uniqueidentifier NOT NULL,
OccurredAtUtc datetime2(7) NOT NULL,
Actor nvarchar(256) NULL,
Action varchar(64) NOT NULL,
Outcome varchar(16) NOT NULL,
Category varchar(32) NOT NULL,
Target nvarchar(256) NULL,
SourceNode varchar(64) NULL,
CorrelationId uniqueidentifier NULL,
DetailsJson nvarchar(max) NULL,
Kind AS JSON_VALUE(DetailsJson,'$.kind') PERSISTED,
Status AS JSON_VALUE(DetailsJson,'$.status') PERSISTED,
SourceSiteId AS JSON_VALUE(DetailsJson,'$.sourceSiteId') PERSISTED,
ExecutionId AS CAST(JSON_VALUE(DetailsJson,'$.executionId') AS uniqueidentifier) PERSISTED,
ParentExecutionId AS CAST(JSON_VALUE(DetailsJson,'$.parentExecutionId') AS uniqueidentifier) PERSISTED,
-- IngestedAtUtc is NOT persisted: the datetimeoffset cast / SWITCHOFFSET is
-- non-deterministic and SQL Server rejects a PERSISTED non-deterministic
-- computed column. It is not indexed, so non-persistence costs nothing.
IngestedAtUtc AS CAST(SWITCHOFFSET(CAST(JSON_VALUE(DetailsJson,'$.ingestedAtUtc') AS datetimeoffset), 0) AS datetime2(7)),
CONSTRAINT PK_AuditLog_v2 PRIMARY KEY CLUSTERED (EventId, OccurredAtUtc)
ON ps_AuditLog_Month(OccurredAtUtc)
) ON ps_AuditLog_Month(OccurredAtUtc);");
// 2) Data copy: project the old 24-column rows into the canonical shape.
// - Action = Channel + '.' + Kind (matches AuditFieldBuilders.BuildAction)
// - Category = Channel (matches AuditFieldBuilders.BuildCategory)
// - Outcome = AuditOutcomeProjector.Project(Status, Kind):
// Kind='InboundAuthFailure' -> 'Denied' (wins over any status);
// Status in ('Failed','Parked','Discarded') -> 'Failure'; else 'Success'.
// - Actor : empty string maps to NULL (canonical Actor is non-null,
// but the old column stored NULL for system/anon — keep NULL).
// - DetailsJson: every domain field re-serialised as a single JSON object
// with camelCase keys matching AuditDetailsCodec. FOR JSON
// PATH, WITHOUT_ARRAY_WRAPPER OMITS null keys by default
// (no INCLUDE_NULL_VALUES), matching the codec's
// JsonIgnoreCondition.WhenWritingNull; channel/kind/status
// are NOT NULL in the legacy table so they are always present
// and emit the enum-name strings the computed Kind/Status
// (and Channel-as-Category) derive from. payloadTruncated is a
// non-null bit so it is always written, matching the codec
// (which always writes the bool).
// The six computed columns auto-derive from DetailsJson on INSERT.
migrationBuilder.Sql(@"
INSERT INTO dbo.AuditLog_v2
(EventId, OccurredAtUtc, Actor, Action, Outcome, Category, Target, SourceNode, CorrelationId, DetailsJson)
SELECT
a.EventId,
a.OccurredAtUtc,
NULLIF(a.Actor, '') AS Actor,
a.Channel + '.' + a.Kind AS Action,
CASE
WHEN a.Kind = 'InboundAuthFailure' THEN 'Denied'
WHEN a.Status IN ('Failed','Parked','Discarded') THEN 'Failure'
ELSE 'Success'
END AS Outcome,
a.Channel AS Category,
a.Target,
a.SourceNode,
a.CorrelationId,
(
SELECT
a.Channel AS channel,
a.Kind AS kind,
a.Status AS status,
a.ExecutionId AS executionId,
a.ParentExecutionId AS parentExecutionId,
a.SourceSiteId AS sourceSiteId,
a.SourceInstanceId AS sourceInstanceId,
a.SourceScript AS sourceScript,
a.HttpStatus AS httpStatus,
a.DurationMs AS durationMs,
a.ErrorMessage AS errorMessage,
a.ErrorDetail AS errorDetail,
a.RequestSummary AS requestSummary,
a.ResponseSummary AS responseSummary,
CAST(a.PayloadTruncated AS bit) AS payloadTruncated,
a.Extra AS extra,
CASE WHEN a.IngestedAtUtc IS NULL THEN NULL
ELSE CONVERT(varchar(33),
CAST(a.IngestedAtUtc AS datetimeoffset(7)) AT TIME ZONE 'UTC',
126)
END AS ingestedAtUtc
FOR JSON PATH, WITHOUT_ARRAY_WRAPPER
) AS DetailsJson
FROM dbo.AuditLog a;");
// 3) Drop the old table and rename v2 into place. The table rename uses
// the 'schema.table' form; the PK constraint rename uses the BARE
// constraint name with @objtype='OBJECT' (a constraint is a schema-
// scoped object, NOT table-qualified — a 'dbo.AuditLog.PK_…' form is
// rejected as ambiguous). The non-clustered indexes were dropped with
// the old table; they are recreated by name in step 4 (no index rename
// needed because the v2 table was created without them).
migrationBuilder.Sql(@"
DROP TABLE dbo.AuditLog;
EXEC sp_rename 'dbo.AuditLog_v2', 'AuditLog';
EXEC sp_rename 'PK_AuditLog_v2', 'PK_AuditLog', 'OBJECT';");
// 4) Recreate the reconciliation/query indexes on the new shape, names
// preserved (alog.md §4 semantics): Channel→Category, Site/Node read
// off the canonical/computed columns. All non-clustered indexes are
// partition-aligned on ps_AuditLog_Month(OccurredAtUtc) so the
// partition-switch purge keeps touching a single partition. The
// UX_AuditLog_EventId unique index is INTENTIONALLY non-aligned (on
// [PRIMARY]) to give single-column EventId uniqueness for
// InsertIfNotExistsAsync idempotency.
migrationBuilder.Sql(@"
CREATE NONCLUSTERED INDEX IX_AuditLog_OccurredAtUtc
ON dbo.AuditLog (OccurredAtUtc DESC)
ON ps_AuditLog_Month(OccurredAtUtc);
CREATE NONCLUSTERED INDEX IX_AuditLog_Site_Occurred
ON dbo.AuditLog (SourceSiteId ASC, OccurredAtUtc DESC)
ON ps_AuditLog_Month(OccurredAtUtc);
CREATE NONCLUSTERED INDEX IX_AuditLog_CorrelationId
ON dbo.AuditLog (CorrelationId)
WHERE CorrelationId IS NOT NULL
ON ps_AuditLog_Month(OccurredAtUtc);
CREATE NONCLUSTERED INDEX IX_AuditLog_Channel_Status_Occurred
ON dbo.AuditLog (Category ASC, Status ASC, OccurredAtUtc DESC)
ON ps_AuditLog_Month(OccurredAtUtc);
CREATE NONCLUSTERED INDEX IX_AuditLog_Target_Occurred
ON dbo.AuditLog (Target ASC, OccurredAtUtc DESC)
WHERE Target IS NOT NULL
ON ps_AuditLog_Month(OccurredAtUtc);
-- IX_AuditLog_Execution / IX_AuditLog_ParentExecution are NOT filtered: SQL
-- Server forbids a filtered-index WHERE predicate from referencing a computed
-- column (ExecutionId / ParentExecutionId are persisted computed columns). An
-- unfiltered index still backs the equality lookups GetExecutionTreeAsync uses;
-- it just also indexes the NULL rows. (The pre-C5 typed columns allowed the
-- IS NOT NULL filter; the computed-column constraint does not.)
CREATE NONCLUSTERED INDEX IX_AuditLog_Execution
ON dbo.AuditLog (ExecutionId)
ON ps_AuditLog_Month(OccurredAtUtc);
CREATE NONCLUSTERED INDEX IX_AuditLog_ParentExecution
ON dbo.AuditLog (ParentExecutionId)
ON ps_AuditLog_Month(OccurredAtUtc);
CREATE NONCLUSTERED INDEX IX_AuditLog_Node_Occurred
ON dbo.AuditLog (SourceNode ASC, OccurredAtUtc ASC)
ON ps_AuditLog_Month(OccurredAtUtc);
CREATE UNIQUE NONCLUSTERED INDEX UX_AuditLog_EventId
ON dbo.AuditLog (EventId)
ON [PRIMARY];");
// 5) Re-grant the append-only roles on the renamed table. The grants were
// object-scoped to the old (now-dropped) table, so they must be re-issued
// against the new one. Idempotent role creation guards a fresh DB. The
// DENY UPDATE / DENY DELETE on the writer role is deliberate — a future
// db_datawriter membership cannot quietly re-enable mutation (DENY > GRANT).
migrationBuilder.Sql(@"
IF DATABASE_PRINCIPAL_ID('scadabridge_audit_writer') IS NULL
EXEC sp_executesql N'CREATE ROLE scadabridge_audit_writer';
GRANT INSERT ON dbo.AuditLog TO scadabridge_audit_writer;
GRANT SELECT ON dbo.AuditLog TO scadabridge_audit_writer;
DENY UPDATE ON dbo.AuditLog TO scadabridge_audit_writer;
DENY DELETE ON dbo.AuditLog TO scadabridge_audit_writer;
IF DATABASE_PRINCIPAL_ID('scadabridge_audit_purger') IS NULL
EXEC sp_executesql N'CREATE ROLE scadabridge_audit_purger';
GRANT SELECT ON dbo.AuditLog TO scadabridge_audit_purger;
GRANT ALTER ON SCHEMA::dbo TO scadabridge_audit_purger;");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
// ONE-WAY MIGRATION. Collapsing the 24 typed columns into the canonical
// shape + a single DetailsJson bag is lossy to reverse byte-for-byte: the
// Action/Category/Outcome derivation discards the original Channel/Kind/
// Status split in a way only DetailsJson can reconstruct, and the codec's
// null-omission + key-order contract cannot be reproduced by a generic
// reverse projection. Reversing in place would also have to rebuild the
// partition-aligned indexes on the dropped typed columns. If a rollback is
// required, restore the central database from a pre-migration backup.
throw new System.NotSupportedException(
"CollapseAuditLogToCanonical is a one-way migration (Task 2.5 C5): the " +
"central dbo.AuditLog collapse projects the legacy typed columns into a " +
"lossy canonical/DetailsJson shape that cannot be reversed automatically. " +
"Restore the database from a pre-migration backup to roll back.");
}
}
}
src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Migrations/ScadaBridgeDbContextModelSnapshot.cs
+123 -139
View File
@@ -41,145 +41,6 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
b.ToTable("DataProtectionKeys");
});
modelBuilder.Entity("ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Entities.AuditLogRow", b =>
{
b.Property<Guid>("EventId")
.HasColumnType("uniqueidentifier");
b.Property<DateTime>("OccurredAtUtc")
.HasColumnType("datetime2");
b.Property<string>("Actor")
.HasMaxLength(128)
.IsUnicode(false)
.HasColumnType("varchar(128)");
b.Property<string>("Channel")
.IsRequired()
.HasMaxLength(32)
.IsUnicode(false)
.HasColumnType("varchar(32)");
b.Property<Guid?>("CorrelationId")
.HasColumnType("uniqueidentifier");
b.Property<int?>("DurationMs")
.HasColumnType("int");
b.Property<string>("ErrorDetail")
.HasColumnType("nvarchar(max)");
b.Property<string>("ErrorMessage")
.HasMaxLength(1024)
.HasColumnType("nvarchar(1024)");
b.Property<Guid?>("ExecutionId")
.HasColumnType("uniqueidentifier");
b.Property<string>("Extra")
.HasColumnType("nvarchar(max)");
b.Property<string>("ForwardState")
.HasMaxLength(32)
.IsUnicode(false)
.HasColumnType("varchar(32)");
b.Property<int?>("HttpStatus")
.HasColumnType("int");
b.Property<DateTime?>("IngestedAtUtc")
.HasColumnType("datetime2");
b.Property<string>("Kind")
.IsRequired()
.HasMaxLength(32)
.IsUnicode(false)
.HasColumnType("varchar(32)");
b.Property<Guid?>("ParentExecutionId")
.HasColumnType("uniqueidentifier");
b.Property<bool>("PayloadTruncated")
.HasColumnType("bit");
b.Property<string>("RequestSummary")
.HasColumnType("nvarchar(max)");
b.Property<string>("ResponseSummary")
.HasColumnType("nvarchar(max)");
b.Property<string>("SourceInstanceId")
.HasMaxLength(128)
.IsUnicode(false)
.HasColumnType("varchar(128)");
b.Property<string>("SourceNode")
.HasMaxLength(64)
.IsUnicode(false)
.HasColumnType("varchar(64)");
b.Property<string>("SourceScript")
.HasMaxLength(128)
.IsUnicode(false)
.HasColumnType("varchar(128)");
b.Property<string>("SourceSiteId")
.HasMaxLength(64)
.IsUnicode(false)
.HasColumnType("varchar(64)");
b.Property<string>("Status")
.IsRequired()
.HasMaxLength(32)
.IsUnicode(false)
.HasColumnType("varchar(32)");
b.Property<string>("Target")
.HasMaxLength(256)
.IsUnicode(false)
.HasColumnType("varchar(256)");
b.HasKey("EventId", "OccurredAtUtc");
b.HasIndex("CorrelationId")
.HasDatabaseName("IX_AuditLog_CorrelationId")
.HasFilter("[CorrelationId] IS NOT NULL");
b.HasIndex("EventId")
.IsUnique()
.HasDatabaseName("UX_AuditLog_EventId");
b.HasIndex("ExecutionId")
.HasDatabaseName("IX_AuditLog_Execution")
.HasFilter("[ExecutionId] IS NOT NULL");
b.HasIndex("OccurredAtUtc")
.IsDescending()
.HasDatabaseName("IX_AuditLog_OccurredAtUtc");
b.HasIndex("ParentExecutionId")
.HasDatabaseName("IX_AuditLog_ParentExecution")
.HasFilter("[ParentExecutionId] IS NOT NULL");
b.HasIndex("SourceNode", "OccurredAtUtc")
.HasDatabaseName("IX_AuditLog_Node_Occurred");
b.HasIndex("SourceSiteId", "OccurredAtUtc")
.IsDescending(false, true)
.HasDatabaseName("IX_AuditLog_Site_Occurred");
b.HasIndex("Target", "OccurredAtUtc")
.IsDescending(false, true)
.HasDatabaseName("IX_AuditLog_Target_Occurred")
.HasFilter("[Target] IS NOT NULL");
b.HasIndex("Channel", "Status", "OccurredAtUtc")
.IsDescending(false, false, true)
.HasDatabaseName("IX_AuditLog_Channel_Status_Occurred");
b.ToTable("AuditLog", (string)null);
});
modelBuilder.Entity("ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit.AuditLogEntry", b =>
{
b.Property<int>("Id")
@@ -1490,6 +1351,129 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
b.ToTable("TemplateScripts");
});
modelBuilder.Entity("ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Entities.AuditLogRow", b =>
{
b.Property<Guid>("EventId")
.HasColumnType("uniqueidentifier");
b.Property<DateTime>("OccurredAtUtc")
.HasColumnType("datetime2");
b.Property<string>("Action")
.IsRequired()
.HasMaxLength(64)
.IsUnicode(false)
.HasColumnType("varchar(64)");
b.Property<string>("Actor")
.HasMaxLength(256)
.HasColumnType("nvarchar(256)");
b.Property<string>("Channel")
.IsRequired()
.HasMaxLength(32)
.IsUnicode(false)
.HasColumnType("varchar(32)")
.HasColumnName("Category");
b.Property<Guid?>("CorrelationId")
.HasColumnType("uniqueidentifier");
b.Property<string>("DetailsJson")
.HasColumnType("nvarchar(max)");
b.Property<Guid?>("ExecutionId")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("uniqueidentifier")
.HasComputedColumnSql("CAST(JSON_VALUE(DetailsJson,'$.executionId') AS uniqueidentifier)", true);
b.Property<DateTime?>("IngestedAtUtc")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("datetime2(7)")
.HasComputedColumnSql("CAST(SWITCHOFFSET(CAST(JSON_VALUE(DetailsJson,'$.ingestedAtUtc') AS datetimeoffset), 0) AS datetime2(7))", false);
b.Property<string>("Kind")
.IsRequired()
.ValueGeneratedOnAddOrUpdate()
.HasMaxLength(32)
.IsUnicode(false)
.HasColumnType("varchar(32)")
.HasComputedColumnSql("JSON_VALUE(DetailsJson,'$.kind')", true);
b.Property<string>("Outcome")
.IsRequired()
.HasMaxLength(16)
.IsUnicode(false)
.HasColumnType("varchar(16)");
b.Property<Guid?>("ParentExecutionId")
.ValueGeneratedOnAddOrUpdate()
.HasColumnType("uniqueidentifier")
.HasComputedColumnSql("CAST(JSON_VALUE(DetailsJson,'$.parentExecutionId') AS uniqueidentifier)", true);
b.Property<string>("SourceNode")
.HasMaxLength(64)
.IsUnicode(false)
.HasColumnType("varchar(64)");
b.Property<string>("SourceSiteId")
.ValueGeneratedOnAddOrUpdate()
.HasMaxLength(64)
.IsUnicode(false)
.HasColumnType("varchar(64)")
.HasComputedColumnSql("JSON_VALUE(DetailsJson,'$.sourceSiteId')", true);
b.Property<string>("Status")
.IsRequired()
.ValueGeneratedOnAddOrUpdate()
.HasMaxLength(32)
.IsUnicode(false)
.HasColumnType("varchar(32)")
.HasComputedColumnSql("JSON_VALUE(DetailsJson,'$.status')", true);
b.Property<string>("Target")
.HasMaxLength(256)
.HasColumnType("nvarchar(256)");
b.HasKey("EventId", "OccurredAtUtc");
b.HasIndex("CorrelationId")
.HasDatabaseName("IX_AuditLog_CorrelationId")
.HasFilter("[CorrelationId] IS NOT NULL");
b.HasIndex("EventId")
.IsUnique()
.HasDatabaseName("UX_AuditLog_EventId");
b.HasIndex("ExecutionId")
.HasDatabaseName("IX_AuditLog_Execution");
b.HasIndex("OccurredAtUtc")
.IsDescending()
.HasDatabaseName("IX_AuditLog_OccurredAtUtc");
b.HasIndex("ParentExecutionId")
.HasDatabaseName("IX_AuditLog_ParentExecution");
b.HasIndex("SourceNode", "OccurredAtUtc")
.HasDatabaseName("IX_AuditLog_Node_Occurred");
b.HasIndex("SourceSiteId", "OccurredAtUtc")
.IsDescending(false, true)
.HasDatabaseName("IX_AuditLog_Site_Occurred");
b.HasIndex("Target", "OccurredAtUtc")
.IsDescending(false, true)
.HasDatabaseName("IX_AuditLog_Target_Occurred")
.HasFilter("[Target] IS NOT NULL");
b.HasIndex("Channel", "Status", "OccurredAtUtc")
.IsDescending(false, false, true)
.HasDatabaseName("IX_AuditLog_Channel_Status_Occurred");
b.ToTable("AuditLog", (string)null);
});
modelBuilder.Entity("ZB.MOM.WW.ScadaBridge.Commons.Entities.Deployment.DeployedConfigSnapshot", b =>
{
b.HasOne("ZB.MOM.WW.ScadaBridge.Commons.Entities.Instances.Instance", null)