feat(audit)!: remove the AuditLog hash chain, head export and two-login maintenance hardening
Owner decision 2026-11-15, at IT security's request, pending a family-wide general solution. Removed: the chaining service, hasher, verifier, chain head/tombstone entities and store, the #139 chain-start cutoff, AuditLog:HashChain (incl. ChainBacklog and HeadExport with its File/Syslog sinks and export service), the audit-chain-head-export health check, the VerifyAuditChain / VerifyAuditChainHeads management commands and the `audit verify-chain` / `audit verify-heads` CLI commands, and the two-login plumbing (AuditLog:MaintenanceConnectionString, AuditMaintenanceContext, the keyed audit-maintenance repository/archive). Purge, archive, retention, partition roll-forward and backfill-source-node all run on the runtime connection again, as before #107. Parquet archive (#42) kept, chain-free: format v2 (ten stored columns, no tombstone file, manifest without chain state), the switch guard is the row count only (51003 gone), verify = data-file size, SHA-256 and row count, and superseded-stale cleanup proves coverage by EventId. Format-v1 archives still list, verify and are cleaned up (their tombstone file too). Migration RemoveAuditLogHashChain drops the chain indexes, the scadabridge_audit_chainer role (members first), both chain tables, ChainSeq/RowHash and the purger's AuditMaintenanceGrants grants; writer/purger roles stay. Down restores them. Every chain-column reference is EXEC-deferred for the idempotent script (#126); FlipAuditLogClusteredKey's two chain index statements get the same deferral, behaviour unchanged, so re-running the idempotent script on a migrated database works. The partition-switch staging table adds the chain columns only when the live table has them, so the new binary purges either schema. Deploy gate: old binaries map the chain columns, so the migration ships with the new binaries, both centrals stopped. A leftover AuditLog:HashChain section or MaintenanceConnectionString (DEV/QA still carry AuditLog__HashChain__Enabled=true) is ignored with one startup warning (RetiredAuditSettingsWarning). Claude-Session: https://claude.ai/code/session_01GT67AFKC45MBz1wPZnVbX8
This commit is contained in:
+40
-23
@@ -29,14 +29,18 @@ public sealed class FlipAuditLogClusteredKeyMigrationTests : IClassFixture<MsSql
|
||||
// The migration immediately before the flip, so "Down" reverts exactly this one.
|
||||
private static readonly string PreviousMigration = MigrationBefore(FlipMigration);
|
||||
|
||||
// The ten non-clustered indexes at head. RemoveAuditLogHashChain (2026-11-15, after the
|
||||
// flip) dropped the two hash-chain indexes; one migration before the flip they still exist.
|
||||
private static readonly string[] NonClusteredIndexes =
|
||||
[
|
||||
"IX_AuditLog_OccurredAtUtc", "IX_AuditLog_Site_Occurred", "IX_AuditLog_CorrelationId",
|
||||
"IX_AuditLog_Channel_Status_Occurred", "IX_AuditLog_Target_Occurred", "IX_AuditLog_Execution",
|
||||
"IX_AuditLog_ParentExecution", "IX_AuditLog_Node_Occurred", "IX_AuditLog_Kind_Occurred",
|
||||
"IX_AuditLog_Actor_Occurred", "IX_AuditLog_Unchained", "IX_AuditLog_ChainSeq",
|
||||
"IX_AuditLog_Actor_Occurred",
|
||||
];
|
||||
|
||||
private static readonly string[] ChainIndexes = ["IX_AuditLog_Unchained", "IX_AuditLog_ChainSeq"];
|
||||
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
private readonly ITestOutputHelper _output;
|
||||
|
||||
@@ -50,9 +54,10 @@ public sealed class FlipAuditLogClusteredKeyMigrationTests : IClassFixture<MsSql
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// After the full chain: clustered PK (OccurredAtUtc, EventId), the unique aligned
|
||||
/// (EventId, OccurredAtUtc) index, all twelve non-clustered indexes with unchanged keys and
|
||||
/// filters, everything on ps_AuditLog_Month, and every append-only/chain grant intact.
|
||||
/// After the full migration chain: clustered PK (OccurredAtUtc, EventId), the unique aligned
|
||||
/// (EventId, OccurredAtUtc) index, the ten non-clustered indexes with unchanged keys and
|
||||
/// filters (the two hash-chain indexes gone), everything on ps_AuditLog_Month, and every
|
||||
/// append-only grant intact.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[SkippableFact]
|
||||
@@ -67,7 +72,7 @@ public sealed class FlipAuditLogClusteredKeyMigrationTests : IClassFixture<MsSql
|
||||
Assert.True(indexes["UX_AuditLog_EventId_Occurred"].Unique);
|
||||
Assert.Equal(NonClusteredIndexes.Length + 2, indexes.Count);
|
||||
Assert.All(indexes.Values, i => Assert.Equal("ps_AuditLog_Month", i.DataSpace));
|
||||
AssertTwelveNonClusteredIndexes(indexes);
|
||||
AssertNonClusteredIndexes(indexes, withChainIndexes: false);
|
||||
Assert.Equal(ExpectedGrants, await ReadGrantsAsync());
|
||||
}
|
||||
|
||||
@@ -86,14 +91,16 @@ public sealed class FlipAuditLogClusteredKeyMigrationTests : IClassFixture<MsSql
|
||||
var down = await ReadIndexesAsync();
|
||||
Assert.Equal("EventId, OccurredAtUtc", down["PK_AuditLog"].Keys);
|
||||
Assert.False(down.ContainsKey("UX_AuditLog_EventId_Occurred"));
|
||||
AssertTwelveNonClusteredIndexes(down);
|
||||
Assert.Equal(GrantsBeforeMaintenanceGrants, await ReadGrantsAsync());
|
||||
AssertNonClusteredIndexes(down, withChainIndexes: true);
|
||||
Assert.Equal(GrantsBeforeTheFlip, await ReadGrantsAsync());
|
||||
|
||||
// Old shape: the purge's staging table must take the old key order.
|
||||
// Old shape: the purge's staging table must take the old key order — and, since this is
|
||||
// also before RemoveAuditLogHashChain, carry the two chain columns the live table still has.
|
||||
await AssertSwitchOutPurgesAsync(new DateTime(2027, 3, 1, 0, 0, 0, DateTimeKind.Utc));
|
||||
|
||||
// Back to head (not just to the flip): migrations after it — AuditMaintenanceGrants —
|
||||
// were reverted by the Down too, and the class leaves the database fully migrated.
|
||||
// Back to head (not just to the flip): migrations after it — AuditMaintenanceGrants,
|
||||
// RemoveAuditLogHashChain — were reverted by the Down too, and the class leaves the
|
||||
// database fully migrated.
|
||||
await MigrateToHeadAsync();
|
||||
var up = await ReadIndexesAsync();
|
||||
Assert.Equal("OccurredAtUtc, EventId", up["PK_AuditLog"].Keys);
|
||||
@@ -146,26 +153,29 @@ public sealed class FlipAuditLogClusteredKeyMigrationTests : IClassFixture<MsSql
|
||||
}
|
||||
}
|
||||
|
||||
// At head: RemoveAuditLogHashChain dropped the chainer role and revoked the purger's
|
||||
// AuditMaintenanceGrants column grant.
|
||||
private static readonly string[] ExpectedGrants =
|
||||
[
|
||||
"scadabridge_audit_chainer|SELECT|GRANT|",
|
||||
"scadabridge_audit_chainer|UPDATE|GRANT|ChainSeq",
|
||||
"scadabridge_audit_chainer|UPDATE|GRANT|RowHash",
|
||||
"scadabridge_audit_purger|DELETE|GRANT|",
|
||||
"scadabridge_audit_purger|SELECT|GRANT|",
|
||||
// AuditMaintenanceGrants (migration 64, after the flip) — the archive path's column grant.
|
||||
"scadabridge_audit_purger|UPDATE|GRANT|SourceNode",
|
||||
"scadabridge_audit_writer|DELETE|DENY|",
|
||||
"scadabridge_audit_writer|INSERT|GRANT|",
|
||||
"scadabridge_audit_writer|SELECT|GRANT|",
|
||||
"scadabridge_audit_writer|UPDATE|DENY|",
|
||||
];
|
||||
|
||||
// The grants one migration before the flip: AuditMaintenanceGrants is not applied yet.
|
||||
private static readonly string[] GrantsBeforeMaintenanceGrants =
|
||||
ExpectedGrants.Where(g => !g.EndsWith("|SourceNode", StringComparison.Ordinal)).ToArray();
|
||||
// One migration before the flip: the hash chain's chainer grants exist (AddAuditLogHashChain)
|
||||
// and AuditMaintenanceGrants is not applied yet.
|
||||
private static readonly string[] GrantsBeforeTheFlip =
|
||||
ExpectedGrants.Concat(
|
||||
[
|
||||
"scadabridge_audit_chainer|SELECT|GRANT|",
|
||||
"scadabridge_audit_chainer|UPDATE|GRANT|ChainSeq",
|
||||
"scadabridge_audit_chainer|UPDATE|GRANT|RowHash",
|
||||
]).Order(StringComparer.Ordinal).ToArray();
|
||||
|
||||
private static void AssertTwelveNonClusteredIndexes(IReadOnlyDictionary<string, IndexShape> indexes)
|
||||
private static void AssertNonClusteredIndexes(IReadOnlyDictionary<string, IndexShape> indexes, bool withChainIndexes)
|
||||
{
|
||||
Assert.Equal("OccurredAtUtc DESC", indexes["IX_AuditLog_OccurredAtUtc"].Keys);
|
||||
Assert.Equal("Status", indexes["IX_AuditLog_OccurredAtUtc"].Included);
|
||||
@@ -180,11 +190,18 @@ public sealed class FlipAuditLogClusteredKeyMigrationTests : IClassFixture<MsSql
|
||||
Assert.Equal("SourceNode, OccurredAtUtc", indexes["IX_AuditLog_Node_Occurred"].Keys);
|
||||
Assert.Equal("Kind, OccurredAtUtc DESC", indexes["IX_AuditLog_Kind_Occurred"].Keys);
|
||||
Assert.Equal("Actor, OccurredAtUtc DESC", indexes["IX_AuditLog_Actor_Occurred"].Keys);
|
||||
Assert.Equal("OccurredAtUtc, EventId", indexes["IX_AuditLog_Unchained"].Keys);
|
||||
Assert.Equal("([ChainSeq] IS NULL)", indexes["IX_AuditLog_Unchained"].Filter);
|
||||
Assert.Equal("ChainSeq", indexes["IX_AuditLog_ChainSeq"].Keys);
|
||||
Assert.Equal("([ChainSeq] IS NOT NULL)", indexes["IX_AuditLog_ChainSeq"].Filter);
|
||||
Assert.All(NonClusteredIndexes, n => Assert.False(indexes[n].Clustered, n));
|
||||
if (withChainIndexes)
|
||||
{
|
||||
Assert.Equal("OccurredAtUtc, EventId", indexes["IX_AuditLog_Unchained"].Keys);
|
||||
Assert.Equal("([ChainSeq] IS NULL)", indexes["IX_AuditLog_Unchained"].Filter);
|
||||
Assert.Equal("ChainSeq", indexes["IX_AuditLog_ChainSeq"].Keys);
|
||||
Assert.Equal("([ChainSeq] IS NOT NULL)", indexes["IX_AuditLog_ChainSeq"].Filter);
|
||||
}
|
||||
else
|
||||
{
|
||||
Assert.All(ChainIndexes, n => Assert.False(indexes.ContainsKey(n), n));
|
||||
}
|
||||
}
|
||||
|
||||
private async Task AssertSwitchOutPurgesAsync(DateTime month)
|
||||
|
||||
+2
-1
@@ -60,7 +60,8 @@ public sealed class IdempotentMigrationScriptTests : IClassFixture<MsSqlMigratio
|
||||
|
||||
await RunScriptAsync(script);
|
||||
Assert.Equal(all, await ReadHistoryAsync());
|
||||
Assert.Equal(2, await CountChainIndexesAsync());
|
||||
// AddAuditLogHashChain created the two chain indexes, RemoveAuditLogHashChain dropped them.
|
||||
Assert.Equal(0, await CountChainIndexesAsync());
|
||||
|
||||
// Second run against the fully-migrated database: every guard is false.
|
||||
await RunScriptAsync(script);
|
||||
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
using Microsoft.EntityFrameworkCore.Migrations.Operations;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
|
||||
/// <summary>
|
||||
/// SQL Server tests for the <c>RemoveAuditLogHashChain</c> migration (2026-11-15): at head the
|
||||
/// hash-chain columns, tables, indexes and chainer role are gone and the purger keeps only its
|
||||
/// original grants; Down restores what <c>AddAuditLogHashChain</c> + <c>AuditMaintenanceGrants</c>
|
||||
/// created; and Up is re-runnable and drops a chainer role that still has members.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The tests in this class migrate the fixture database down and back up, so they own their
|
||||
/// fixture (one database per test class) and every test leaves the database fully migrated.
|
||||
/// </remarks>
|
||||
public sealed class RemoveAuditLogHashChainMigrationTests : IClassFixture<MsSqlMigrationFixture>
|
||||
{
|
||||
private const string RemoveMigration = "20261115221558_RemoveAuditLogHashChain";
|
||||
|
||||
private static readonly string PreviousMigration = MigrationBefore(RemoveMigration);
|
||||
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
|
||||
/// <summary>Creates the test class with its own migrated SQL Server database.</summary>
|
||||
/// <param name="fixture">The per-class migration fixture.</param>
|
||||
public RemoveAuditLogHashChainMigrationTests(MsSqlMigrationFixture fixture) => _fixture = fixture;
|
||||
|
||||
/// <summary>At head nothing of the hash chain or the maintenance grants is left.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[SkippableFact]
|
||||
public async Task AtHead_TheChainSchema_AndTheMaintenanceGrants_AreGone()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
await AssertRemovedAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Down restores the chain schema exactly (columns, tables, indexes, chainer role and grants,
|
||||
/// the purger's maintenance grants); Up then drops it again, also when the chainer role has a
|
||||
/// member; and re-running Up's SQL on the migrated schema changes nothing.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[SkippableFact]
|
||||
public async Task DownUp_RoundTrips_DropsAChainerRoleWithMembers_AndIsReRunnable()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
await MigrateToAsync(PreviousMigration);
|
||||
try
|
||||
{
|
||||
Assert.True(await ScalarAsync<int>("SELECT CASE WHEN COL_LENGTH('dbo.AuditLog','ChainSeq') IS NULL THEN 0 ELSE 1 END;") == 1);
|
||||
Assert.True(await ScalarAsync<int>("SELECT CASE WHEN COL_LENGTH('dbo.AuditLog','RowHash') IS NULL THEN 0 ELSE 1 END;") == 1);
|
||||
Assert.True(await ScalarAsync<int>("SELECT CASE WHEN OBJECT_ID('dbo.AuditLogChainHead','U') IS NULL THEN 0 ELSE 1 END;") == 1);
|
||||
Assert.True(await ScalarAsync<int>("SELECT CASE WHEN OBJECT_ID('dbo.AuditLogChainTombstone','U') IS NULL THEN 0 ELSE 1 END;") == 1);
|
||||
Assert.Equal(2, await ScalarAsync<int>(
|
||||
"SELECT COUNT(*) FROM sys.indexes WHERE object_id = OBJECT_ID('dbo.AuditLog') AND name IN ('IX_AuditLog_Unchained','IX_AuditLog_ChainSeq');"));
|
||||
Assert.Contains("scadabridge_audit_chainer|UPDATE|GRANT|ChainSeq", await ReadAuditLogGrantsAsync());
|
||||
Assert.Contains("scadabridge_audit_purger|UPDATE|GRANT|SourceNode", await ReadAuditLogGrantsAsync());
|
||||
Assert.Equal(1, await ScalarAsync<int>(
|
||||
"SELECT COUNT(*) FROM sys.database_permissions p JOIN sys.database_principals pr ON pr.principal_id = p.grantee_principal_id "
|
||||
+ "WHERE pr.name = 'scadabridge_audit_purger' AND p.class = 0 AND p.permission_name = 'ALTER ANY DATASPACE' AND p.state_desc = 'GRANT';"));
|
||||
|
||||
// A login was put in the chainer role (the hardened two-login setup): Up must still drop the role.
|
||||
await ExecAsync("IF DATABASE_PRINCIPAL_ID('rm_chain_test_user') IS NULL CREATE USER rm_chain_test_user WITHOUT LOGIN;"
|
||||
+ "ALTER ROLE scadabridge_audit_chainer ADD MEMBER rm_chain_test_user;");
|
||||
}
|
||||
finally
|
||||
{
|
||||
await MigrateToHeadAsync();
|
||||
}
|
||||
|
||||
await AssertRemovedAsync();
|
||||
Assert.Equal(1, await ScalarAsync<int>("SELECT CASE WHEN DATABASE_PRINCIPAL_ID('rm_chain_test_user') IS NULL THEN 0 ELSE 1 END;"));
|
||||
|
||||
// Re-run Up's SQL against the migrated schema: every statement is guarded.
|
||||
foreach (var op in new RemoveAuditLogHashChain().UpOperations.OfType<SqlOperation>())
|
||||
{
|
||||
await ExecAsync(op.Sql);
|
||||
}
|
||||
|
||||
await AssertRemovedAsync();
|
||||
}
|
||||
|
||||
private async Task AssertRemovedAsync()
|
||||
{
|
||||
Assert.Equal(0, await ScalarAsync<int>("SELECT CASE WHEN COL_LENGTH('dbo.AuditLog','ChainSeq') IS NULL THEN 0 ELSE 1 END;"));
|
||||
Assert.Equal(0, await ScalarAsync<int>("SELECT CASE WHEN COL_LENGTH('dbo.AuditLog','RowHash') IS NULL THEN 0 ELSE 1 END;"));
|
||||
Assert.Equal(0, await ScalarAsync<int>("SELECT CASE WHEN OBJECT_ID('dbo.AuditLogChainHead','U') IS NULL THEN 0 ELSE 1 END;"));
|
||||
Assert.Equal(0, await ScalarAsync<int>("SELECT CASE WHEN OBJECT_ID('dbo.AuditLogChainTombstone','U') IS NULL THEN 0 ELSE 1 END;"));
|
||||
Assert.Equal(0, await ScalarAsync<int>(
|
||||
"SELECT COUNT(*) FROM sys.indexes WHERE object_id = OBJECT_ID('dbo.AuditLog') AND name IN ('IX_AuditLog_Unchained','IX_AuditLog_ChainSeq');"));
|
||||
Assert.Equal(0, await ScalarAsync<int>("SELECT CASE WHEN DATABASE_PRINCIPAL_ID('scadabridge_audit_chainer') IS NULL THEN 0 ELSE 1 END;"));
|
||||
Assert.Equal(0, await ScalarAsync<int>(
|
||||
"SELECT COUNT(*) FROM sys.database_permissions p JOIN sys.database_principals pr ON pr.principal_id = p.grantee_principal_id "
|
||||
+ "WHERE pr.name = 'scadabridge_audit_purger' AND p.class = 0 AND p.permission_name = 'ALTER ANY DATASPACE';"));
|
||||
|
||||
// The roles that predate the chain stay, with exactly their original AuditLog grants.
|
||||
Assert.Equal(
|
||||
new[]
|
||||
{
|
||||
"scadabridge_audit_purger|DELETE|GRANT|",
|
||||
"scadabridge_audit_purger|SELECT|GRANT|",
|
||||
"scadabridge_audit_writer|DELETE|DENY|",
|
||||
"scadabridge_audit_writer|INSERT|GRANT|",
|
||||
"scadabridge_audit_writer|SELECT|GRANT|",
|
||||
"scadabridge_audit_writer|UPDATE|DENY|",
|
||||
},
|
||||
await ReadAuditLogGrantsAsync());
|
||||
}
|
||||
|
||||
private async Task<T> ScalarAsync<T>(string sql)
|
||||
{
|
||||
await using var conn = _fixture.OpenConnection();
|
||||
await using var cmd = new SqlCommand(sql, conn);
|
||||
return (T)Convert.ChangeType((await cmd.ExecuteScalarAsync())!, typeof(T), System.Globalization.CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private async Task ExecAsync(string sql)
|
||||
{
|
||||
await using var conn = _fixture.OpenConnection();
|
||||
await using var cmd = new SqlCommand(sql, conn) { CommandTimeout = 300 };
|
||||
await cmd.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
private async Task<string[]> ReadAuditLogGrantsAsync()
|
||||
{
|
||||
await using var conn = _fixture.OpenConnection();
|
||||
await using var cmd = new SqlCommand(@"
|
||||
SELECT pr.name, p.permission_name, p.state_desc, COL_NAME(p.major_id, p.minor_id)
|
||||
FROM sys.database_permissions p JOIN sys.database_principals pr ON pr.principal_id = p.grantee_principal_id
|
||||
WHERE p.major_id = OBJECT_ID('dbo.AuditLog') AND p.class = 1;", conn);
|
||||
var list = new List<string>();
|
||||
await using var reader = await cmd.ExecuteReaderAsync();
|
||||
while (await reader.ReadAsync())
|
||||
{
|
||||
list.Add($"{reader.GetString(0)}|{reader.GetString(1)}|{reader.GetString(2)}|{(reader.IsDBNull(3) ? "" : reader.GetString(3))}");
|
||||
}
|
||||
|
||||
return list.Order(StringComparer.Ordinal).ToArray();
|
||||
}
|
||||
|
||||
private async Task MigrateToAsync(string migration)
|
||||
{
|
||||
await using var ctx = new ScadaBridgeDbContext(new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(_fixture.ConnectionString, sql => sql.CommandTimeout(600)).Options);
|
||||
await ctx.GetService<IMigrator>().MigrateAsync(migration);
|
||||
}
|
||||
|
||||
private async Task MigrateToHeadAsync()
|
||||
{
|
||||
await using var ctx = new ScadaBridgeDbContext(new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer(_fixture.ConnectionString, sql => sql.CommandTimeout(600)).Options);
|
||||
await ctx.GetService<IMigrator>().MigrateAsync();
|
||||
}
|
||||
|
||||
private static string MigrationBefore(string migration)
|
||||
{
|
||||
using var ctx = new ScadaBridgeDbContext(new DbContextOptionsBuilder<ScadaBridgeDbContext>()
|
||||
.UseSqlServer("Server=unused;Database=unused").Options);
|
||||
var all = ctx.Database.GetMigrations().ToList();
|
||||
return all[all.IndexOf(migration) - 1];
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user