feat(audit)!: remove the AuditLog hash chain, head export and two-login maintenance hardening

Owner decision 2026-11-15, at IT security's request, pending a family-wide general solution.

Removed: the chaining service, hasher, verifier, chain head/tombstone entities and store, the #139
chain-start cutoff, AuditLog:HashChain (incl. ChainBacklog and HeadExport with its File/Syslog sinks
and export service), the audit-chain-head-export health check, the VerifyAuditChain /
VerifyAuditChainHeads management commands and the `audit verify-chain` / `audit verify-heads` CLI
commands, and the two-login plumbing (AuditLog:MaintenanceConnectionString, AuditMaintenanceContext,
the keyed audit-maintenance repository/archive). Purge, archive, retention, partition roll-forward
and backfill-source-node all run on the runtime connection again, as before #107.

Parquet archive (#42) kept, chain-free: format v2 (ten stored columns, no tombstone file, manifest
without chain state), the switch guard is the row count only (51003 gone), verify = data-file size,
SHA-256 and row count, and superseded-stale cleanup proves coverage by EventId. Format-v1 archives
still list, verify and are cleaned up (their tombstone file too).

Migration RemoveAuditLogHashChain drops the chain indexes, the scadabridge_audit_chainer role
(members first), both chain tables, ChainSeq/RowHash and the purger's AuditMaintenanceGrants grants;
writer/purger roles stay. Down restores them. Every chain-column reference is EXEC-deferred for the
idempotent script (#126); FlipAuditLogClusteredKey's two chain index statements get the same
deferral, behaviour unchanged, so re-running the idempotent script on a migrated database works.
The partition-switch staging table adds the chain columns only when the live table has them, so the
new binary purges either schema. Deploy gate: old binaries map the chain columns, so the migration
ships with the new binaries, both centrals stopped.

A leftover AuditLog:HashChain section or MaintenanceConnectionString (DEV/QA still carry
AuditLog__HashChain__Enabled=true) is ignored with one startup warning (RetiredAuditSettingsWarning).

Claude-Session: https://claude.ai/code/session_01GT67AFKC45MBz1wPZnVbX8
This commit is contained in:
Joseph Doherty
2026-11-15 17:37:33 -05:00
parent 125db03232
commit ed2ed0a367
82 changed files with 3203 additions and 7603 deletions
@@ -29,14 +29,18 @@ public sealed class FlipAuditLogClusteredKeyMigrationTests : IClassFixture<MsSql
// The migration immediately before the flip, so "Down" reverts exactly this one.
private static readonly string PreviousMigration = MigrationBefore(FlipMigration);
// The ten non-clustered indexes at head. RemoveAuditLogHashChain (2026-11-15, after the
// flip) dropped the two hash-chain indexes; one migration before the flip they still exist.
private static readonly string[] NonClusteredIndexes =
[
"IX_AuditLog_OccurredAtUtc", "IX_AuditLog_Site_Occurred", "IX_AuditLog_CorrelationId",
"IX_AuditLog_Channel_Status_Occurred", "IX_AuditLog_Target_Occurred", "IX_AuditLog_Execution",
"IX_AuditLog_ParentExecution", "IX_AuditLog_Node_Occurred", "IX_AuditLog_Kind_Occurred",
"IX_AuditLog_Actor_Occurred", "IX_AuditLog_Unchained", "IX_AuditLog_ChainSeq",
"IX_AuditLog_Actor_Occurred",
];
private static readonly string[] ChainIndexes = ["IX_AuditLog_Unchained", "IX_AuditLog_ChainSeq"];
private readonly MsSqlMigrationFixture _fixture;
private readonly ITestOutputHelper _output;
@@ -50,9 +54,10 @@ public sealed class FlipAuditLogClusteredKeyMigrationTests : IClassFixture<MsSql
}
/// <summary>
/// After the full chain: clustered PK (OccurredAtUtc, EventId), the unique aligned
/// (EventId, OccurredAtUtc) index, all twelve non-clustered indexes with unchanged keys and
/// filters, everything on ps_AuditLog_Month, and every append-only/chain grant intact.
/// After the full migration chain: clustered PK (OccurredAtUtc, EventId), the unique aligned
/// (EventId, OccurredAtUtc) index, the ten non-clustered indexes with unchanged keys and
/// filters (the two hash-chain indexes gone), everything on ps_AuditLog_Month, and every
/// append-only grant intact.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
@@ -67,7 +72,7 @@ public sealed class FlipAuditLogClusteredKeyMigrationTests : IClassFixture<MsSql
Assert.True(indexes["UX_AuditLog_EventId_Occurred"].Unique);
Assert.Equal(NonClusteredIndexes.Length + 2, indexes.Count);
Assert.All(indexes.Values, i => Assert.Equal("ps_AuditLog_Month", i.DataSpace));
AssertTwelveNonClusteredIndexes(indexes);
AssertNonClusteredIndexes(indexes, withChainIndexes: false);
Assert.Equal(ExpectedGrants, await ReadGrantsAsync());
}
@@ -86,14 +91,16 @@ public sealed class FlipAuditLogClusteredKeyMigrationTests : IClassFixture<MsSql
var down = await ReadIndexesAsync();
Assert.Equal("EventId, OccurredAtUtc", down["PK_AuditLog"].Keys);
Assert.False(down.ContainsKey("UX_AuditLog_EventId_Occurred"));
AssertTwelveNonClusteredIndexes(down);
Assert.Equal(GrantsBeforeMaintenanceGrants, await ReadGrantsAsync());
AssertNonClusteredIndexes(down, withChainIndexes: true);
Assert.Equal(GrantsBeforeTheFlip, await ReadGrantsAsync());
// Old shape: the purge's staging table must take the old key order.
// Old shape: the purge's staging table must take the old key order — and, since this is
// also before RemoveAuditLogHashChain, carry the two chain columns the live table still has.
await AssertSwitchOutPurgesAsync(new DateTime(2027, 3, 1, 0, 0, 0, DateTimeKind.Utc));
// Back to head (not just to the flip): migrations after it — AuditMaintenanceGrants —
// were reverted by the Down too, and the class leaves the database fully migrated.
// Back to head (not just to the flip): migrations after it — AuditMaintenanceGrants,
// RemoveAuditLogHashChain — were reverted by the Down too, and the class leaves the
// database fully migrated.
await MigrateToHeadAsync();
var up = await ReadIndexesAsync();
Assert.Equal("OccurredAtUtc, EventId", up["PK_AuditLog"].Keys);
@@ -146,26 +153,29 @@ public sealed class FlipAuditLogClusteredKeyMigrationTests : IClassFixture<MsSql
}
}
// At head: RemoveAuditLogHashChain dropped the chainer role and revoked the purger's
// AuditMaintenanceGrants column grant.
private static readonly string[] ExpectedGrants =
[
"scadabridge_audit_chainer|SELECT|GRANT|",
"scadabridge_audit_chainer|UPDATE|GRANT|ChainSeq",
"scadabridge_audit_chainer|UPDATE|GRANT|RowHash",
"scadabridge_audit_purger|DELETE|GRANT|",
"scadabridge_audit_purger|SELECT|GRANT|",
// AuditMaintenanceGrants (migration 64, after the flip) — the archive path's column grant.
"scadabridge_audit_purger|UPDATE|GRANT|SourceNode",
"scadabridge_audit_writer|DELETE|DENY|",
"scadabridge_audit_writer|INSERT|GRANT|",
"scadabridge_audit_writer|SELECT|GRANT|",
"scadabridge_audit_writer|UPDATE|DENY|",
];
// The grants one migration before the flip: AuditMaintenanceGrants is not applied yet.
private static readonly string[] GrantsBeforeMaintenanceGrants =
ExpectedGrants.Where(g => !g.EndsWith("|SourceNode", StringComparison.Ordinal)).ToArray();
// One migration before the flip: the hash chain's chainer grants exist (AddAuditLogHashChain)
// and AuditMaintenanceGrants is not applied yet.
private static readonly string[] GrantsBeforeTheFlip =
ExpectedGrants.Concat(
[
"scadabridge_audit_chainer|SELECT|GRANT|",
"scadabridge_audit_chainer|UPDATE|GRANT|ChainSeq",
"scadabridge_audit_chainer|UPDATE|GRANT|RowHash",
]).Order(StringComparer.Ordinal).ToArray();
private static void AssertTwelveNonClusteredIndexes(IReadOnlyDictionary<string, IndexShape> indexes)
private static void AssertNonClusteredIndexes(IReadOnlyDictionary<string, IndexShape> indexes, bool withChainIndexes)
{
Assert.Equal("OccurredAtUtc DESC", indexes["IX_AuditLog_OccurredAtUtc"].Keys);
Assert.Equal("Status", indexes["IX_AuditLog_OccurredAtUtc"].Included);
@@ -180,11 +190,18 @@ public sealed class FlipAuditLogClusteredKeyMigrationTests : IClassFixture<MsSql
Assert.Equal("SourceNode, OccurredAtUtc", indexes["IX_AuditLog_Node_Occurred"].Keys);
Assert.Equal("Kind, OccurredAtUtc DESC", indexes["IX_AuditLog_Kind_Occurred"].Keys);
Assert.Equal("Actor, OccurredAtUtc DESC", indexes["IX_AuditLog_Actor_Occurred"].Keys);
Assert.Equal("OccurredAtUtc, EventId", indexes["IX_AuditLog_Unchained"].Keys);
Assert.Equal("([ChainSeq] IS NULL)", indexes["IX_AuditLog_Unchained"].Filter);
Assert.Equal("ChainSeq", indexes["IX_AuditLog_ChainSeq"].Keys);
Assert.Equal("([ChainSeq] IS NOT NULL)", indexes["IX_AuditLog_ChainSeq"].Filter);
Assert.All(NonClusteredIndexes, n => Assert.False(indexes[n].Clustered, n));
if (withChainIndexes)
{
Assert.Equal("OccurredAtUtc, EventId", indexes["IX_AuditLog_Unchained"].Keys);
Assert.Equal("([ChainSeq] IS NULL)", indexes["IX_AuditLog_Unchained"].Filter);
Assert.Equal("ChainSeq", indexes["IX_AuditLog_ChainSeq"].Keys);
Assert.Equal("([ChainSeq] IS NOT NULL)", indexes["IX_AuditLog_ChainSeq"].Filter);
}
else
{
Assert.All(ChainIndexes, n => Assert.False(indexes.ContainsKey(n), n));
}
}
private async Task AssertSwitchOutPurgesAsync(DateTime month)
@@ -60,7 +60,8 @@ public sealed class IdempotentMigrationScriptTests : IClassFixture<MsSqlMigratio
await RunScriptAsync(script);
Assert.Equal(all, await ReadHistoryAsync());
Assert.Equal(2, await CountChainIndexesAsync());
// AddAuditLogHashChain created the two chain indexes, RemoveAuditLogHashChain dropped them.
Assert.Equal(0, await CountChainIndexesAsync());
// Second run against the fully-migrated database: every guard is false.
await RunScriptAsync(script);
@@ -0,0 +1,167 @@
using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
using Microsoft.EntityFrameworkCore.Migrations.Operations;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations;
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
/// <summary>
/// SQL Server tests for the <c>RemoveAuditLogHashChain</c> migration (2026-11-15): at head the
/// hash-chain columns, tables, indexes and chainer role are gone and the purger keeps only its
/// original grants; Down restores what <c>AddAuditLogHashChain</c> + <c>AuditMaintenanceGrants</c>
/// created; and Up is re-runnable and drops a chainer role that still has members.
/// </summary>
/// <remarks>
/// The tests in this class migrate the fixture database down and back up, so they own their
/// fixture (one database per test class) and every test leaves the database fully migrated.
/// </remarks>
public sealed class RemoveAuditLogHashChainMigrationTests : IClassFixture<MsSqlMigrationFixture>
{
private const string RemoveMigration = "20261115221558_RemoveAuditLogHashChain";
private static readonly string PreviousMigration = MigrationBefore(RemoveMigration);
private readonly MsSqlMigrationFixture _fixture;
/// <summary>Creates the test class with its own migrated SQL Server database.</summary>
/// <param name="fixture">The per-class migration fixture.</param>
public RemoveAuditLogHashChainMigrationTests(MsSqlMigrationFixture fixture) => _fixture = fixture;
/// <summary>At head nothing of the hash chain or the maintenance grants is left.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task AtHead_TheChainSchema_AndTheMaintenanceGrants_AreGone()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
await AssertRemovedAsync();
}
/// <summary>
/// Down restores the chain schema exactly (columns, tables, indexes, chainer role and grants,
/// the purger's maintenance grants); Up then drops it again, also when the chainer role has a
/// member; and re-running Up's SQL on the migrated schema changes nothing.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task DownUp_RoundTrips_DropsAChainerRoleWithMembers_AndIsReRunnable()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
await MigrateToAsync(PreviousMigration);
try
{
Assert.True(await ScalarAsync<int>("SELECT CASE WHEN COL_LENGTH('dbo.AuditLog','ChainSeq') IS NULL THEN 0 ELSE 1 END;") == 1);
Assert.True(await ScalarAsync<int>("SELECT CASE WHEN COL_LENGTH('dbo.AuditLog','RowHash') IS NULL THEN 0 ELSE 1 END;") == 1);
Assert.True(await ScalarAsync<int>("SELECT CASE WHEN OBJECT_ID('dbo.AuditLogChainHead','U') IS NULL THEN 0 ELSE 1 END;") == 1);
Assert.True(await ScalarAsync<int>("SELECT CASE WHEN OBJECT_ID('dbo.AuditLogChainTombstone','U') IS NULL THEN 0 ELSE 1 END;") == 1);
Assert.Equal(2, await ScalarAsync<int>(
"SELECT COUNT(*) FROM sys.indexes WHERE object_id = OBJECT_ID('dbo.AuditLog') AND name IN ('IX_AuditLog_Unchained','IX_AuditLog_ChainSeq');"));
Assert.Contains("scadabridge_audit_chainer|UPDATE|GRANT|ChainSeq", await ReadAuditLogGrantsAsync());
Assert.Contains("scadabridge_audit_purger|UPDATE|GRANT|SourceNode", await ReadAuditLogGrantsAsync());
Assert.Equal(1, await ScalarAsync<int>(
"SELECT COUNT(*) FROM sys.database_permissions p JOIN sys.database_principals pr ON pr.principal_id = p.grantee_principal_id "
+ "WHERE pr.name = 'scadabridge_audit_purger' AND p.class = 0 AND p.permission_name = 'ALTER ANY DATASPACE' AND p.state_desc = 'GRANT';"));
// A login was put in the chainer role (the hardened two-login setup): Up must still drop the role.
await ExecAsync("IF DATABASE_PRINCIPAL_ID('rm_chain_test_user') IS NULL CREATE USER rm_chain_test_user WITHOUT LOGIN;"
+ "ALTER ROLE scadabridge_audit_chainer ADD MEMBER rm_chain_test_user;");
}
finally
{
await MigrateToHeadAsync();
}
await AssertRemovedAsync();
Assert.Equal(1, await ScalarAsync<int>("SELECT CASE WHEN DATABASE_PRINCIPAL_ID('rm_chain_test_user') IS NULL THEN 0 ELSE 1 END;"));
// Re-run Up's SQL against the migrated schema: every statement is guarded.
foreach (var op in new RemoveAuditLogHashChain().UpOperations.OfType<SqlOperation>())
{
await ExecAsync(op.Sql);
}
await AssertRemovedAsync();
}
private async Task AssertRemovedAsync()
{
Assert.Equal(0, await ScalarAsync<int>("SELECT CASE WHEN COL_LENGTH('dbo.AuditLog','ChainSeq') IS NULL THEN 0 ELSE 1 END;"));
Assert.Equal(0, await ScalarAsync<int>("SELECT CASE WHEN COL_LENGTH('dbo.AuditLog','RowHash') IS NULL THEN 0 ELSE 1 END;"));
Assert.Equal(0, await ScalarAsync<int>("SELECT CASE WHEN OBJECT_ID('dbo.AuditLogChainHead','U') IS NULL THEN 0 ELSE 1 END;"));
Assert.Equal(0, await ScalarAsync<int>("SELECT CASE WHEN OBJECT_ID('dbo.AuditLogChainTombstone','U') IS NULL THEN 0 ELSE 1 END;"));
Assert.Equal(0, await ScalarAsync<int>(
"SELECT COUNT(*) FROM sys.indexes WHERE object_id = OBJECT_ID('dbo.AuditLog') AND name IN ('IX_AuditLog_Unchained','IX_AuditLog_ChainSeq');"));
Assert.Equal(0, await ScalarAsync<int>("SELECT CASE WHEN DATABASE_PRINCIPAL_ID('scadabridge_audit_chainer') IS NULL THEN 0 ELSE 1 END;"));
Assert.Equal(0, await ScalarAsync<int>(
"SELECT COUNT(*) FROM sys.database_permissions p JOIN sys.database_principals pr ON pr.principal_id = p.grantee_principal_id "
+ "WHERE pr.name = 'scadabridge_audit_purger' AND p.class = 0 AND p.permission_name = 'ALTER ANY DATASPACE';"));
// The roles that predate the chain stay, with exactly their original AuditLog grants.
Assert.Equal(
new[]
{
"scadabridge_audit_purger|DELETE|GRANT|",
"scadabridge_audit_purger|SELECT|GRANT|",
"scadabridge_audit_writer|DELETE|DENY|",
"scadabridge_audit_writer|INSERT|GRANT|",
"scadabridge_audit_writer|SELECT|GRANT|",
"scadabridge_audit_writer|UPDATE|DENY|",
},
await ReadAuditLogGrantsAsync());
}
private async Task<T> ScalarAsync<T>(string sql)
{
await using var conn = _fixture.OpenConnection();
await using var cmd = new SqlCommand(sql, conn);
return (T)Convert.ChangeType((await cmd.ExecuteScalarAsync())!, typeof(T), System.Globalization.CultureInfo.InvariantCulture);
}
private async Task ExecAsync(string sql)
{
await using var conn = _fixture.OpenConnection();
await using var cmd = new SqlCommand(sql, conn) { CommandTimeout = 300 };
await cmd.ExecuteNonQueryAsync();
}
private async Task<string[]> ReadAuditLogGrantsAsync()
{
await using var conn = _fixture.OpenConnection();
await using var cmd = new SqlCommand(@"
SELECT pr.name, p.permission_name, p.state_desc, COL_NAME(p.major_id, p.minor_id)
FROM sys.database_permissions p JOIN sys.database_principals pr ON pr.principal_id = p.grantee_principal_id
WHERE p.major_id = OBJECT_ID('dbo.AuditLog') AND p.class = 1;", conn);
var list = new List<string>();
await using var reader = await cmd.ExecuteReaderAsync();
while (await reader.ReadAsync())
{
list.Add($"{reader.GetString(0)}|{reader.GetString(1)}|{reader.GetString(2)}|{(reader.IsDBNull(3) ? "" : reader.GetString(3))}");
}
return list.Order(StringComparer.Ordinal).ToArray();
}
private async Task MigrateToAsync(string migration)
{
await using var ctx = new ScadaBridgeDbContext(new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer(_fixture.ConnectionString, sql => sql.CommandTimeout(600)).Options);
await ctx.GetService<IMigrator>().MigrateAsync(migration);
}
private async Task MigrateToHeadAsync()
{
await using var ctx = new ScadaBridgeDbContext(new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer(_fixture.ConnectionString, sql => sql.CommandTimeout(600)).Options);
await ctx.GetService<IMigrator>().MigrateAsync();
}
private static string MigrationBefore(string migration)
{
using var ctx = new ScadaBridgeDbContext(new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer("Server=unused;Database=unused").Options);
var all = ctx.Database.GetMigrations().ToList();
return all[all.IndexOf(migration) - 1];
}
}