feat(audit)!: remove the AuditLog hash chain, head export and two-login maintenance hardening

Owner decision 2026-11-15, at IT security's request, pending a family-wide general solution.

Removed: the chaining service, hasher, verifier, chain head/tombstone entities and store, the #139
chain-start cutoff, AuditLog:HashChain (incl. ChainBacklog and HeadExport with its File/Syslog sinks
and export service), the audit-chain-head-export health check, the VerifyAuditChain /
VerifyAuditChainHeads management commands and the `audit verify-chain` / `audit verify-heads` CLI
commands, and the two-login plumbing (AuditLog:MaintenanceConnectionString, AuditMaintenanceContext,
the keyed audit-maintenance repository/archive). Purge, archive, retention, partition roll-forward
and backfill-source-node all run on the runtime connection again, as before #107.

Parquet archive (#42) kept, chain-free: format v2 (ten stored columns, no tombstone file, manifest
without chain state), the switch guard is the row count only (51003 gone), verify = data-file size,
SHA-256 and row count, and superseded-stale cleanup proves coverage by EventId. Format-v1 archives
still list, verify and are cleaned up (their tombstone file too).

Migration RemoveAuditLogHashChain drops the chain indexes, the scadabridge_audit_chainer role
(members first), both chain tables, ChainSeq/RowHash and the purger's AuditMaintenanceGrants grants;
writer/purger roles stay. Down restores them. Every chain-column reference is EXEC-deferred for the
idempotent script (#126); FlipAuditLogClusteredKey's two chain index statements get the same
deferral, behaviour unchanged, so re-running the idempotent script on a migrated database works.
The partition-switch staging table adds the chain columns only when the live table has them, so the
new binary purges either schema. Deploy gate: old binaries map the chain columns, so the migration
ships with the new binaries, both centrals stopped.

A leftover AuditLog:HashChain section or MaintenanceConnectionString (DEV/QA still carry
AuditLog__HashChain__Enabled=true) is ignored with one startup warning (RetiredAuditSettingsWarning).

Claude-Session: https://claude.ai/code/session_01GT67AFKC45MBz1wPZnVbX8
This commit is contained in:
Joseph Doherty
2026-11-15 17:37:33 -05:00
parent 125db03232
commit ed2ed0a367
82 changed files with 3203 additions and 7603 deletions
@@ -31,9 +31,8 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFi
public void Dispose() => _root.Delete(recursive: true);
/// <summary>
/// The retried purge: archive A goes stale (a row arrives and the chain advances), B purges.
/// Dry run reports A as superseded and deletes nothing; the real pass deletes exactly A's
/// three files and B still verifies.
/// The retried purge: archive A goes stale (a row arrives), B purges. Dry run reports A as
/// superseded and deletes nothing; the real pass deletes exactly A's files and B still verifies.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
@@ -42,14 +41,11 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFi
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var dir = Dir("retried");
var month = Month(2027, 1);
await InsertAsync(Enumerable.Range(1, 5).Select(i => NewEvent(month.AddDays(i))));
await DrainAsync();
await InsertAsync([NewEvent(month.AddDays(7))]); // unchained in A
await InsertAsync(Enumerable.Range(1, 6).Select(i => NewEvent(month.AddDays(i))));
var stale = await ArchiveAsync(dir, month);
Assert.Equal(1, stale.UnchainedRowCount);
Assert.Equal(6, stale.RowCount);
await InsertAsync([NewEvent(month.AddDays(9))]);
await DrainAsync(); // A's unchained row is now chained; the chain moved past A's end
await using (var ctx = CreateContext())
{
await Assert.ThrowsAsync<AuditArchiveStaleException>(() => Archive(ctx, dir).PurgeArchivedPartitionAsync(stale));
@@ -83,8 +79,8 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFi
}
/// <summary>
/// A stale archive holding an unchained row that a per-channel purge removed before the
/// later archive (no tombstone for an unchained row) is the only copy of that row: kept.
/// A stale archive holding a row that a per-channel purge removed before the later archive
/// is the only copy of that row: kept.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
@@ -94,8 +90,7 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFi
var dir = Dir("kept");
var month = Month(2027, 3);
await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(month.AddDays(i))));
await DrainAsync();
await InsertAsync([NewEvent(month.AddHours(1), AuditChannel.DbOutbound)]); // unchained
await InsertAsync([NewEvent(month.AddHours(1), AuditChannel.DbOutbound)]);
var stale = await ArchiveAsync(dir, month);
await using (var ctx = CreateContext())
@@ -114,7 +109,7 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFi
var kept = Assert.Single(result.Items);
Assert.Equal((stale.Name, AuditArchiveCleanupAction.KeptStale), (kept.Name, kept.Action));
Assert.Contains("unchained row", kept.Detail);
Assert.Contains("1 row(s) of this archive are not in", kept.Detail);
Assert.True(File.Exists(ManifestPath(dir, stale)));
}
@@ -130,7 +125,6 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFi
var dir = Dir("sole");
var month = Month(2027, 4);
await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(month.AddDays(i))));
await DrainAsync();
var manifest = await ArchiveAsync(dir, month);
await using (var ctx = CreateContext())
{
@@ -165,6 +159,10 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFi
await InsertAsync([NewEvent(liveMonth.AddDays(1))]);
var live = await ArchiveAsync(dir, liveMonth); // never purged: rows still in the table
// As if it were a format-v1 archive (written with the hash chain): it also has a tombstone file.
var legacyTombstones = Path.Combine(dir, "2027", purged.Name + AuditArchiveParquet.LegacyTombstoneSuffix);
await File.WriteAllTextAsync(legacyTombstones, "v1");
var monthEnd = purgedMonth.AddMonths(1);
var farFuture = liveMonth.AddMonths(1).AddDays(1000);
Assert.Empty((await CleanupAsync(dir, dryRun: false, retentionDays: 0, now: farFuture)).Items);
@@ -175,10 +173,11 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFi
var expired = Assert.Single(result.Items);
Assert.Equal((purged.Name, AuditArchiveCleanupAction.DeletedExpired), (expired.Name, expired.Action));
Assert.False(File.Exists(ManifestPath(dir, purged)));
Assert.False(File.Exists(legacyTombstones)); // a format-v1 archive's tombstone file goes with it
Assert.True(File.Exists(ManifestPath(dir, live)));
}
/// <summary>Deletion uses the archive name's own three files — never a path from the manifest.</summary>
/// <summary>Deletion uses the archive name's own files — never a path from the manifest.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task Deletion_NeverFollowsAManifestPath_OutOfTheArchiveFolder()
@@ -199,7 +198,6 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFi
var tampered = JsonSerializer.Deserialize<AuditArchiveManifest>(await File.ReadAllTextAsync(path), Json)! with
{
DataFile = "../../outside.parquet",
TombstoneFile = "../../outside.parquet",
};
await File.WriteAllTextAsync(path, JsonSerializer.Serialize(tampered, Json));
@@ -223,7 +221,6 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFi
var dir = Dir("orphans");
var month = Month(2027, 6);
await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(month.AddDays(i))));
await DrainAsync();
var committed = await ArchiveAsync(dir, month);
var year = Directory.CreateDirectory(Path.Combine(dir, "2027")).FullName;
@@ -239,7 +236,7 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFi
}
var oldTmpData = Make(orphanA + ".parquet.tmp");
var oldTmpTomb = Make(orphanA + ".tombstones.parquet.tmp");
var oldTmpTomb = Make(orphanA + ".tombstones.parquet.tmp"); // a format-v1 writer's temp file
var oldTmpManifest = Make(orphanA + ".manifest.json.tmp");
var oldNoManifestData = Make(orphanA + ".parquet");
var oldNoManifestTomb = Make(orphanA + ".tombstones.parquet");
@@ -252,7 +249,7 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFi
var otherDir = Path.Combine(dir, "misc", orphanA + ".parquet.tmp");
File.WriteAllText(otherDir, "x");
File.SetLastWriteTimeUtc(otherDir, old);
var committedFiles = new[] { ManifestPath(dir, committed), Path.Combine(year, committed.Name + ".parquet"), Path.Combine(year, committed.Name + ".tombstones.parquet") };
var committedFiles = new[] { ManifestPath(dir, committed), Path.Combine(year, committed.Name + ".parquet") };
foreach (var f in committedFiles)
{
File.SetLastWriteTimeUtc(f, old);
@@ -323,22 +320,6 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFi
}
}
private async Task DrainAsync()
{
await using var ctx = CreateContext();
var chain = new AuditLogHashChain(ctx);
for (var i = 0; i < 1000; i++)
{
var r = await chain.ChainPendingAsync(500, chainBacklog: true);
if (r.RowsChained == 0 && !r.LockBusy)
{
return;
}
}
throw new InvalidOperationException("Chain did not drain.");
}
private static DateTime Month(int year, int month) => new(year, month, 1, 0, 0, 0, DateTimeKind.Utc);
private ScadaBridgeDbContext CreateContext()