Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditLogArchiveCleanupTests.cs
T
Joseph Doherty ed2ed0a367 feat(audit)!: remove the AuditLog hash chain, head export and two-login maintenance hardening
Owner decision 2026-11-15, at IT security's request, pending a family-wide general solution.

Removed: the chaining service, hasher, verifier, chain head/tombstone entities and store, the #139
chain-start cutoff, AuditLog:HashChain (incl. ChainBacklog and HeadExport with its File/Syslog sinks
and export service), the audit-chain-head-export health check, the VerifyAuditChain /
VerifyAuditChainHeads management commands and the `audit verify-chain` / `audit verify-heads` CLI
commands, and the two-login plumbing (AuditLog:MaintenanceConnectionString, AuditMaintenanceContext,
the keyed audit-maintenance repository/archive). Purge, archive, retention, partition roll-forward
and backfill-source-node all run on the runtime connection again, as before #107.

Parquet archive (#42) kept, chain-free: format v2 (ten stored columns, no tombstone file, manifest
without chain state), the switch guard is the row count only (51003 gone), verify = data-file size,
SHA-256 and row count, and superseded-stale cleanup proves coverage by EventId. Format-v1 archives
still list, verify and are cleaned up (their tombstone file too).

Migration RemoveAuditLogHashChain drops the chain indexes, the scadabridge_audit_chainer role
(members first), both chain tables, ChainSeq/RowHash and the purger's AuditMaintenanceGrants grants;
writer/purger roles stay. Down restores them. Every chain-column reference is EXEC-deferred for the
idempotent script (#126); FlipAuditLogClusteredKey's two chain index statements get the same
deferral, behaviour unchanged, so re-running the idempotent script on a migrated database works.
The partition-switch staging table adds the chain columns only when the live table has them, so the
new binary purges either schema. Deploy gate: old binaries map the chain columns, so the migration
ships with the new binaries, both centrals stopped.

A leftover AuditLog:HashChain section or MaintenanceConnectionString (DEV/QA still carry
AuditLog__HashChain__Enabled=true) is ignored with one startup warning (RetiredAuditSettingsWarning).

Claude-Session: https://claude.ai/code/session_01GT67AFKC45MBz1wPZnVbX8
2026-11-15 17:38:22 -05:00

340 lines
16 KiB
C#

using System.Text.Json;
using Microsoft.EntityFrameworkCore;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Maintenance;
/// <summary>
/// SQL Server tests for AuditLog archive cleanup (ScadaBridge#109): a stale archive a retried
/// purge left behind is deleted only once a later purged archive provably holds all its rows;
/// a stale archive with a row no later archive holds, or a sole copy left at <c>Archived</c>, is
/// kept; retention deletes only expired archives whose partition is gone; deletion never
/// follows a path out of the archive folder.
/// </summary>
/// <remarks>Each test owns one month of the shared fixture database and its own archive directory.</remarks>
public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFixture>, IDisposable
{
private readonly MsSqlMigrationFixture _fixture;
private readonly DirectoryInfo _root = Directory.CreateTempSubdirectory("auditlog-cleanup-");
/// <summary>Creates the test class with the shared migrated SQL Server fixture.</summary>
/// <param name="fixture">The shared migration fixture.</param>
public AuditLogArchiveCleanupTests(MsSqlMigrationFixture fixture) => _fixture = fixture;
/// <inheritdoc />
public void Dispose() => _root.Delete(recursive: true);
/// <summary>
/// The retried purge: archive A goes stale (a row arrives), B purges. Dry run reports A as
/// superseded and deletes nothing; the real pass deletes exactly A's files and B still verifies.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task RetriedPurge_StaleArchiveIsDeleted_OnceALaterPurgedArchiveCoversIt()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var dir = Dir("retried");
var month = Month(2027, 1);
await InsertAsync(Enumerable.Range(1, 6).Select(i => NewEvent(month.AddDays(i))));
var stale = await ArchiveAsync(dir, month);
Assert.Equal(6, stale.RowCount);
await InsertAsync([NewEvent(month.AddDays(9))]);
await using (var ctx = CreateContext())
{
await Assert.ThrowsAsync<AuditArchiveStaleException>(() => Archive(ctx, dir).PurgeArchivedPartitionAsync(stale));
}
var fresh = await ArchiveAsync(dir, month);
await using (var ctx = CreateContext())
{
Assert.Equal(7, await Archive(ctx, dir).PurgeArchivedPartitionAsync(fresh));
}
var dry = await CleanupAsync(dir, dryRun: true);
var item = Assert.Single(dry.Items);
Assert.Equal((stale.Name, AuditArchiveCleanupAction.DeletedSuperseded), (item.Name, item.Action));
Assert.True(File.Exists(ManifestPath(dir, stale)));
var real = await CleanupAsync(dir, dryRun: false);
Assert.Equal(AuditArchiveCleanupAction.DeletedSuperseded, Assert.Single(real.Items).Action);
Assert.False(File.Exists(ManifestPath(dir, stale)));
Assert.False(File.Exists(Path.Combine(dir, "2027", stale.Name + ".parquet")));
Assert.False(File.Exists(Path.Combine(dir, "2027", stale.Name + ".tombstones.parquet")));
await using (var ctx = CreateContext())
{
var listed = Assert.Single(await Archive(ctx, dir).ListAsync());
Assert.Equal(fresh.Name, listed.Name);
Assert.True((await Archive(ctx, dir).VerifyAsync(fresh.Name)).Verified);
}
Assert.Empty((await CleanupAsync(dir, dryRun: false)).Items);
}
/// <summary>
/// A stale archive holding a row that a per-channel purge removed before the later archive
/// is the only copy of that row: kept.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task StaleArchive_WithARowNoLaterArchiveHolds_IsKept()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var dir = Dir("kept");
var month = Month(2027, 3);
await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(month.AddDays(i))));
await InsertAsync([NewEvent(month.AddHours(1), AuditChannel.DbOutbound)]);
var stale = await ArchiveAsync(dir, month);
await using (var ctx = CreateContext())
{
Assert.Equal(1, await new AuditLogRepository(ctx).PurgeChannelOlderThanAsync("DbOutbound", month.AddDays(1), 100));
await Assert.ThrowsAsync<AuditArchiveStaleException>(() => Archive(ctx, dir).PurgeArchivedPartitionAsync(stale));
}
var fresh = await ArchiveAsync(dir, month);
await using (var ctx = CreateContext())
{
await Archive(ctx, dir).PurgeArchivedPartitionAsync(fresh);
}
var result = await CleanupAsync(dir, dryRun: false);
var kept = Assert.Single(result.Items);
Assert.Equal((stale.Name, AuditArchiveCleanupAction.KeptStale), (kept.Name, kept.Action));
Assert.Contains("1 row(s) of this archive are not in", kept.Detail);
Assert.True(File.Exists(ManifestPath(dir, stale)));
}
/// <summary>
/// An archive left at <c>Archived</c> although its partition WAS switched out (the node
/// stopped before the manifest update) has no later purged archive: it is the sole copy, kept.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task SoleCopyLeftAtArchived_IsNotTouched()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var dir = Dir("sole");
var month = Month(2027, 4);
await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(month.AddDays(i))));
var manifest = await ArchiveAsync(dir, month);
await using (var ctx = CreateContext())
{
await Archive(ctx, dir).PurgeArchivedPartitionAsync(manifest);
}
await SetStatusAsync(dir, manifest, AuditArchiveStatus.Archived);
Assert.Empty((await CleanupAsync(dir, dryRun: false)).Items);
Assert.True(File.Exists(ManifestPath(dir, manifest)));
}
/// <summary>
/// Retention: 0 never deletes; a purged archive is kept until its month ended more than
/// RetentionDays ago, then deleted; an expired archive whose partition still holds rows is kept.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task Retention_DeletesOnlyExpiredArchivesWhosePartitionIsGone()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var dir = Dir("retention");
var purgedMonth = Month(2027, 5);
await InsertAsync(Enumerable.Range(1, 2).Select(i => NewEvent(purgedMonth.AddDays(i))));
var purged = await ArchiveAsync(dir, purgedMonth);
await using (var ctx = CreateContext())
{
await Archive(ctx, dir).PurgeArchivedPartitionAsync(purged);
}
var liveMonth = Month(2027, 6);
await InsertAsync([NewEvent(liveMonth.AddDays(1))]);
var live = await ArchiveAsync(dir, liveMonth); // never purged: rows still in the table
// As if it were a format-v1 archive (written with the hash chain): it also has a tombstone file.
var legacyTombstones = Path.Combine(dir, "2027", purged.Name + AuditArchiveParquet.LegacyTombstoneSuffix);
await File.WriteAllTextAsync(legacyTombstones, "v1");
var monthEnd = purgedMonth.AddMonths(1);
var farFuture = liveMonth.AddMonths(1).AddDays(1000);
Assert.Empty((await CleanupAsync(dir, dryRun: false, retentionDays: 0, now: farFuture)).Items);
Assert.Empty((await CleanupAsync(dir, dryRun: false, retentionDays: 400, now: monthEnd.AddDays(399))).Items);
var result = await CleanupAsync(dir, dryRun: false, retentionDays: 400, now: farFuture);
var expired = Assert.Single(result.Items);
Assert.Equal((purged.Name, AuditArchiveCleanupAction.DeletedExpired), (expired.Name, expired.Action));
Assert.False(File.Exists(ManifestPath(dir, purged)));
Assert.False(File.Exists(legacyTombstones)); // a format-v1 archive's tombstone file goes with it
Assert.True(File.Exists(ManifestPath(dir, live)));
}
/// <summary>Deletion uses the archive name's own files — never a path from the manifest.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task Deletion_NeverFollowsAManifestPath_OutOfTheArchiveFolder()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var dir = Dir("paths");
var month = Month(2027, 7);
await InsertAsync([NewEvent(month.AddDays(1))]);
var manifest = await ArchiveAsync(dir, month);
await using (var ctx = CreateContext())
{
await Archive(ctx, dir).PurgeArchivedPartitionAsync(manifest);
}
var outside = Path.Combine(_root.FullName, "outside.parquet");
await File.WriteAllTextAsync(outside, "not an archive");
var path = ManifestPath(dir, manifest);
var tampered = JsonSerializer.Deserialize<AuditArchiveManifest>(await File.ReadAllTextAsync(path), Json)! with
{
DataFile = "../../outside.parquet",
};
await File.WriteAllTextAsync(path, JsonSerializer.Serialize(tampered, Json));
var result = await CleanupAsync(dir, dryRun: false, retentionDays: 30, now: month.AddYears(5));
Assert.Equal(AuditArchiveCleanupAction.DeletedExpired, Assert.Single(result.Items).Action);
Assert.True(File.Exists(outside));
Assert.Empty(Directory.EnumerateFiles(Path.Combine(dir, "2027")));
}
/// <summary>
/// ScadaBridge#114: leftovers of an interrupted archive write are removed — but only the
/// writer's own temp / manifest-less data files, old enough, in a year folder. Fresh files,
/// committed archives and foreign files survive; dry run deletes nothing.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task InterruptedWriteOrphans_AreRemoved_OnlyWhenOldAndWriterNamed()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var dir = Dir("orphans");
var month = Month(2027, 6);
await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(month.AddDays(i))));
var committed = await ArchiveAsync(dir, month);
var year = Directory.CreateDirectory(Path.Combine(dir, "2027")).FullName;
const string orphanA = "auditlog-2027-05-20270601T000000001Z";
const string freshB = "auditlog-2027-05-20270601T000000002Z";
var old = DateTime.UtcNow.AddDays(-3);
string Make(string name, DateTime? written = null)
{
var path = Path.Combine(year, name);
File.WriteAllText(path, "x");
File.SetLastWriteTimeUtc(path, written ?? old);
return path;
}
var oldTmpData = Make(orphanA + ".parquet.tmp");
var oldTmpTomb = Make(orphanA + ".tombstones.parquet.tmp"); // a format-v1 writer's temp file
var oldTmpManifest = Make(orphanA + ".manifest.json.tmp");
var oldNoManifestData = Make(orphanA + ".parquet");
var oldNoManifestTomb = Make(orphanA + ".tombstones.parquet");
var freshTmp = Make(freshB + ".parquet.tmp", DateTime.UtcNow.AddHours(-1));
var freshNoManifest = Make(freshB + ".parquet", DateTime.UtcNow.AddHours(-1));
var foreign = Make("notes.tmp");
var wrongName = Make("auditlog-bad.parquet.tmp");
var wrongYear = Make("auditlog-2026-05-20260601T000000003Z.parquet.tmp");
Directory.CreateDirectory(Path.Combine(dir, "misc"));
var otherDir = Path.Combine(dir, "misc", orphanA + ".parquet.tmp");
File.WriteAllText(otherDir, "x");
File.SetLastWriteTimeUtc(otherDir, old);
var committedFiles = new[] { ManifestPath(dir, committed), Path.Combine(year, committed.Name + ".parquet") };
foreach (var f in committedFiles)
{
File.SetLastWriteTimeUtc(f, old);
}
var removed = new[] { oldTmpData, oldTmpTomb, oldTmpManifest, oldNoManifestData, oldNoManifestTomb };
var dry = await CleanupAsync(dir, dryRun: true);
Assert.Equal(5, dry.Items.Count(i => i.Action == AuditArchiveCleanupAction.DeletedOrphan));
Assert.All(removed, f => Assert.True(File.Exists(f)));
var real = await CleanupAsync(dir, dryRun: false);
Assert.Equal(5, real.Items.Count(i => i.Action == AuditArchiveCleanupAction.DeletedOrphan));
Assert.All(removed, f => Assert.False(File.Exists(f)));
Assert.All(new[] { freshTmp, freshNoManifest, foreign, wrongName, wrongYear, otherDir }.Concat(committedFiles), f => Assert.True(File.Exists(f), f));
Assert.Empty((await CleanupAsync(dir, dryRun: false)).Items);
}
private static readonly JsonSerializerOptions Json = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase };
private sealed class FixedClock(DateTime now) : TimeProvider
{
public override DateTimeOffset GetUtcNow() => new(DateTime.SpecifyKind(now, DateTimeKind.Utc));
}
private string Dir(string name) => Directory.CreateDirectory(Path.Combine(_root.FullName, name)).FullName;
private static string ManifestPath(string dir, AuditArchiveManifest m) => Path.Combine(dir, m.Partition[..4], m.Name + ".manifest.json");
private static async Task SetStatusAsync(string dir, AuditArchiveManifest manifest, AuditArchiveStatus status)
{
var path = ManifestPath(dir, manifest);
var current = JsonSerializer.Deserialize<AuditArchiveManifest>(await File.ReadAllTextAsync(path), Json)!;
await File.WriteAllTextAsync(path, JsonSerializer.Serialize(current with { Status = status, PurgedAtUtc = null }, Json));
}
private static AuditLogPartitionArchive Archive(
ScadaBridgeDbContext ctx, string dir, int retentionDays = 0, DateTime? now = null) => new(ctx, new AuditLogArchiveOptions
{
Enabled = true,
Directory = dir,
RowGroupSize = 1_000,
RetentionDays = retentionDays,
}, time: now is null ? null : new FixedClock(now.Value));
private async Task<AuditArchiveManifest> ArchiveAsync(string dir, DateTime month)
{
await using var ctx = CreateContext();
var manifest = await Archive(ctx, dir).ArchivePartitionAsync(month);
await Task.Delay(5); // distinct millisecond archive names
return manifest;
}
private async Task<AuditArchiveCleanupResult> CleanupAsync(string dir, bool dryRun, int retentionDays = 0, DateTime? now = null)
{
await using var ctx = CreateContext();
return await Archive(ctx, dir, retentionDays, now).CleanupAsync(dryRun);
}
private async Task InsertAsync(IEnumerable<AuditEvent> events)
{
await using var ctx = CreateContext();
var repo = new AuditLogRepository(ctx);
foreach (var e in events)
{
await repo.InsertIfNotExistsAsync(e);
}
}
private static DateTime Month(int year, int month) => new(year, month, 1, 0, 0, 0, DateTimeKind.Utc);
private ScadaBridgeDbContext CreateContext()
=> new(new DbContextOptionsBuilder<ScadaBridgeDbContext>().UseSqlServer(_fixture.ConnectionString).Options);
private static AuditEvent NewEvent(DateTime occurredAtUtc, AuditChannel channel = AuditChannel.ApiOutbound) =>
ScadaBridgeAuditEventFactory.Create(
channel: channel,
kind: channel == AuditChannel.DbOutbound ? AuditKind.DbWrite : AuditKind.ApiCall,
status: AuditStatus.Delivered,
occurredAtUtc: occurredAtUtc,
actor: "cleanup-test",
target: "erp",
sourceNode: "central-a",
sourceSiteId: "cleanup-test",
correlationId: Guid.NewGuid());
}