feat(audit)!: remove the AuditLog hash chain, head export and two-login maintenance hardening
Owner decision 2026-11-15, at IT security's request, pending a family-wide general solution. Removed: the chaining service, hasher, verifier, chain head/tombstone entities and store, the #139 chain-start cutoff, AuditLog:HashChain (incl. ChainBacklog and HeadExport with its File/Syslog sinks and export service), the audit-chain-head-export health check, the VerifyAuditChain / VerifyAuditChainHeads management commands and the `audit verify-chain` / `audit verify-heads` CLI commands, and the two-login plumbing (AuditLog:MaintenanceConnectionString, AuditMaintenanceContext, the keyed audit-maintenance repository/archive). Purge, archive, retention, partition roll-forward and backfill-source-node all run on the runtime connection again, as before #107. Parquet archive (#42) kept, chain-free: format v2 (ten stored columns, no tombstone file, manifest without chain state), the switch guard is the row count only (51003 gone), verify = data-file size, SHA-256 and row count, and superseded-stale cleanup proves coverage by EventId. Format-v1 archives still list, verify and are cleaned up (their tombstone file too). Migration RemoveAuditLogHashChain drops the chain indexes, the scadabridge_audit_chainer role (members first), both chain tables, ChainSeq/RowHash and the purger's AuditMaintenanceGrants grants; writer/purger roles stay. Down restores them. Every chain-column reference is EXEC-deferred for the idempotent script (#126); FlipAuditLogClusteredKey's two chain index statements get the same deferral, behaviour unchanged, so re-running the idempotent script on a migrated database works. The partition-switch staging table adds the chain columns only when the live table has them, so the new binary purges either schema. Deploy gate: old binaries map the chain columns, so the migration ships with the new binaries, both centrals stopped. A leftover AuditLog:HashChain section or MaintenanceConnectionString (DEV/QA still carry AuditLog__HashChain__Enabled=true) is ignored with one startup warning (RetiredAuditSettingsWarning). Claude-Session: https://claude.ai/code/session_01GT67AFKC45MBz1wPZnVbX8
This commit is contained in:
+66
-89
@@ -1,14 +1,14 @@
|
||||
using Parquet;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using Parquet.Schema;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Maintenance;
|
||||
|
||||
/// <summary>
|
||||
/// Storage-free tests for the AuditLog archive Parquet layout (ScadaBridge#42): rows and
|
||||
/// tombstones round-trip exactly (tick-precise timestamps, NULL vs empty strings, unicode,
|
||||
/// the chain columns), the archived rows re-verify against the archived chain head, and an
|
||||
/// edit made inside the Parquet file breaks that verification.
|
||||
/// Storage-free tests for the AuditLog archive Parquet layout (ScadaBridge#42): rows round-trip
|
||||
/// exactly (tick-precise timestamps, NULL vs empty strings, unicode), and a format-v1 data file
|
||||
/// — written while the hash chain existed, with its <c>ChainSeq</c>/<c>RowHash</c> columns —
|
||||
/// still reads.
|
||||
/// </summary>
|
||||
public class AuditArchiveParquetTests
|
||||
{
|
||||
@@ -19,8 +19,7 @@ public class AuditArchiveParquetTests
|
||||
[Fact]
|
||||
public async Task Rows_RoundTrip_Exactly_AcrossRowGroups()
|
||||
{
|
||||
var (rows, _) = BuildChain(2_503);
|
||||
rows.Add(Row(9_999) with { ChainSeq = null, RowHash = null }); // an unchained row
|
||||
var rows = Enumerable.Range(1, 2_503).Select(Row).ToList();
|
||||
|
||||
var bytes = await WriteAsync(rows, rowGroupSize: 1_000);
|
||||
var back = await ReadAsync(bytes);
|
||||
@@ -28,100 +27,44 @@ public class AuditArchiveParquetTests
|
||||
Assert.Equal(rows.Select(Key), back.Select(Key));
|
||||
await using var reader = await ParquetReader.CreateAsync(new MemoryStream(bytes));
|
||||
Assert.Equal(3, reader.RowGroupCount);
|
||||
Assert.Equal("1", reader.CustomMetadata[AuditArchiveParquet.FormatVersionKey]);
|
||||
Assert.Equal("2", reader.CustomMetadata[AuditArchiveParquet.FormatVersionKey]);
|
||||
Assert.Equal("2025-10", reader.CustomMetadata[AuditArchiveParquet.PartitionKey]);
|
||||
}
|
||||
|
||||
/// <summary>Tombstones round-trip exactly.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
/// <summary>The v2 data file carries the ten stored columns and nothing from the removed hash chain.</summary>
|
||||
[Fact]
|
||||
public async Task Tombstones_RoundTrip_Exactly()
|
||||
public void DataSchema_HasNoChainColumns()
|
||||
{
|
||||
var tombstones = Enumerable.Range(1, 5).Select(i => new AuditArchiveTombstone(
|
||||
Month, i * 3, Enumerable.Repeat((byte)i, 32).ToArray(), Guid.NewGuid(),
|
||||
new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc).AddMilliseconds(i))).ToList();
|
||||
|
||||
var stream = new MemoryStream();
|
||||
await using (var writer = await AuditArchiveParquet.CreateTombstoneWriterAsync(stream, "2025-10", 2))
|
||||
{
|
||||
foreach (var t in tombstones)
|
||||
{
|
||||
await writer.AddAsync(t);
|
||||
}
|
||||
}
|
||||
|
||||
var back = new List<AuditArchiveTombstone>();
|
||||
await foreach (var t in AuditArchiveParquet.ReadTombstonesAsync(new MemoryStream(stream.ToArray())))
|
||||
{
|
||||
back.Add(t);
|
||||
}
|
||||
|
||||
Assert.Equal(
|
||||
tombstones.Select(t => (t.ChainMonth, t.ChainSeq, Convert.ToHexString(t.RowHash), t.EventId, t.PurgedAtUtc)),
|
||||
back.Select(t => (t.ChainMonth, t.ChainSeq, Convert.ToHexString(t.RowHash), t.EventId, t.PurgedAtUtc)));
|
||||
var names = AuditArchiveParquet.DataSchema.DataFields.Select(f => f.Name).ToList();
|
||||
Assert.DoesNotContain("ChainSeq", names);
|
||||
Assert.DoesNotContain("RowHash", names);
|
||||
Assert.Contains("DetailsJson", names); // positive control: the schema was read
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The rows read back from an archive re-verify against the archived head — the same
|
||||
/// check <c>audit archive verify</c> makes long after the database dropped them.
|
||||
/// A format-v1 archive data file (the v2 columns plus <c>ChainSeq</c>/<c>RowHash</c>) reads
|
||||
/// with the v2 reader: the extra columns are ignored and every stored column is intact.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ArchivedRows_ReverifyAgainstTheArchivedHead()
|
||||
public async Task FormatV1DataFile_WithChainColumns_StillReads()
|
||||
{
|
||||
var (rows, head) = BuildChain(50);
|
||||
var verifier = new AuditChainVerifier(Month, head, new Dictionary<long, byte[]>());
|
||||
foreach (var row in await ReadAsync(await WriteAsync(rows, 16)))
|
||||
var rows = Enumerable.Range(1, 40).Select(Row).ToList();
|
||||
var bytes = await WriteV1Async(rows);
|
||||
|
||||
await using (var raw = await ParquetReader.CreateAsync(new MemoryStream(bytes)))
|
||||
{
|
||||
verifier.Accept(row);
|
||||
// Positive control: the file really carries the v1 chain columns.
|
||||
Assert.Contains(raw.Schema.DataFields, f => f.Name == "ChainSeq");
|
||||
}
|
||||
|
||||
var result = verifier.Complete(0);
|
||||
Assert.Equal(AuditChainStatus.Intact, result.Status);
|
||||
Assert.Equal(50, result.RowsVerified);
|
||||
var back = await ReadAsync(bytes);
|
||||
Assert.Equal(rows.Select(Key), back.Select(Key));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Positive control for the test above: a row edited inside the archive (re-written
|
||||
/// file, as a forger would) fails re-verification at that row.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RowEditedInsideTheArchive_FailsReverification()
|
||||
{
|
||||
var (rows, head) = BuildChain(50);
|
||||
rows[17] = rows[17] with { DetailsJson = "{\"kind\":\"ApiCall\",\"forged\":true}" };
|
||||
|
||||
var verifier = new AuditChainVerifier(Month, head, new Dictionary<long, byte[]>());
|
||||
foreach (var row in await ReadAsync(await WriteAsync(rows, 16)))
|
||||
{
|
||||
verifier.Accept(row);
|
||||
}
|
||||
|
||||
var result = verifier.Complete(0);
|
||||
Assert.Equal(AuditChainStatus.Broken, result.Status);
|
||||
Assert.Equal(AuditChainBreakKind.HashMismatch, result.FirstBreak!.Kind);
|
||||
Assert.Equal(18, result.FirstBreak.ChainSeq);
|
||||
}
|
||||
|
||||
internal static (List<AuditChainRow> Rows, AuditChainHeadState Head) BuildChain(int count)
|
||||
{
|
||||
var rows = new List<AuditChainRow>();
|
||||
var previous = AuditChainHasher.GenesisHash.ToArray();
|
||||
for (var i = 1; i <= count; i++)
|
||||
{
|
||||
var row = Row(i);
|
||||
var hash = AuditChainHasher.ComputeRowHash(previous, Month, i, row);
|
||||
rows.Add(row with { ChainSeq = i, RowHash = hash });
|
||||
previous = hash;
|
||||
}
|
||||
|
||||
return (rows, new AuditChainHeadState(count, previous, null, null));
|
||||
}
|
||||
|
||||
private static AuditChainRow Row(int i) => new(
|
||||
private static AuditArchiveRow Row(int i) => new(
|
||||
Guid.NewGuid(),
|
||||
// Odd ticks on purpose: the archive must keep datetime2(7) precision the hash covers.
|
||||
// Odd ticks on purpose: the archive must keep datetime2(7) precision.
|
||||
Month.AddMinutes(i).AddTicks(1_234_567 + i),
|
||||
i % 5 == 0 ? null : i % 7 == 0 ? string.Empty : "api-key-" + i,
|
||||
"ApiOutbound.ApiCall",
|
||||
@@ -132,12 +75,11 @@ public class AuditArchiveParquetTests
|
||||
i % 2 == 0 ? Guid.NewGuid() : null,
|
||||
i % 8 == 0 ? null : $"{{\"kind\":\"ApiCall\",\"status\":\"Delivered\",\"n\":{i}}}");
|
||||
|
||||
private static string Key(AuditChainRow r) => string.Join('|',
|
||||
private static string Key(AuditArchiveRow r) => string.Join('|',
|
||||
r.EventId, r.OccurredAtUtc.Ticks, r.OccurredAtUtc.Kind, r.Actor ?? "<null>", r.Action, r.Outcome, r.Category,
|
||||
r.Target ?? "<null>", r.SourceNode ?? "<null>", r.CorrelationId?.ToString() ?? "<null>", r.DetailsJson ?? "<null>",
|
||||
r.ChainSeq?.ToString() ?? "<null>", r.RowHash is null ? "<null>" : Convert.ToHexString(r.RowHash));
|
||||
r.Target ?? "<null>", r.SourceNode ?? "<null>", r.CorrelationId?.ToString() ?? "<null>", r.DetailsJson ?? "<null>");
|
||||
|
||||
private static async Task<byte[]> WriteAsync(IEnumerable<AuditChainRow> rows, int rowGroupSize)
|
||||
internal static async Task<byte[]> WriteAsync(IEnumerable<AuditArchiveRow> rows, int rowGroupSize)
|
||||
{
|
||||
var stream = new MemoryStream();
|
||||
await using (var writer = await AuditArchiveParquet.CreateDataWriterAsync(stream, "2025-10", rowGroupSize))
|
||||
@@ -151,9 +93,44 @@ public class AuditArchiveParquetTests
|
||||
return stream.ToArray();
|
||||
}
|
||||
|
||||
private static async Task<List<AuditChainRow>> ReadAsync(byte[] bytes)
|
||||
/// <summary>Writes rows in the format-v1 layout: the v2 columns, then the two chain columns.</summary>
|
||||
private static async Task<byte[]> WriteV1Async(IReadOnlyList<AuditArchiveRow> rows)
|
||||
{
|
||||
var rows = new List<AuditChainRow>();
|
||||
var v2 = AuditArchiveParquet.DataSchema.DataFields;
|
||||
var chainSeq = new DataField<long>("ChainSeq", true);
|
||||
var rowHash = new DataField<byte[]>("RowHash", true);
|
||||
var schema = new ParquetSchema(v2.Cast<Field>().Append(chainSeq).Append(rowHash).ToArray());
|
||||
|
||||
var stream = new MemoryStream();
|
||||
await using (var writer = await ParquetWriter.CreateAsync(schema, stream))
|
||||
{
|
||||
writer.CustomMetadata = new Dictionary<string, string>
|
||||
{
|
||||
[AuditArchiveParquet.FormatVersionKey] = "1",
|
||||
[AuditArchiveParquet.PartitionKey] = "2025-10",
|
||||
};
|
||||
using var group = writer.CreateRowGroup();
|
||||
await group.WriteAsync((DataField<string>)v2[0], rows.Select(r => r.EventId.ToString("D")).ToArray());
|
||||
await group.WriteAsync<DateTime>((DateTimeDataField)v2[1], rows.Select(r => r.OccurredAtUtc).ToArray());
|
||||
await group.WriteAsync<long>((DataField<long>)v2[2], rows.Select(r => r.OccurredAtUtc.Ticks).ToArray());
|
||||
await group.WriteAsync((DataField<string>)v2[3], rows.Select(r => r.Actor).ToArray()!);
|
||||
await group.WriteAsync((DataField<string>)v2[4], rows.Select(r => r.Action).ToArray());
|
||||
await group.WriteAsync((DataField<string>)v2[5], rows.Select(r => r.Outcome).ToArray());
|
||||
await group.WriteAsync((DataField<string>)v2[6], rows.Select(r => r.Category).ToArray());
|
||||
await group.WriteAsync((DataField<string>)v2[7], rows.Select(r => r.Target).ToArray()!);
|
||||
await group.WriteAsync((DataField<string>)v2[8], rows.Select(r => r.SourceNode).ToArray()!);
|
||||
await group.WriteAsync((DataField<string>)v2[9], rows.Select(r => r.CorrelationId?.ToString("D")).ToArray()!);
|
||||
await group.WriteAsync((DataField<string>)v2[10], rows.Select(r => r.DetailsJson).ToArray()!);
|
||||
await group.WriteAsync<long>(chainSeq, rows.Select((_, i) => (long?)(i + 1)).ToArray());
|
||||
await group.WriteAsync(rowHash, rows.Select(_ => new byte[32]).ToArray());
|
||||
}
|
||||
|
||||
return stream.ToArray();
|
||||
}
|
||||
|
||||
internal static async Task<List<AuditArchiveRow>> ReadAsync(byte[] bytes)
|
||||
{
|
||||
var rows = new List<AuditArchiveRow>();
|
||||
await foreach (var row in AuditArchiveParquet.ReadRowsAsync(new MemoryStream(bytes)))
|
||||
{
|
||||
rows.Add(row);
|
||||
|
||||
Reference in New Issue
Block a user