Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditArchiveParquetTests.cs
T
Joseph Doherty ed2ed0a367 feat(audit)!: remove the AuditLog hash chain, head export and two-login maintenance hardening
Owner decision 2026-11-15, at IT security's request, pending a family-wide general solution.

Removed: the chaining service, hasher, verifier, chain head/tombstone entities and store, the #139
chain-start cutoff, AuditLog:HashChain (incl. ChainBacklog and HeadExport with its File/Syslog sinks
and export service), the audit-chain-head-export health check, the VerifyAuditChain /
VerifyAuditChainHeads management commands and the `audit verify-chain` / `audit verify-heads` CLI
commands, and the two-login plumbing (AuditLog:MaintenanceConnectionString, AuditMaintenanceContext,
the keyed audit-maintenance repository/archive). Purge, archive, retention, partition roll-forward
and backfill-source-node all run on the runtime connection again, as before #107.

Parquet archive (#42) kept, chain-free: format v2 (ten stored columns, no tombstone file, manifest
without chain state), the switch guard is the row count only (51003 gone), verify = data-file size,
SHA-256 and row count, and superseded-stale cleanup proves coverage by EventId. Format-v1 archives
still list, verify and are cleaned up (their tombstone file too).

Migration RemoveAuditLogHashChain drops the chain indexes, the scadabridge_audit_chainer role
(members first), both chain tables, ChainSeq/RowHash and the purger's AuditMaintenanceGrants grants;
writer/purger roles stay. Down restores them. Every chain-column reference is EXEC-deferred for the
idempotent script (#126); FlipAuditLogClusteredKey's two chain index statements get the same
deferral, behaviour unchanged, so re-running the idempotent script on a migrated database works.
The partition-switch staging table adds the chain columns only when the live table has them, so the
new binary purges either schema. Deploy gate: old binaries map the chain columns, so the migration
ships with the new binaries, both centrals stopped.

A leftover AuditLog:HashChain section or MaintenanceConnectionString (DEV/QA still carry
AuditLog__HashChain__Enabled=true) is ignored with one startup warning (RetiredAuditSettingsWarning).

Claude-Session: https://claude.ai/code/session_01GT67AFKC45MBz1wPZnVbX8
2026-11-15 17:38:22 -05:00

142 lines
6.6 KiB
C#

using Parquet;
using Parquet.Schema;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Maintenance;
/// <summary>
/// Storage-free tests for the AuditLog archive Parquet layout (ScadaBridge#42): rows round-trip
/// exactly (tick-precise timestamps, NULL vs empty strings, unicode), and a format-v1 data file
/// — written while the hash chain existed, with its <c>ChainSeq</c>/<c>RowHash</c> columns —
/// still reads.
/// </summary>
public class AuditArchiveParquetTests
{
private static readonly DateTime Month = new(2025, 10, 1, 0, 0, 0, DateTimeKind.Utc);
/// <summary>Rows written to the archive layout read back equal, column for column, across row groups.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task Rows_RoundTrip_Exactly_AcrossRowGroups()
{
var rows = Enumerable.Range(1, 2_503).Select(Row).ToList();
var bytes = await WriteAsync(rows, rowGroupSize: 1_000);
var back = await ReadAsync(bytes);
Assert.Equal(rows.Select(Key), back.Select(Key));
await using var reader = await ParquetReader.CreateAsync(new MemoryStream(bytes));
Assert.Equal(3, reader.RowGroupCount);
Assert.Equal("2", reader.CustomMetadata[AuditArchiveParquet.FormatVersionKey]);
Assert.Equal("2025-10", reader.CustomMetadata[AuditArchiveParquet.PartitionKey]);
}
/// <summary>The v2 data file carries the ten stored columns and nothing from the removed hash chain.</summary>
[Fact]
public void DataSchema_HasNoChainColumns()
{
var names = AuditArchiveParquet.DataSchema.DataFields.Select(f => f.Name).ToList();
Assert.DoesNotContain("ChainSeq", names);
Assert.DoesNotContain("RowHash", names);
Assert.Contains("DetailsJson", names); // positive control: the schema was read
}
/// <summary>
/// A format-v1 archive data file (the v2 columns plus <c>ChainSeq</c>/<c>RowHash</c>) reads
/// with the v2 reader: the extra columns are ignored and every stored column is intact.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task FormatV1DataFile_WithChainColumns_StillReads()
{
var rows = Enumerable.Range(1, 40).Select(Row).ToList();
var bytes = await WriteV1Async(rows);
await using (var raw = await ParquetReader.CreateAsync(new MemoryStream(bytes)))
{
// Positive control: the file really carries the v1 chain columns.
Assert.Contains(raw.Schema.DataFields, f => f.Name == "ChainSeq");
}
var back = await ReadAsync(bytes);
Assert.Equal(rows.Select(Key), back.Select(Key));
}
private static AuditArchiveRow Row(int i) => new(
Guid.NewGuid(),
// Odd ticks on purpose: the archive must keep datetime2(7) precision.
Month.AddMinutes(i).AddTicks(1_234_567 + i),
i % 5 == 0 ? null : i % 7 == 0 ? string.Empty : "api-key-" + i,
"ApiOutbound.ApiCall",
i % 3 == 0 ? "Failure" : "Success",
"ApiOutbound",
i % 4 == 0 ? null : "erp-ü-日本",
i % 6 == 0 ? null : "central-a",
i % 2 == 0 ? Guid.NewGuid() : null,
i % 8 == 0 ? null : $"{{\"kind\":\"ApiCall\",\"status\":\"Delivered\",\"n\":{i}}}");
private static string Key(AuditArchiveRow r) => string.Join('|',
r.EventId, r.OccurredAtUtc.Ticks, r.OccurredAtUtc.Kind, r.Actor ?? "<null>", r.Action, r.Outcome, r.Category,
r.Target ?? "<null>", r.SourceNode ?? "<null>", r.CorrelationId?.ToString() ?? "<null>", r.DetailsJson ?? "<null>");
internal static async Task<byte[]> WriteAsync(IEnumerable<AuditArchiveRow> rows, int rowGroupSize)
{
var stream = new MemoryStream();
await using (var writer = await AuditArchiveParquet.CreateDataWriterAsync(stream, "2025-10", rowGroupSize))
{
foreach (var row in rows)
{
await writer.AddAsync(row);
}
}
return stream.ToArray();
}
/// <summary>Writes rows in the format-v1 layout: the v2 columns, then the two chain columns.</summary>
private static async Task<byte[]> WriteV1Async(IReadOnlyList<AuditArchiveRow> rows)
{
var v2 = AuditArchiveParquet.DataSchema.DataFields;
var chainSeq = new DataField<long>("ChainSeq", true);
var rowHash = new DataField<byte[]>("RowHash", true);
var schema = new ParquetSchema(v2.Cast<Field>().Append(chainSeq).Append(rowHash).ToArray());
var stream = new MemoryStream();
await using (var writer = await ParquetWriter.CreateAsync(schema, stream))
{
writer.CustomMetadata = new Dictionary<string, string>
{
[AuditArchiveParquet.FormatVersionKey] = "1",
[AuditArchiveParquet.PartitionKey] = "2025-10",
};
using var group = writer.CreateRowGroup();
await group.WriteAsync((DataField<string>)v2[0], rows.Select(r => r.EventId.ToString("D")).ToArray());
await group.WriteAsync<DateTime>((DateTimeDataField)v2[1], rows.Select(r => r.OccurredAtUtc).ToArray());
await group.WriteAsync<long>((DataField<long>)v2[2], rows.Select(r => r.OccurredAtUtc.Ticks).ToArray());
await group.WriteAsync((DataField<string>)v2[3], rows.Select(r => r.Actor).ToArray()!);
await group.WriteAsync((DataField<string>)v2[4], rows.Select(r => r.Action).ToArray());
await group.WriteAsync((DataField<string>)v2[5], rows.Select(r => r.Outcome).ToArray());
await group.WriteAsync((DataField<string>)v2[6], rows.Select(r => r.Category).ToArray());
await group.WriteAsync((DataField<string>)v2[7], rows.Select(r => r.Target).ToArray()!);
await group.WriteAsync((DataField<string>)v2[8], rows.Select(r => r.SourceNode).ToArray()!);
await group.WriteAsync((DataField<string>)v2[9], rows.Select(r => r.CorrelationId?.ToString("D")).ToArray()!);
await group.WriteAsync((DataField<string>)v2[10], rows.Select(r => r.DetailsJson).ToArray()!);
await group.WriteAsync<long>(chainSeq, rows.Select((_, i) => (long?)(i + 1)).ToArray());
await group.WriteAsync(rowHash, rows.Select(_ => new byte[32]).ToArray());
}
return stream.ToArray();
}
internal static async Task<List<AuditArchiveRow>> ReadAsync(byte[] bytes)
{
var rows = new List<AuditArchiveRow>();
await foreach (var row in AuditArchiveParquet.ReadRowsAsync(new MemoryStream(bytes)))
{
rows.Add(row);
}
return rows;
}
}