using Parquet; using Parquet.Schema; using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance; namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Maintenance; /// /// Storage-free tests for the AuditLog archive Parquet layout (ScadaBridge#42): rows round-trip /// exactly (tick-precise timestamps, NULL vs empty strings, unicode), and a format-v1 data file /// — written while the hash chain existed, with its ChainSeq/RowHash columns — /// still reads. /// public class AuditArchiveParquetTests { private static readonly DateTime Month = new(2025, 10, 1, 0, 0, 0, DateTimeKind.Utc); /// Rows written to the archive layout read back equal, column for column, across row groups. /// A task that represents the asynchronous operation. [Fact] public async Task Rows_RoundTrip_Exactly_AcrossRowGroups() { var rows = Enumerable.Range(1, 2_503).Select(Row).ToList(); var bytes = await WriteAsync(rows, rowGroupSize: 1_000); var back = await ReadAsync(bytes); Assert.Equal(rows.Select(Key), back.Select(Key)); await using var reader = await ParquetReader.CreateAsync(new MemoryStream(bytes)); Assert.Equal(3, reader.RowGroupCount); Assert.Equal("2", reader.CustomMetadata[AuditArchiveParquet.FormatVersionKey]); Assert.Equal("2025-10", reader.CustomMetadata[AuditArchiveParquet.PartitionKey]); } /// The v2 data file carries the ten stored columns and nothing from the removed hash chain. [Fact] public void DataSchema_HasNoChainColumns() { var names = AuditArchiveParquet.DataSchema.DataFields.Select(f => f.Name).ToList(); Assert.DoesNotContain("ChainSeq", names); Assert.DoesNotContain("RowHash", names); Assert.Contains("DetailsJson", names); // positive control: the schema was read } /// /// A format-v1 archive data file (the v2 columns plus ChainSeq/RowHash) reads /// with the v2 reader: the extra columns are ignored and every stored column is intact. /// /// A task that represents the asynchronous operation. [Fact] public async Task FormatV1DataFile_WithChainColumns_StillReads() { var rows = Enumerable.Range(1, 40).Select(Row).ToList(); var bytes = await WriteV1Async(rows); await using (var raw = await ParquetReader.CreateAsync(new MemoryStream(bytes))) { // Positive control: the file really carries the v1 chain columns. Assert.Contains(raw.Schema.DataFields, f => f.Name == "ChainSeq"); } var back = await ReadAsync(bytes); Assert.Equal(rows.Select(Key), back.Select(Key)); } private static AuditArchiveRow Row(int i) => new( Guid.NewGuid(), // Odd ticks on purpose: the archive must keep datetime2(7) precision. Month.AddMinutes(i).AddTicks(1_234_567 + i), i % 5 == 0 ? null : i % 7 == 0 ? string.Empty : "api-key-" + i, "ApiOutbound.ApiCall", i % 3 == 0 ? "Failure" : "Success", "ApiOutbound", i % 4 == 0 ? null : "erp-ü-日本", i % 6 == 0 ? null : "central-a", i % 2 == 0 ? Guid.NewGuid() : null, i % 8 == 0 ? null : $"{{\"kind\":\"ApiCall\",\"status\":\"Delivered\",\"n\":{i}}}"); private static string Key(AuditArchiveRow r) => string.Join('|', r.EventId, r.OccurredAtUtc.Ticks, r.OccurredAtUtc.Kind, r.Actor ?? "", r.Action, r.Outcome, r.Category, r.Target ?? "", r.SourceNode ?? "", r.CorrelationId?.ToString() ?? "", r.DetailsJson ?? ""); internal static async Task WriteAsync(IEnumerable rows, int rowGroupSize) { var stream = new MemoryStream(); await using (var writer = await AuditArchiveParquet.CreateDataWriterAsync(stream, "2025-10", rowGroupSize)) { foreach (var row in rows) { await writer.AddAsync(row); } } return stream.ToArray(); } /// Writes rows in the format-v1 layout: the v2 columns, then the two chain columns. private static async Task WriteV1Async(IReadOnlyList rows) { var v2 = AuditArchiveParquet.DataSchema.DataFields; var chainSeq = new DataField("ChainSeq", true); var rowHash = new DataField("RowHash", true); var schema = new ParquetSchema(v2.Cast().Append(chainSeq).Append(rowHash).ToArray()); var stream = new MemoryStream(); await using (var writer = await ParquetWriter.CreateAsync(schema, stream)) { writer.CustomMetadata = new Dictionary { [AuditArchiveParquet.FormatVersionKey] = "1", [AuditArchiveParquet.PartitionKey] = "2025-10", }; using var group = writer.CreateRowGroup(); await group.WriteAsync((DataField)v2[0], rows.Select(r => r.EventId.ToString("D")).ToArray()); await group.WriteAsync((DateTimeDataField)v2[1], rows.Select(r => r.OccurredAtUtc).ToArray()); await group.WriteAsync((DataField)v2[2], rows.Select(r => r.OccurredAtUtc.Ticks).ToArray()); await group.WriteAsync((DataField)v2[3], rows.Select(r => r.Actor).ToArray()!); await group.WriteAsync((DataField)v2[4], rows.Select(r => r.Action).ToArray()); await group.WriteAsync((DataField)v2[5], rows.Select(r => r.Outcome).ToArray()); await group.WriteAsync((DataField)v2[6], rows.Select(r => r.Category).ToArray()); await group.WriteAsync((DataField)v2[7], rows.Select(r => r.Target).ToArray()!); await group.WriteAsync((DataField)v2[8], rows.Select(r => r.SourceNode).ToArray()!); await group.WriteAsync((DataField)v2[9], rows.Select(r => r.CorrelationId?.ToString("D")).ToArray()!); await group.WriteAsync((DataField)v2[10], rows.Select(r => r.DetailsJson).ToArray()!); await group.WriteAsync(chainSeq, rows.Select((_, i) => (long?)(i + 1)).ToArray()); await group.WriteAsync(rowHash, rows.Select(_ => new byte[32]).ToArray()); } return stream.ToArray(); } internal static async Task> ReadAsync(byte[] bytes) { var rows = new List(); await foreach (var row in AuditArchiveParquet.ReadRowsAsync(new MemoryStream(bytes))) { rows.Add(row); } return rows; } }