feat(management): parquet audit export and audit archive list/verify (ScadaBridge#42)

GET /api/audit/export?format=parquet returned 501. It now streams the same 24 columns as csv/jsonl (AuditExportRow, CsvHeader order) as one Parquet file, one 10,000-row row group at a time. Each row group is encoded into a small non-seekable staging buffer and copied to the response asynchronously, because Kestrel rejects the synchronous writes the Parquet writer issues. New management commands, with CLI parity: - ListAuditArchives -> `audit archive list` (AuditArchiveListResult: enabled flag + every manifest). - VerifyAuditArchive(Name) -> `audit archive verify --name` (files, chain, purge anchor); exit 0 verified, 3 not verified, 1 error, 2 unauthorized. Both carry verify-chain's gate: OperationalAudit roles (Administrator, Viewer), site-scoped non-Administrator refused because an archive spans every site. VerifyAuditArchive uses the long-running Ask timeout. Error codes ARCHIVE_NOT_CONFIGURED, INVALID_ARGUMENT, NOT_FOUND. RequiredRoleMatrixTests freezes both gates; ManagementSurfaceParityTests now probes `audit archive list|verify` alongside `audit verify-chain`. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-09 17:00:58 -05:00
parent 7be4f19861
commit d1fc336e18
17 changed files with 982 additions and 42 deletions
@@ -0,0 +1,147 @@
+using System.Text.Json;
+using Akka.Actor;
+using Akka.TestKit.Xunit2;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+using NSubstitute;
+using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
+using ZB.MOM.WW.ScadaBridge.Commons.Messages.Management;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
+
+namespace ZB.MOM.WW.ScadaBridge.ManagementService.Tests;
+
+/// <summary>
+/// The <c>ListAuditArchives</c> / <c>VerifyAuditArchive</c> management commands
+/// (ScadaBridge#42): role gate, site-scope refusal, an unconfigured archive directory, bad
+/// names, and the results reaching the wire with enums as names.
+/// </summary>
+public class AuditArchiveCommandTests : TestKit
+{
+    private const string Name = "auditlog-2025-10-20261109T120000123Z";
+
+    private readonly IAuditLogArchive _archive = Substitute.For<IAuditLogArchive>();
+
+    /// <summary>Configures the archive substitute as a configured, enabled archive.</summary>
+    public AuditArchiveCommandTests()
+    {
+        _archive.IsConfigured.Returns(true);
+        _archive.IsEnabled.Returns(true);
+    }
+
+    private IActorRef CreateActor(bool registerArchive = true)
+    {
+        var services = new ServiceCollection().AddTestConfigurationTransaction();
+        if (registerArchive)
+        {
+            services.AddScoped(_ => _archive);
+        }
+
+        return Sys.ActorOf(Props.Create(() => new ManagementActor(
+            services.BuildServiceProvider(), NullLogger<ManagementActor>.Instance)));
+    }
+
+    private static ManagementEnvelope Envelope(object command, string[] sites, params string[] roles) =>
+        new(new AuthenticatedUser("u", "U", roles, sites), command, Guid.NewGuid().ToString("N"));
+
+    private static AuditArchiveManifest Manifest() => new(
+        1, Name, new DateTime(2025, 10, 1, 0, 0, 0, DateTimeKind.Utc), "2025-10",
+        new DateTime(2026, 11, 9, 12, 0, 0, DateTimeKind.Utc), AuditArchiveStatus.Purged,
+        new DateTime(2026, 11, 9, 12, 0, 1, DateTimeKind.Utc), 10, 9, 1, null, null,
+        Name + ".parquet", "ab", 1234, Name + ".tombstones.parquet", "cd", 0,
+        [new AuditArchiveChainState("2025-10", 9, new string('0', 64), null, null, 0)]);
+
+    /// <summary>A Viewer lists the archives; status renders as a name.</summary>
+    [Fact]
+    public void Viewer_ListsArchives()
+    {
+        _archive.ListAsync(Arg.Any<CancellationToken>()).Returns([Manifest()]);
+
+        CreateActor().Tell(Envelope(new ListAuditArchivesCommand(), [], "Viewer"));
+
+        var response = ExpectMsg<ManagementSuccess>(TimeSpan.FromSeconds(5));
+        using var json = JsonDocument.Parse(response.JsonData);
+        Assert.True(json.RootElement.GetProperty("enabled").GetBoolean());
+        var archive = json.RootElement.GetProperty("archives")[0];
+        Assert.Equal(Name, archive.GetProperty("name").GetString());
+        Assert.Equal("Purged", archive.GetProperty("status").GetString());
+    }
+
+    /// <summary>Verification results reach the wire with the anchor and chain status as names.</summary>
+    [Fact]
+    public void Viewer_GetsVerificationResult()
+    {
+        var chain = new AuditChainVerificationResult("2025-10", AuditChainStatus.Intact, 9, 0, null, 9, 1, null);
+        _archive.VerifyAsync(Name, Arg.Any<CancellationToken>()).Returns(new AuditArchiveVerificationResult(
+            Name, "2025-10", AuditArchiveStatus.Purged, true, true, null, 10,
+            [new AuditArchiveChainVerification(chain, AuditArchiveAnchorStatus.Matches, "ends at the anchor")]));
+
+        CreateActor().Tell(Envelope(new VerifyAuditArchiveCommand(Name), [], "Viewer"));
+
+        var response = ExpectMsg<ManagementSuccess>(TimeSpan.FromSeconds(5));
+        using var json = JsonDocument.Parse(response.JsonData);
+        Assert.True(json.RootElement.GetProperty("verified").GetBoolean());
+        var c = json.RootElement.GetProperty("chains")[0];
+        Assert.Equal("Matches", c.GetProperty("anchor").GetString());
+        Assert.Equal("Intact", c.GetProperty("chain").GetProperty("status").GetString());
+    }
+
+    /// <summary>Designer/Deployer without an OperationalAudit role are refused both commands.</summary>
+    [Fact]
+    public void NonAuditRole_IsUnauthorized()
+    {
+        var actor = CreateActor();
+        actor.Tell(Envelope(new ListAuditArchivesCommand(), [], "Designer", "Deployer"));
+        ExpectMsg<ManagementUnauthorized>(TimeSpan.FromSeconds(5));
+        actor.Tell(Envelope(new VerifyAuditArchiveCommand(Name), [], "Designer", "Deployer"));
+        ExpectMsg<ManagementUnauthorized>(TimeSpan.FromSeconds(5));
+        _archive.DidNotReceiveWithAnyArgs().ListAsync(default);
+        _archive.DidNotReceiveWithAnyArgs().VerifyAsync(default!, default);
+    }
+
+    /// <summary>A site-scoped Viewer is refused: an archive holds every site's rows.</summary>
+    [Fact]
+    public void SiteScopedViewer_IsRefused()
+    {
+        CreateActor().Tell(Envelope(new VerifyAuditArchiveCommand(Name), ["3"], "Viewer"));
+
+        var response = ExpectMsg<ManagementUnauthorized>(TimeSpan.FromSeconds(5));
+        Assert.Contains("every site", response.Message);
+        _archive.DidNotReceiveWithAnyArgs().VerifyAsync(default!, default);
+    }
+
+    /// <summary>No archive directory configured is an explicit error code, not an empty list.</summary>
+    [Fact]
+    public void UnconfiguredArchive_IsArchiveNotConfigured()
+    {
+        _archive.IsConfigured.Returns(false);
+
+        CreateActor().Tell(Envelope(new ListAuditArchivesCommand(), [], "Administrator"));
+
+        var response = ExpectMsg<ManagementError>(TimeSpan.FromSeconds(5));
+        Assert.Equal("ARCHIVE_NOT_CONFIGURED", response.ErrorCode);
+    }
+
+    /// <summary>A malformed name is INVALID_ARGUMENT; an unknown one is NOT_FOUND.</summary>
+    [Fact]
+    public void BadNames_MapToCallerErrors()
+    {
+        _archive.VerifyAsync("../etc", Arg.Any<CancellationToken>())
+            .Returns<Task<AuditArchiveVerificationResult>>(_ => throw new ArgumentException("not an archive name"));
+        _archive.VerifyAsync(Name, Arg.Any<CancellationToken>())
+            .Returns<Task<AuditArchiveVerificationResult>>(_ => throw new FileNotFoundException("no archive"));
+
+        var actor = CreateActor();
+        actor.Tell(Envelope(new VerifyAuditArchiveCommand("../etc"), [], "Administrator"));
+        Assert.Equal("INVALID_ARGUMENT", ExpectMsg<ManagementError>(TimeSpan.FromSeconds(5)).ErrorCode);
+        actor.Tell(Envelope(new VerifyAuditArchiveCommand(Name), [], "Administrator"));
+        Assert.Equal("NOT_FOUND", ExpectMsg<ManagementError>(TimeSpan.FromSeconds(5)).ErrorCode);
+    }
+
+    /// <summary>Archive verification runs under the long-running Ask timeout.</summary>
+    [Fact]
+    public void VerifyAuditArchive_UsesLongRunningAskTimeout()
+    {
+        Assert.Equal(TimeSpan.FromMinutes(5),
+            ManagementEndpoints.ResolveAskTimeout(new ManagementServiceOptions(), typeof(VerifyAuditArchiveCommand)));
+    }
+}
@@ -506,21 +506,43 @@ public class AuditEndpointsTests
        Assert.Equal(propertyNames, AuditEndpoints.CsvHeader.Split(','));
    }

-    /// <summary>Verifies that a Parquet export returns 501.</summary>
+    /// <summary>
+    /// <c>format=parquet</c> (ScadaBridge#42; it used to be a 501) streams one readable Parquet
+    /// file whose columns are the csv/jsonl export's columns, in order, holding every row.
+    /// </summary>
    /// <returns>A task that represents the asynchronous operation.</returns>
    [Fact]
-    public async Task Export_Parquet_Returns501()
+    public async Task Export_Parquet_StreamsReadableParquet_WithTheCsvColumns()
    {
-        // Parquet archival is deferred to v1.x (Component-AuditLog.md) — no
-        // library is referenced, so the endpoint returns 501 with guidance.
-        var (client, _, host) = await BuildHostAsync(roles: new[] { "Administrator" });
+        var (client, _, host) = await BuildHostAsync(
+            roles: new[] { "Administrator" },
+            queryPages: new[]
+            {
+                (IReadOnlyList<AuditEvent>)new[]
+                {
+                    SampleEvent(Guid.Parse("cccccccc-0000-0000-0000-000000000001")),
+                    SampleEvent(Guid.Parse("cccccccc-0000-0000-0000-000000000002")),
+                },
+                (IReadOnlyList<AuditEvent>)Array.Empty<AuditEvent>(),
+            });
        using (host)
        {
-            var response = await client.SendAsync(Get("/api/audit/export?format=parquet"));
+            var response = await client.SendAsync(Get("/api/audit/export?format=Parquet"));

-            Assert.Equal(HttpStatusCode.NotImplemented, response.StatusCode);
-            var body = await response.Content.ReadAsStringAsync();
-            Assert.Contains("Parquet export deferred to v1.x", body);
+            Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+            Assert.Equal(AuditEndpoints.ParquetContentType, response.Content.Headers.ContentType!.MediaType);
+            Assert.Contains(".parquet", response.Content.Headers.ContentDisposition?.ToString()
+                ?? response.Headers.GetValues("Content-Disposition").FirstOrDefault()
+                ?? response.Content.Headers.GetValues("Content-Disposition").First());
+
+            var bytes = await response.Content.ReadAsByteArrayAsync();
+            await using var reader = await Parquet.ParquetReader.CreateAsync(new MemoryStream(bytes));
+            Assert.Equal(AuditEndpoints.CsvHeader.Split(','), reader.Schema.GetDataFields().Select(f => f.Name));
+            using var group = reader.OpenRowGroupReader(0);
+            Assert.Equal(2, group.RowCount);
+            var ids = new string?[2];
+            await group.ReadAsync(reader.Schema.GetDataFields()[0], ids!);
+            Assert.Equal(new[] { "cccccccc-0000-0000-0000-000000000001", "cccccccc-0000-0000-0000-000000000002" }, ids);
        }
    }

@@ -0,0 +1,150 @@
+using Parquet;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
+using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
+
+namespace ZB.MOM.WW.ScadaBridge.ManagementService.Tests;
+
+/// <summary>
+/// <see cref="AuditExportParquet"/> (ScadaBridge#42): the Parquet export round-trips every
+/// one of the 24 export columns, splits into row groups, and never writes synchronously to
+/// the output (Kestrel rejects synchronous response writes).
+/// </summary>
+public class AuditExportParquetTests
+{
+    /// <summary>Rows written and read back are equal, column for column, across row groups.</summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task RoundTrip_EveryColumn_AcrossRowGroups()
+    {
+        var rows = Enumerable.Range(0, AuditExportParquet.RowGroupRows + 7).Select(Row).ToList();
+        var output = new AsyncOnlyStream();
+
+        var written = await AuditExportParquet.WriteAsync(Pages(rows, 1000), output);
+
+        Assert.Equal(rows.Count, written);
+        await using var reader = await ParquetReader.CreateAsync(new MemoryStream(output.ToArray()));
+        Assert.Equal(2, reader.RowGroupCount);
+        var back = new List<AuditExportRow>();
+        var fields = reader.Schema.GetDataFields();
+        for (var g = 0; g < reader.RowGroupCount; g++)
+        {
+            using var group = reader.OpenRowGroupReader(g);
+            var n = (int)group.RowCount;
+            async Task<string?[]> S(int i) { var a = new string?[n]; await group.ReadAsync(fields[i], a!); return a; }
+            async Task<T?[]> V<T>(int i) where T : struct { var a = new T?[n]; await group.ReadAsync<T>(fields[i], a); return a; }
+            var id = await S(0); var occurred = await V<DateTime>(1); var ingested = await V<DateTime>(2);
+            var channel = await S(3); var kind = await S(4); var corr = await S(5); var exec = await S(6); var parent = await S(7);
+            var site = await S(8); var node = await S(9); var inst = await S(10); var script = await S(11); var actor = await S(12);
+            var target = await S(13); var status = await S(14); var http = await V<int>(15); var dur = await V<int>(16);
+            var errMsg = await S(17); var errDetail = await S(18); var req = await S(19); var resp = await S(20);
+            var trunc = await V<bool>(21); var extra = await S(22); var fwd = await S(23);
+            for (var i = 0; i < n; i++)
+            {
+                back.Add(new AuditExportRow
+                {
+                    EventId = Guid.Parse(id[i]!),
+                    OccurredAtUtc = occurred[i]!.Value,
+                    IngestedAtUtc = ingested[i],
+                    Channel = Enum.Parse<AuditChannel>(channel[i]!),
+                    Kind = Enum.Parse<AuditKind>(kind[i]!),
+                    CorrelationId = corr[i] is { } c ? Guid.Parse(c) : null,
+                    ExecutionId = exec[i] is { } e ? Guid.Parse(e) : null,
+                    ParentExecutionId = parent[i] is { } p ? Guid.Parse(p) : null,
+                    SourceSiteId = site[i],
+                    SourceNode = node[i],
+                    SourceInstanceId = inst[i],
+                    SourceScript = script[i],
+                    Actor = actor[i],
+                    Target = target[i],
+                    Status = Enum.Parse<AuditStatus>(status[i]!),
+                    HttpStatus = http[i],
+                    DurationMs = dur[i],
+                    ErrorMessage = errMsg[i],
+                    ErrorDetail = errDetail[i],
+                    RequestSummary = req[i],
+                    ResponseSummary = resp[i],
+                    PayloadTruncated = trunc[i]!.Value,
+                    Extra = extra[i],
+                    ForwardState = fwd[i] is { } f ? Enum.Parse<AuditForwardState>(f) : null,
+                });
+            }
+        }
+
+        Assert.Equal(rows, back);
+    }
+
+    /// <summary>An export with no rows is still a valid (empty) Parquet file.</summary>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Fact]
+    public async Task NoRows_IsAValidEmptyFile()
+    {
+        var output = new AsyncOnlyStream();
+        Assert.Equal(0, await AuditExportParquet.WriteAsync(Pages([], 10), output));
+        await using var reader = await ParquetReader.CreateAsync(new MemoryStream(output.ToArray()));
+        Assert.Equal(0, reader.RowGroupCount);
+        Assert.Equal(24, reader.Schema.GetDataFields().Length);
+    }
+
+    private static AuditExportRow Row(int i) => new()
+    {
+        EventId = Guid.NewGuid(),
+        // Microsecond precision: the export timestamp column is TIMESTAMP(MICROS).
+        OccurredAtUtc = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc).AddTicks(i * 10L),
+        IngestedAtUtc = i % 3 == 0 ? null : new DateTime(2026, 5, 20, 12, 0, 1, DateTimeKind.Utc),
+        Channel = (AuditChannel)(i % Enum.GetValues<AuditChannel>().Length),
+        Kind = AuditKind.ApiCall,
+        CorrelationId = i % 2 == 0 ? Guid.NewGuid() : null,
+        ExecutionId = Guid.NewGuid(),
+        ParentExecutionId = i % 5 == 0 ? Guid.NewGuid() : null,
+        SourceSiteId = i % 4 == 0 ? null : "plant-a",
+        SourceNode = "node-a",
+        SourceInstanceId = "Plant1.Boiler",
+        SourceScript = i % 2 == 0 ? "OnTick" : null,
+        Actor = i % 7 == 0 ? string.Empty : "api-key-1",
+        Target = "erp",
+        Status = AuditStatus.Delivered,
+        HttpStatus = i % 2 == 0 ? 200 : null,
+        DurationMs = i,
+        ErrorMessage = i % 9 == 0 ? "boom, \"quoted\"\nline" : null,
+        ErrorDetail = null,
+        RequestSummary = "{\"ü\":\"日本\"}",
+        ResponseSummary = null,
+        PayloadTruncated = i % 2 == 1,
+        Extra = null,
+        ForwardState = i % 11 == 0 ? AuditForwardState.Forwarded : null,
+    };
+
+    private static async IAsyncEnumerable<IReadOnlyList<AuditExportRow>> Pages(List<AuditExportRow> rows, int size)
+    {
+        for (var i = 0; i < rows.Count; i += size)
+        {
+            await Task.Yield();
+            yield return rows.Skip(i).Take(size).ToList();
+        }
+    }
+
+    /// <summary>A response-body stand-in that fails any synchronous write, like Kestrel.</summary>
+    private sealed class AsyncOnlyStream : Stream
+    {
+        private readonly MemoryStream _inner = new();
+
+        public byte[] ToArray() => _inner.ToArray();
+
+        public override bool CanRead => false;
+        public override bool CanSeek => false;
+        public override bool CanWrite => true;
+        public override long Length => throw new NotSupportedException();
+        public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); }
+        public override void Flush() => throw new InvalidOperationException("Synchronous operations are disallowed.");
+        public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask;
+        public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
+        public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
+        public override void SetLength(long value) => throw new NotSupportedException();
+        public override void Write(byte[] buffer, int offset, int count) => throw new InvalidOperationException("Synchronous operations are disallowed.");
+        public override ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default)
+        {
+            _inner.Write(buffer.Span);
+            return ValueTask.CompletedTask;
+        }
+    }
+}
@@ -171,6 +171,9 @@ public class RequiredRoleMatrixTests
        // AuditLog hash-chain verification (ScadaBridge#41) carries audit query's
        // OperationalAudit permission (alog.md §15.1); written out by hand to keep the freeze.
        ["VerifyAuditChain"] = [Roles.Administrator, Roles.Viewer],
+        // AuditLog Parquet archives (ScadaBridge#42): same audit-read gate as verify-chain.
+        ["ListAuditArchives"] = [Roles.Administrator, Roles.Viewer],
+        ["VerifyAuditArchive"] = [Roles.Administrator, Roles.Viewer],

        // ---- Two-person secured write ----------------------------------------------
        ["SubmitSecuredWrite"] = [Roles.Operator],