feat(management): parquet audit export and audit archive list/verify (ScadaBridge#42)

GET /api/audit/export?format=parquet returned 501. It now streams the same 24 columns as csv/jsonl (AuditExportRow, CsvHeader order) as one Parquet file, one 10,000-row row group at a time. Each row group is encoded into a small non-seekable staging buffer and copied to the response asynchronously, because Kestrel rejects the synchronous writes the Parquet writer issues. New management commands, with CLI parity: - ListAuditArchives -> `audit archive list` (AuditArchiveListResult: enabled flag + every manifest). - VerifyAuditArchive(Name) -> `audit archive verify --name` (files, chain, purge anchor); exit 0 verified, 3 not verified, 1 error, 2 unauthorized. Both carry verify-chain's gate: OperationalAudit roles (Administrator, Viewer), site-scoped non-Administrator refused because an archive spans every site. VerifyAuditArchive uses the long-running Ask timeout. Error codes ARCHIVE_NOT_CONFIGURED, INVALID_ARGUMENT, NOT_FOUND. RequiredRoleMatrixTests freezes both gates; ManagementSurfaceParityTests now probes `audit archive list|verify` alongside `audit verify-chain`. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-09 17:00:58 -05:00
parent 7be4f19861
commit d1fc336e18
17 changed files with 982 additions and 42 deletions
@@ -0,0 +1,117 @@
+using System.Globalization;
+using System.Text.Json;
+
+namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
+
+/// <summary>
+/// Helpers for the <c>audit archive list</c> / <c>audit archive verify</c> subcommands
+/// (AuditLog Parquet cold archive, ScadaBridge#42). Both send a management command
+/// (<c>ListAuditArchives</c>, <c>VerifyAuditArchive</c>); the archives live on the central
+/// nodes' archive directory, not on the operator's machine.
+/// </summary>
+/// <remarks>
+/// <c>verify</c> exit codes: <c>0</c> verified, <see cref="NotVerifiedExitCode"/> when the
+/// files, the chain or the purge anchor disagree, <c>1</c> on a request error, <c>2</c> on an
+/// authorization failure.
+/// </remarks>
+public static class AuditArchiveHelpers
+{
+    /// <summary>Process exit code when an archive failed verification (tamper evidence).</summary>
+    public const int NotVerifiedExitCode = 3;
+
+    /// <summary>
+    /// Client deadline for <c>verify</c>: the server hashes and re-chains a whole month under
+    /// the long-running management Ask (5 min); the client waits a little longer.
+    /// </summary>
+    public static readonly TimeSpan VerifyRequestTimeout = TimeSpan.FromMinutes(6);
+
+    /// <summary>Renders a <c>ListAuditArchives</c> result.</summary>
+    /// <param name="json">The success body (an <c>AuditArchiveListResult</c>, camelCase).</param>
+    /// <param name="format">Resolved output format.</param>
+    /// <param name="output">Writer for the report.</param>
+    /// <returns>Always <c>0</c>.</returns>
+    public static int RenderList(string json, string format, TextWriter output)
+    {
+        if (!string.Equals(format, "table", StringComparison.OrdinalIgnoreCase))
+        {
+            output.WriteLine(json);
+            output.Flush();
+            return 0;
+        }
+
+        using var doc = JsonDocument.Parse(json);
+        var root = doc.RootElement;
+        var enabled = root.TryGetProperty("enabled", out var e) && e.ValueKind == JsonValueKind.True;
+        output.WriteLine($"Archiving: {(enabled ? "enabled" : "disabled")}");
+        if (!root.TryGetProperty("archives", out var archives) || archives.GetArrayLength() == 0)
+        {
+            output.WriteLine("No archives.");
+            output.Flush();
+            return 0;
+        }
+
+        output.WriteLine($"{"NAME",-42} {"PARTITION",-9} {"STATUS",-8} {"ROWS",12} {"CHAINED",12} {"BYTES",14}");
+        foreach (var a in archives.EnumerateArray())
+        {
+            output.WriteLine(string.Create(CultureInfo.InvariantCulture,
+                $"{Str(a, "name"),-42} {Str(a, "partition"),-9} {Str(a, "status"),-8} {Num(a, "rowCount"),12} {Num(a, "chainedRowCount"),12} {Num(a, "dataBytes"),14}"));
+        }
+
+        output.Flush();
+        return 0;
+    }
+
+    /// <summary>Renders a <c>VerifyAuditArchive</c> result and maps it to the exit code.</summary>
+    /// <param name="json">The success body (an <c>AuditArchiveVerificationResult</c>, camelCase).</param>
+    /// <param name="format">Resolved output format.</param>
+    /// <param name="output">Writer for the report.</param>
+    /// <returns><c>0</c> when verified, <see cref="NotVerifiedExitCode"/> otherwise.</returns>
+    public static int RenderVerify(string json, string format, TextWriter output)
+    {
+        using var doc = JsonDocument.Parse(json);
+        var root = doc.RootElement;
+        var verified = root.TryGetProperty("verified", out var v) && v.ValueKind == JsonValueKind.True;
+
+        if (!string.Equals(format, "table", StringComparison.OrdinalIgnoreCase))
+        {
+            output.WriteLine(json);
+            output.Flush();
+            return verified ? 0 : NotVerifiedExitCode;
+        }
+
+        output.WriteLine($"Archive     : {Str(root, "name")}");
+        output.WriteLine($"Partition   : {Str(root, "partition")}");
+        output.WriteLine($"Manifest    : {Str(root, "manifestStatus")}");
+        output.WriteLine($"Verified    : {(verified ? "yes" : "NO")}");
+        output.WriteLine($"Files intact: {(root.TryGetProperty("filesIntact", out var f) && f.ValueKind == JsonValueKind.True ? "yes" : "NO")}");
+        if (Str(root, "fileProblem") is { Length: > 0 } problem)
+        {
+            output.WriteLine($"  problem   : {problem}");
+        }
+
+        output.WriteLine($"Rows read   : {Num(root, "rowsRead")}");
+        if (root.TryGetProperty("chains", out var chains) && chains.ValueKind == JsonValueKind.Array)
+        {
+            foreach (var c in chains.EnumerateArray())
+            {
+                var chain = c.GetProperty("chain");
+                output.WriteLine($"Chain {Str(chain, "partition")}: {Str(chain, "status")}, {Num(chain, "rowsVerified")} rows verified, "
+                    + $"{Num(chain, "tombstonesApplied")} tombstones, {Num(chain, "unchainedRows")} unchained; anchor {Str(c, "anchor")}");
+                output.WriteLine($"  {Str(c, "anchorDetail")}");
+                if (chain.TryGetProperty("firstBreak", out var brk) && brk.ValueKind == JsonValueKind.Object)
+                {
+                    output.WriteLine($"  first break: chainSeq {Num(brk, "chainSeq")}, eventId {Str(brk, "eventId", "-")}, {Str(brk, "kind")} — {Str(brk, "detail")}");
+                }
+            }
+        }
+
+        output.Flush();
+        return verified ? 0 : NotVerifiedExitCode;
+    }
+
+    private static string Str(JsonElement e, string name, string missing = "")
+        => e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() ?? missing : missing;
+
+    private static string Num(JsonElement e, string name, string missing = "0")
+        => e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.Number ? v.GetInt64().ToString(CultureInfo.InvariantCulture) : missing;
+}
@@ -9,7 +9,9 @@ namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
 /// the centralized append-only Audit Log via the REST endpoints
 /// (<c>GET /api/audit/query</c>, <c>GET /api/audit/export</c>,
 /// <c>GET /api/audit/tree</c>), plus <c>verify-chain</c>, which sends the
-/// <c>VerifyAuditChain</c> management command (hash-chain tamper evidence, ScadaBridge#41).
+/// <c>VerifyAuditChain</c> management command (hash-chain tamper evidence, ScadaBridge#41),
+/// and <c>archive list</c> / <c>archive verify</c>, which send <c>ListAuditArchives</c> /
+/// <c>VerifyAuditArchive</c> (Parquet cold archive, ScadaBridge#42).
 /// </summary>
 public static class AuditCommands
 {
@@ -31,6 +33,7 @@ public static class AuditCommands
        command.Add(BuildTree(urlOption, formatOption, usernameOption, passwordOption));
        command.Add(BuildVerifyChain(urlOption, formatOption, usernameOption, passwordOption));
        command.Add(BuildBackfillSourceNode(urlOption, formatOption, usernameOption, passwordOption));
+        command.Add(BuildArchive(urlOption, formatOption, usernameOption, passwordOption));

        return command;
    }
@@ -316,6 +319,49 @@ public static class AuditCommands
        return cmd;
    }

+    /// <summary>
+    /// Builds the <c>audit archive</c> group (ScadaBridge#42): <c>list</c> and <c>verify</c>
+    /// over the AuditLog Parquet cold archives on the central archive directory.
+    /// </summary>
+    private static Command BuildArchive(Option<string> urlOption, Option<string> formatOption, Option<string> usernameOption, Option<string> passwordOption)
+    {
+        var group = new Command("archive") { Description = "List and verify the AuditLog Parquet cold archives" };
+
+        var list = new Command("list") { Description = "List the AuditLog Parquet archives written by the retention purge" };
+        list.SetAction(async (ParseResult result) =>
+        {
+            var format = AuditCommandHelpers.ResolveFormat(result, formatOption);
+            return await CommandHelpers.ExecuteCommandAsync(
+                result, urlOption, formatOption, usernameOption, passwordOption,
+                new ListAuditArchivesCommand(),
+                onSuccess: json => AuditArchiveHelpers.RenderList(json, format, Console.Out));
+        });
+        group.Add(list);
+
+        var nameOption = new Option<string>("--name")
+        {
+            Description = "Archive name, as 'audit archive list' prints it (auditlog-YYYY-MM-yyyyMMddTHHmmssfffZ)",
+            Required = true,
+        };
+        var verify = new Command("verify")
+        {
+            Description = "Verify an archive's files, its hash chain and its purge anchor; exit 3 if it does not verify",
+        };
+        verify.Add(nameOption);
+        verify.SetAction(async (ParseResult result) =>
+        {
+            var format = AuditCommandHelpers.ResolveFormat(result, formatOption);
+            return await CommandHelpers.ExecuteCommandAsync(
+                result, urlOption, formatOption, usernameOption, passwordOption,
+                new VerifyAuditArchiveCommand(result.GetValue(nameOption)!),
+                timeout: AuditArchiveHelpers.VerifyRequestTimeout,
+                onSuccess: json => AuditArchiveHelpers.RenderVerify(json, format, Console.Out));
+        });
+        group.Add(verify);
+
+        return group;
+    }
+
    /// <summary>
    /// Builds the <c>audit backfill-source-node</c> sub-command.
    /// Sets <c>SourceNode</c> on historical pre-feature rows whose <c>SourceNode IS NULL</c>
@@ -134,8 +134,8 @@ public static class AuditExportHelpers
    /// Executes the export: GETs <c>/api/audit/export</c> and copies the response body
    /// stream directly to <see cref="AuditExportArgs.Output"/>. The body is never fully
    /// buffered — the copy streams in fixed-size chunks via <see cref="Stream.ReadAsync(Memory{byte}, CancellationToken)"/>.
-    /// A <c>501 Not Implemented</c> (parquet not yet supported server-side) prints the
-    /// server message and returns a non-zero exit code.
+    /// A <c>501 Not Implemented</c> (a central older than ScadaBridge#42 does not serve
+    /// <c>format=parquet</c>) prints the server message and returns a non-zero exit code.
    /// </summary>
    /// <param name="client">The management HTTP client for API communication.</param>
    /// <param name="args">The export arguments containing filters and output file path.</param>