feat(management): parquet audit export and audit archive list/verify (ScadaBridge#42)
GET /api/audit/export?format=parquet returned 501. It now streams the same 24 columns as csv/jsonl (AuditExportRow, CsvHeader order) as one Parquet file, one 10,000-row row group at a time. Each row group is encoded into a small non-seekable staging buffer and copied to the response asynchronously, because Kestrel rejects the synchronous writes the Parquet writer issues. New management commands, with CLI parity: - ListAuditArchives -> `audit archive list` (AuditArchiveListResult: enabled flag + every manifest). - VerifyAuditArchive(Name) -> `audit archive verify --name` (files, chain, purge anchor); exit 0 verified, 3 not verified, 1 error, 2 unauthorized. Both carry verify-chain's gate: OperationalAudit roles (Administrator, Viewer), site-scoped non-Administrator refused because an archive spans every site. VerifyAuditArchive uses the long-running Ask timeout. Error codes ARCHIVE_NOT_CONFIGURED, INVALID_ARGUMENT, NOT_FOUND. RequiredRoleMatrixTests freezes both gates; ManagementSurfaceParityTests now probes `audit archive list|verify` alongside `audit verify-chain`. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -22,8 +22,8 @@ namespace ZB.MOM.WW.ScadaBridge.ManagementService;
|
||||
/// <list type="bullet">
|
||||
/// <item><c>GET /api/audit/query</c> — keyset-paged JSON page, gated on the
|
||||
/// <see cref="AuthorizationPolicies.OperationalAudit"/> permission.</item>
|
||||
/// <item><c>GET /api/audit/export</c> — streamed bulk export (csv / jsonl;
|
||||
/// parquet returns HTTP 501), gated on the
|
||||
/// <item><c>GET /api/audit/export</c> — streamed bulk export (csv / jsonl /
|
||||
/// parquet), gated on the
|
||||
/// <see cref="AuthorizationPolicies.AuditExport"/> permission.</item>
|
||||
/// <item><c>GET /api/audit/tree</c> — execution-chain tree rooted at the
|
||||
/// topmost ancestor of a given <c>executionId</c>, returned as a JSON array
|
||||
@@ -47,11 +47,8 @@ namespace ZB.MOM.WW.ScadaBridge.ManagementService;
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Parquet.</b> No Parquet library is referenced by the solution and
|
||||
/// <c>Component-AuditLog.md</c> explicitly defers Parquet archival to v1.x, so
|
||||
/// <c>format=parquet</c> returns <c>501 Not Implemented</c> with a guidance
|
||||
/// message rather than a half-built binary stream. csv + jsonl are fully
|
||||
/// implemented.
|
||||
/// <b>Parquet.</b> <c>format=parquet</c> (ScadaBridge#42) streams the same 24 columns as
|
||||
/// csv / jsonl as one Parquet file (<see cref="AuditExportParquet"/>), a row group at a time.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class AuditEndpoints
|
||||
@@ -65,6 +62,9 @@ public static class AuditEndpoints
|
||||
/// <summary>Repository round-trip size used while streaming an export.</summary>
|
||||
public const int ExportPageSize = 1000;
|
||||
|
||||
/// <summary>Content type of a <c>format=parquet</c> export (ScadaBridge#42).</summary>
|
||||
public const string ParquetContentType = "application/vnd.apache.parquet";
|
||||
|
||||
/// <summary>
|
||||
/// Serializer for individual <see cref="AuditEvent"/> rows (query results +
|
||||
/// jsonl export lines). Drops null columns so a sparse audit row stays
|
||||
@@ -185,10 +185,10 @@ public static class AuditEndpoints
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Handles <c>GET /api/audit/export</c>: authenticates, checks the AuditExport permission, and streams the matching rows as CSV or JSONL.
|
||||
/// Handles <c>GET /api/audit/export</c>: authenticates, checks the AuditExport permission, and streams the matching rows as CSV, JSONL or Parquet.
|
||||
/// </summary>
|
||||
/// <param name="context">The HTTP context for the current request.</param>
|
||||
/// <returns>A task that resolves to the HTTP result (streaming CSV/JSONL, 400, 401, 403, or 501).</returns>
|
||||
/// <returns>A task that resolves to the HTTP result (streaming CSV/JSONL/Parquet, 400, 401, or 403).</returns>
|
||||
internal static async Task<IResult> HandleExport(HttpContext context)
|
||||
{
|
||||
var auth = await AuthenticateAsync(context);
|
||||
@@ -208,17 +208,7 @@ public static class AuditEndpoints
|
||||
format = "csv";
|
||||
}
|
||||
|
||||
if (format == "parquet")
|
||||
{
|
||||
// Deferred to v1.x (Component-AuditLog.md §"Deferred"): no Parquet
|
||||
// library is in the solution. Return a 501 with actionable guidance
|
||||
// rather than a partial binary stream.
|
||||
return Results.Json(
|
||||
new { error = "Parquet export deferred to v1.x; use csv or jsonl.", code = "NOT_IMPLEMENTED" },
|
||||
statusCode: 501);
|
||||
}
|
||||
|
||||
if (format != "csv" && format != "jsonl")
|
||||
if (format != "csv" && format != "jsonl" && format != "parquet")
|
||||
{
|
||||
return Results.Json(
|
||||
new { error = $"Unsupported export format '{format}'. Use csv, jsonl, or parquet.", code = "BAD_REQUEST" },
|
||||
@@ -243,8 +233,12 @@ public static class AuditEndpoints
|
||||
filter = restricted;
|
||||
var repo = context.RequestServices.GetRequiredService<IAuditLogRepository>();
|
||||
|
||||
var contentType = format == "csv" ? "text/csv; charset=utf-8" : "application/x-ndjson";
|
||||
var extension = format == "csv" ? "csv" : "jsonl";
|
||||
var (contentType, extension) = format switch
|
||||
{
|
||||
"csv" => ("text/csv; charset=utf-8", "csv"),
|
||||
"parquet" => (ParquetContentType, "parquet"),
|
||||
_ => ("application/x-ndjson", "jsonl"),
|
||||
};
|
||||
var fileName = $"audit-log-{DateTime.UtcNow:yyyyMMdd-HHmmss}.{extension}";
|
||||
|
||||
context.Response.ContentType = contentType;
|
||||
@@ -256,6 +250,12 @@ public static class AuditEndpoints
|
||||
{
|
||||
await StreamCsvAsync(repo, filter, context.Response.Body, context.RequestAborted);
|
||||
}
|
||||
else if (format == "parquet")
|
||||
{
|
||||
// ScadaBridge#42: same 24 columns as csv/jsonl, one row group per 10k rows.
|
||||
await AuditExportParquet.WriteAsync(
|
||||
ExportRowPagesAsync(repo, filter, context.RequestAborted), context.Response.Body, context.RequestAborted);
|
||||
}
|
||||
else
|
||||
{
|
||||
await StreamJsonlAsync(repo, filter, context.Response.Body, context.RequestAborted);
|
||||
@@ -494,6 +494,18 @@ public static class AuditEndpoints
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Projects <see cref="PageAsync"/> pages onto the flat export row shape.</summary>
|
||||
private static async IAsyncEnumerable<IReadOnlyList<AuditExportRow>> ExportRowPagesAsync(
|
||||
IAuditLogRepository repo,
|
||||
AuditLogQueryFilter filter,
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct)
|
||||
{
|
||||
await foreach (var page in PageAsync(repo, filter, ct))
|
||||
{
|
||||
yield return page.Select(AuditExportRow.From).ToList();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lazily yields full repository pages, advancing the keyset cursor until a
|
||||
/// short page signals the end. Shared by the csv + jsonl streamers.
|
||||
|
||||
Reference in New Issue
Block a user