feat(management): parquet audit export and audit archive list/verify (ScadaBridge#42)
GET /api/audit/export?format=parquet returned 501. It now streams the same 24 columns as csv/jsonl (AuditExportRow, CsvHeader order) as one Parquet file, one 10,000-row row group at a time. Each row group is encoded into a small non-seekable staging buffer and copied to the response asynchronously, because Kestrel rejects the synchronous writes the Parquet writer issues. New management commands, with CLI parity: - ListAuditArchives -> `audit archive list` (AuditArchiveListResult: enabled flag + every manifest). - VerifyAuditArchive(Name) -> `audit archive verify --name` (files, chain, purge anchor); exit 0 verified, 3 not verified, 1 error, 2 unauthorized. Both carry verify-chain's gate: OperationalAudit roles (Administrator, Viewer), site-scoped non-Administrator refused because an archive spans every site. VerifyAuditArchive uses the long-running Ask timeout. Error codes ARCHIVE_NOT_CONFIGURED, INVALID_ARGUMENT, NOT_FOUND. RequiredRoleMatrixTests freezes both gates; ManagementSurfaceParityTests now probes `audit archive list|verify` alongside `audit verify-chain`. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,117 @@
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
|
||||
|
||||
/// <summary>
|
||||
/// Helpers for the <c>audit archive list</c> / <c>audit archive verify</c> subcommands
|
||||
/// (AuditLog Parquet cold archive, ScadaBridge#42). Both send a management command
|
||||
/// (<c>ListAuditArchives</c>, <c>VerifyAuditArchive</c>); the archives live on the central
|
||||
/// nodes' archive directory, not on the operator's machine.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <c>verify</c> exit codes: <c>0</c> verified, <see cref="NotVerifiedExitCode"/> when the
|
||||
/// files, the chain or the purge anchor disagree, <c>1</c> on a request error, <c>2</c> on an
|
||||
/// authorization failure.
|
||||
/// </remarks>
|
||||
public static class AuditArchiveHelpers
|
||||
{
|
||||
/// <summary>Process exit code when an archive failed verification (tamper evidence).</summary>
|
||||
public const int NotVerifiedExitCode = 3;
|
||||
|
||||
/// <summary>
|
||||
/// Client deadline for <c>verify</c>: the server hashes and re-chains a whole month under
|
||||
/// the long-running management Ask (5 min); the client waits a little longer.
|
||||
/// </summary>
|
||||
public static readonly TimeSpan VerifyRequestTimeout = TimeSpan.FromMinutes(6);
|
||||
|
||||
/// <summary>Renders a <c>ListAuditArchives</c> result.</summary>
|
||||
/// <param name="json">The success body (an <c>AuditArchiveListResult</c>, camelCase).</param>
|
||||
/// <param name="format">Resolved output format.</param>
|
||||
/// <param name="output">Writer for the report.</param>
|
||||
/// <returns>Always <c>0</c>.</returns>
|
||||
public static int RenderList(string json, string format, TextWriter output)
|
||||
{
|
||||
if (!string.Equals(format, "table", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
output.WriteLine(json);
|
||||
output.Flush();
|
||||
return 0;
|
||||
}
|
||||
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
var root = doc.RootElement;
|
||||
var enabled = root.TryGetProperty("enabled", out var e) && e.ValueKind == JsonValueKind.True;
|
||||
output.WriteLine($"Archiving: {(enabled ? "enabled" : "disabled")}");
|
||||
if (!root.TryGetProperty("archives", out var archives) || archives.GetArrayLength() == 0)
|
||||
{
|
||||
output.WriteLine("No archives.");
|
||||
output.Flush();
|
||||
return 0;
|
||||
}
|
||||
|
||||
output.WriteLine($"{"NAME",-42} {"PARTITION",-9} {"STATUS",-8} {"ROWS",12} {"CHAINED",12} {"BYTES",14}");
|
||||
foreach (var a in archives.EnumerateArray())
|
||||
{
|
||||
output.WriteLine(string.Create(CultureInfo.InvariantCulture,
|
||||
$"{Str(a, "name"),-42} {Str(a, "partition"),-9} {Str(a, "status"),-8} {Num(a, "rowCount"),12} {Num(a, "chainedRowCount"),12} {Num(a, "dataBytes"),14}"));
|
||||
}
|
||||
|
||||
output.Flush();
|
||||
return 0;
|
||||
}
|
||||
|
||||
/// <summary>Renders a <c>VerifyAuditArchive</c> result and maps it to the exit code.</summary>
|
||||
/// <param name="json">The success body (an <c>AuditArchiveVerificationResult</c>, camelCase).</param>
|
||||
/// <param name="format">Resolved output format.</param>
|
||||
/// <param name="output">Writer for the report.</param>
|
||||
/// <returns><c>0</c> when verified, <see cref="NotVerifiedExitCode"/> otherwise.</returns>
|
||||
public static int RenderVerify(string json, string format, TextWriter output)
|
||||
{
|
||||
using var doc = JsonDocument.Parse(json);
|
||||
var root = doc.RootElement;
|
||||
var verified = root.TryGetProperty("verified", out var v) && v.ValueKind == JsonValueKind.True;
|
||||
|
||||
if (!string.Equals(format, "table", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
output.WriteLine(json);
|
||||
output.Flush();
|
||||
return verified ? 0 : NotVerifiedExitCode;
|
||||
}
|
||||
|
||||
output.WriteLine($"Archive : {Str(root, "name")}");
|
||||
output.WriteLine($"Partition : {Str(root, "partition")}");
|
||||
output.WriteLine($"Manifest : {Str(root, "manifestStatus")}");
|
||||
output.WriteLine($"Verified : {(verified ? "yes" : "NO")}");
|
||||
output.WriteLine($"Files intact: {(root.TryGetProperty("filesIntact", out var f) && f.ValueKind == JsonValueKind.True ? "yes" : "NO")}");
|
||||
if (Str(root, "fileProblem") is { Length: > 0 } problem)
|
||||
{
|
||||
output.WriteLine($" problem : {problem}");
|
||||
}
|
||||
|
||||
output.WriteLine($"Rows read : {Num(root, "rowsRead")}");
|
||||
if (root.TryGetProperty("chains", out var chains) && chains.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
foreach (var c in chains.EnumerateArray())
|
||||
{
|
||||
var chain = c.GetProperty("chain");
|
||||
output.WriteLine($"Chain {Str(chain, "partition")}: {Str(chain, "status")}, {Num(chain, "rowsVerified")} rows verified, "
|
||||
+ $"{Num(chain, "tombstonesApplied")} tombstones, {Num(chain, "unchainedRows")} unchained; anchor {Str(c, "anchor")}");
|
||||
output.WriteLine($" {Str(c, "anchorDetail")}");
|
||||
if (chain.TryGetProperty("firstBreak", out var brk) && brk.ValueKind == JsonValueKind.Object)
|
||||
{
|
||||
output.WriteLine($" first break: chainSeq {Num(brk, "chainSeq")}, eventId {Str(brk, "eventId", "-")}, {Str(brk, "kind")} — {Str(brk, "detail")}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output.Flush();
|
||||
return verified ? 0 : NotVerifiedExitCode;
|
||||
}
|
||||
|
||||
private static string Str(JsonElement e, string name, string missing = "")
|
||||
=> e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() ?? missing : missing;
|
||||
|
||||
private static string Num(JsonElement e, string name, string missing = "0")
|
||||
=> e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.Number ? v.GetInt64().ToString(CultureInfo.InvariantCulture) : missing;
|
||||
}
|
||||
@@ -9,7 +9,9 @@ namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
|
||||
/// the centralized append-only Audit Log via the REST endpoints
|
||||
/// (<c>GET /api/audit/query</c>, <c>GET /api/audit/export</c>,
|
||||
/// <c>GET /api/audit/tree</c>), plus <c>verify-chain</c>, which sends the
|
||||
/// <c>VerifyAuditChain</c> management command (hash-chain tamper evidence, ScadaBridge#41).
|
||||
/// <c>VerifyAuditChain</c> management command (hash-chain tamper evidence, ScadaBridge#41),
|
||||
/// and <c>archive list</c> / <c>archive verify</c>, which send <c>ListAuditArchives</c> /
|
||||
/// <c>VerifyAuditArchive</c> (Parquet cold archive, ScadaBridge#42).
|
||||
/// </summary>
|
||||
public static class AuditCommands
|
||||
{
|
||||
@@ -31,6 +33,7 @@ public static class AuditCommands
|
||||
command.Add(BuildTree(urlOption, formatOption, usernameOption, passwordOption));
|
||||
command.Add(BuildVerifyChain(urlOption, formatOption, usernameOption, passwordOption));
|
||||
command.Add(BuildBackfillSourceNode(urlOption, formatOption, usernameOption, passwordOption));
|
||||
command.Add(BuildArchive(urlOption, formatOption, usernameOption, passwordOption));
|
||||
|
||||
return command;
|
||||
}
|
||||
@@ -316,6 +319,49 @@ public static class AuditCommands
|
||||
return cmd;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the <c>audit archive</c> group (ScadaBridge#42): <c>list</c> and <c>verify</c>
|
||||
/// over the AuditLog Parquet cold archives on the central archive directory.
|
||||
/// </summary>
|
||||
private static Command BuildArchive(Option<string> urlOption, Option<string> formatOption, Option<string> usernameOption, Option<string> passwordOption)
|
||||
{
|
||||
var group = new Command("archive") { Description = "List and verify the AuditLog Parquet cold archives" };
|
||||
|
||||
var list = new Command("list") { Description = "List the AuditLog Parquet archives written by the retention purge" };
|
||||
list.SetAction(async (ParseResult result) =>
|
||||
{
|
||||
var format = AuditCommandHelpers.ResolveFormat(result, formatOption);
|
||||
return await CommandHelpers.ExecuteCommandAsync(
|
||||
result, urlOption, formatOption, usernameOption, passwordOption,
|
||||
new ListAuditArchivesCommand(),
|
||||
onSuccess: json => AuditArchiveHelpers.RenderList(json, format, Console.Out));
|
||||
});
|
||||
group.Add(list);
|
||||
|
||||
var nameOption = new Option<string>("--name")
|
||||
{
|
||||
Description = "Archive name, as 'audit archive list' prints it (auditlog-YYYY-MM-yyyyMMddTHHmmssfffZ)",
|
||||
Required = true,
|
||||
};
|
||||
var verify = new Command("verify")
|
||||
{
|
||||
Description = "Verify an archive's files, its hash chain and its purge anchor; exit 3 if it does not verify",
|
||||
};
|
||||
verify.Add(nameOption);
|
||||
verify.SetAction(async (ParseResult result) =>
|
||||
{
|
||||
var format = AuditCommandHelpers.ResolveFormat(result, formatOption);
|
||||
return await CommandHelpers.ExecuteCommandAsync(
|
||||
result, urlOption, formatOption, usernameOption, passwordOption,
|
||||
new VerifyAuditArchiveCommand(result.GetValue(nameOption)!),
|
||||
timeout: AuditArchiveHelpers.VerifyRequestTimeout,
|
||||
onSuccess: json => AuditArchiveHelpers.RenderVerify(json, format, Console.Out));
|
||||
});
|
||||
group.Add(verify);
|
||||
|
||||
return group;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the <c>audit backfill-source-node</c> sub-command.
|
||||
/// Sets <c>SourceNode</c> on historical pre-feature rows whose <c>SourceNode IS NULL</c>
|
||||
|
||||
@@ -134,8 +134,8 @@ public static class AuditExportHelpers
|
||||
/// Executes the export: GETs <c>/api/audit/export</c> and copies the response body
|
||||
/// stream directly to <see cref="AuditExportArgs.Output"/>. The body is never fully
|
||||
/// buffered — the copy streams in fixed-size chunks via <see cref="Stream.ReadAsync(Memory{byte}, CancellationToken)"/>.
|
||||
/// A <c>501 Not Implemented</c> (parquet not yet supported server-side) prints the
|
||||
/// server message and returns a non-zero exit code.
|
||||
/// A <c>501 Not Implemented</c> (a central older than ScadaBridge#42 does not serve
|
||||
/// <c>format=parquet</c>) prints the server message and returns a non-zero exit code.
|
||||
/// </summary>
|
||||
/// <param name="client">The management HTTP client for API communication.</param>
|
||||
/// <param name="args">The export arguments containing filters and output file path.</param>
|
||||
|
||||
@@ -31,3 +31,19 @@ public record AuditLogQueryResult(IReadOnlyList<AuditLogEntry> Entries, int Tota
|
||||
/// </summary>
|
||||
/// <param name="Month">The month to verify, <c>YYYY-MM</c>.</param>
|
||||
public record VerifyAuditChainCommand(string Month);
|
||||
|
||||
/// <summary>
|
||||
/// Lists the AuditLog Parquet cold archives under the configured archive directory (audit
|
||||
/// roadmap T2, ScadaBridge#42). Read-only; OperationalAudit roles, refused for a site-scoped
|
||||
/// caller because an archive holds every site's rows.
|
||||
/// </summary>
|
||||
public record ListAuditArchivesCommand;
|
||||
|
||||
/// <summary>
|
||||
/// Verifies one AuditLog Parquet archive (ScadaBridge#42): file integrity against its
|
||||
/// manifest, the hash chain recomputed over the archived rows, and the chain end against the
|
||||
/// checkpoint anchor the partition purge left. Read-only; same gate as
|
||||
/// <see cref="ListAuditArchivesCommand"/>.
|
||||
/// </summary>
|
||||
/// <param name="Name">The archive name, as <c>audit archive list</c> reports it.</param>
|
||||
public record VerifyAuditArchiveCommand(string Name);
|
||||
|
||||
@@ -22,8 +22,8 @@ namespace ZB.MOM.WW.ScadaBridge.ManagementService;
|
||||
/// <list type="bullet">
|
||||
/// <item><c>GET /api/audit/query</c> — keyset-paged JSON page, gated on the
|
||||
/// <see cref="AuthorizationPolicies.OperationalAudit"/> permission.</item>
|
||||
/// <item><c>GET /api/audit/export</c> — streamed bulk export (csv / jsonl;
|
||||
/// parquet returns HTTP 501), gated on the
|
||||
/// <item><c>GET /api/audit/export</c> — streamed bulk export (csv / jsonl /
|
||||
/// parquet), gated on the
|
||||
/// <see cref="AuthorizationPolicies.AuditExport"/> permission.</item>
|
||||
/// <item><c>GET /api/audit/tree</c> — execution-chain tree rooted at the
|
||||
/// topmost ancestor of a given <c>executionId</c>, returned as a JSON array
|
||||
@@ -47,11 +47,8 @@ namespace ZB.MOM.WW.ScadaBridge.ManagementService;
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// <b>Parquet.</b> No Parquet library is referenced by the solution and
|
||||
/// <c>Component-AuditLog.md</c> explicitly defers Parquet archival to v1.x, so
|
||||
/// <c>format=parquet</c> returns <c>501 Not Implemented</c> with a guidance
|
||||
/// message rather than a half-built binary stream. csv + jsonl are fully
|
||||
/// implemented.
|
||||
/// <b>Parquet.</b> <c>format=parquet</c> (ScadaBridge#42) streams the same 24 columns as
|
||||
/// csv / jsonl as one Parquet file (<see cref="AuditExportParquet"/>), a row group at a time.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class AuditEndpoints
|
||||
@@ -65,6 +62,9 @@ public static class AuditEndpoints
|
||||
/// <summary>Repository round-trip size used while streaming an export.</summary>
|
||||
public const int ExportPageSize = 1000;
|
||||
|
||||
/// <summary>Content type of a <c>format=parquet</c> export (ScadaBridge#42).</summary>
|
||||
public const string ParquetContentType = "application/vnd.apache.parquet";
|
||||
|
||||
/// <summary>
|
||||
/// Serializer for individual <see cref="AuditEvent"/> rows (query results +
|
||||
/// jsonl export lines). Drops null columns so a sparse audit row stays
|
||||
@@ -185,10 +185,10 @@ public static class AuditEndpoints
|
||||
// ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>
|
||||
/// Handles <c>GET /api/audit/export</c>: authenticates, checks the AuditExport permission, and streams the matching rows as CSV or JSONL.
|
||||
/// Handles <c>GET /api/audit/export</c>: authenticates, checks the AuditExport permission, and streams the matching rows as CSV, JSONL or Parquet.
|
||||
/// </summary>
|
||||
/// <param name="context">The HTTP context for the current request.</param>
|
||||
/// <returns>A task that resolves to the HTTP result (streaming CSV/JSONL, 400, 401, 403, or 501).</returns>
|
||||
/// <returns>A task that resolves to the HTTP result (streaming CSV/JSONL/Parquet, 400, 401, or 403).</returns>
|
||||
internal static async Task<IResult> HandleExport(HttpContext context)
|
||||
{
|
||||
var auth = await AuthenticateAsync(context);
|
||||
@@ -208,17 +208,7 @@ public static class AuditEndpoints
|
||||
format = "csv";
|
||||
}
|
||||
|
||||
if (format == "parquet")
|
||||
{
|
||||
// Deferred to v1.x (Component-AuditLog.md §"Deferred"): no Parquet
|
||||
// library is in the solution. Return a 501 with actionable guidance
|
||||
// rather than a partial binary stream.
|
||||
return Results.Json(
|
||||
new { error = "Parquet export deferred to v1.x; use csv or jsonl.", code = "NOT_IMPLEMENTED" },
|
||||
statusCode: 501);
|
||||
}
|
||||
|
||||
if (format != "csv" && format != "jsonl")
|
||||
if (format != "csv" && format != "jsonl" && format != "parquet")
|
||||
{
|
||||
return Results.Json(
|
||||
new { error = $"Unsupported export format '{format}'. Use csv, jsonl, or parquet.", code = "BAD_REQUEST" },
|
||||
@@ -243,8 +233,12 @@ public static class AuditEndpoints
|
||||
filter = restricted;
|
||||
var repo = context.RequestServices.GetRequiredService<IAuditLogRepository>();
|
||||
|
||||
var contentType = format == "csv" ? "text/csv; charset=utf-8" : "application/x-ndjson";
|
||||
var extension = format == "csv" ? "csv" : "jsonl";
|
||||
var (contentType, extension) = format switch
|
||||
{
|
||||
"csv" => ("text/csv; charset=utf-8", "csv"),
|
||||
"parquet" => (ParquetContentType, "parquet"),
|
||||
_ => ("application/x-ndjson", "jsonl"),
|
||||
};
|
||||
var fileName = $"audit-log-{DateTime.UtcNow:yyyyMMdd-HHmmss}.{extension}";
|
||||
|
||||
context.Response.ContentType = contentType;
|
||||
@@ -256,6 +250,12 @@ public static class AuditEndpoints
|
||||
{
|
||||
await StreamCsvAsync(repo, filter, context.Response.Body, context.RequestAborted);
|
||||
}
|
||||
else if (format == "parquet")
|
||||
{
|
||||
// ScadaBridge#42: same 24 columns as csv/jsonl, one row group per 10k rows.
|
||||
await AuditExportParquet.WriteAsync(
|
||||
ExportRowPagesAsync(repo, filter, context.RequestAborted), context.Response.Body, context.RequestAborted);
|
||||
}
|
||||
else
|
||||
{
|
||||
await StreamJsonlAsync(repo, filter, context.Response.Body, context.RequestAborted);
|
||||
@@ -494,6 +494,18 @@ public static class AuditEndpoints
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Projects <see cref="PageAsync"/> pages onto the flat export row shape.</summary>
|
||||
private static async IAsyncEnumerable<IReadOnlyList<AuditExportRow>> ExportRowPagesAsync(
|
||||
IAuditLogRepository repo,
|
||||
AuditLogQueryFilter filter,
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct)
|
||||
{
|
||||
await foreach (var page in PageAsync(repo, filter, ct))
|
||||
{
|
||||
yield return page.Select(AuditExportRow.From).ToList();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lazily yields full repository pages, advancing the keyset cursor until a
|
||||
/// short page signals the end. Shared by the csv + jsonl streamers.
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
using Parquet;
|
||||
using Parquet.Schema;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ManagementService;
|
||||
|
||||
/// <summary>
|
||||
/// <c>GET /api/audit/export?format=parquet</c> (ScadaBridge#42): the same 24 columns as the
|
||||
/// csv / jsonl export (<see cref="AuditExportRow"/>, in <see cref="AuditEndpoints.CsvHeader"/>
|
||||
/// order) as one Parquet file, one row group per <see cref="RowGroupRows"/> rows.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Types: ids are <c>D</c>-format strings, timestamps are UTC Parquet timestamps
|
||||
/// (microseconds), enums are their names, <c>HttpStatus</c>/<c>DurationMs</c> are int32,
|
||||
/// <c>PayloadTruncated</c> is boolean, everything else is a UTF-8 string.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Streaming.</b> Kestrel forbids synchronous writes to the response body and the Parquet
|
||||
/// writer issues some, so each row group is encoded into a small in-memory buffer and then
|
||||
/// copied to the response asynchronously. Memory is bounded by one row group, and the client
|
||||
/// starts receiving bytes after the first one. A Parquet file's footer is written last, so a
|
||||
/// response cut off mid-stream is not a readable file — the CLI treats it as a failed export.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// This is the ad-hoc export. The retention archive (alog.md §15.2) is a different layout
|
||||
/// that carries the hash-chain columns; see <c>AuditArchiveParquet</c>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public static class AuditExportParquet
|
||||
{
|
||||
/// <summary>Rows per Parquet row group.</summary>
|
||||
public const int RowGroupRows = 10_000;
|
||||
|
||||
private static readonly DataField<string> EventId = new(nameof(AuditExportRow.EventId), false);
|
||||
private static readonly DateTimeDataField OccurredAtUtc = Timestamp(nameof(AuditExportRow.OccurredAtUtc), false);
|
||||
private static readonly DateTimeDataField IngestedAtUtc = Timestamp(nameof(AuditExportRow.IngestedAtUtc), true);
|
||||
private static readonly DataField<string> Channel = new(nameof(AuditExportRow.Channel), false);
|
||||
private static readonly DataField<string> Kind = new(nameof(AuditExportRow.Kind), false);
|
||||
private static readonly DataField<string> CorrelationId = Text(nameof(AuditExportRow.CorrelationId));
|
||||
private static readonly DataField<string> ExecutionId = Text(nameof(AuditExportRow.ExecutionId));
|
||||
private static readonly DataField<string> ParentExecutionId = Text(nameof(AuditExportRow.ParentExecutionId));
|
||||
private static readonly DataField<string> SourceSiteId = Text(nameof(AuditExportRow.SourceSiteId));
|
||||
private static readonly DataField<string> SourceNode = Text(nameof(AuditExportRow.SourceNode));
|
||||
private static readonly DataField<string> SourceInstanceId = Text(nameof(AuditExportRow.SourceInstanceId));
|
||||
private static readonly DataField<string> SourceScript = Text(nameof(AuditExportRow.SourceScript));
|
||||
private static readonly DataField<string> Actor = Text(nameof(AuditExportRow.Actor));
|
||||
private static readonly DataField<string> Target = Text(nameof(AuditExportRow.Target));
|
||||
private static readonly DataField<string> Status = new(nameof(AuditExportRow.Status), false);
|
||||
private static readonly DataField<int> HttpStatus = new(nameof(AuditExportRow.HttpStatus), true);
|
||||
private static readonly DataField<int> DurationMs = new(nameof(AuditExportRow.DurationMs), true);
|
||||
private static readonly DataField<string> ErrorMessage = Text(nameof(AuditExportRow.ErrorMessage));
|
||||
private static readonly DataField<string> ErrorDetail = Text(nameof(AuditExportRow.ErrorDetail));
|
||||
private static readonly DataField<string> RequestSummary = Text(nameof(AuditExportRow.RequestSummary));
|
||||
private static readonly DataField<string> ResponseSummary = Text(nameof(AuditExportRow.ResponseSummary));
|
||||
private static readonly DataField<bool> PayloadTruncated = new(nameof(AuditExportRow.PayloadTruncated), false);
|
||||
private static readonly DataField<string> Extra = Text(nameof(AuditExportRow.Extra));
|
||||
private static readonly DataField<string> ForwardState = Text(nameof(AuditExportRow.ForwardState));
|
||||
|
||||
/// <summary>The export schema; column names and order equal <see cref="AuditEndpoints.CsvHeader"/>.</summary>
|
||||
public static ParquetSchema Schema { get; } = new(
|
||||
EventId, OccurredAtUtc, IngestedAtUtc, Channel, Kind, CorrelationId, ExecutionId, ParentExecutionId,
|
||||
SourceSiteId, SourceNode, SourceInstanceId, SourceScript, Actor, Target, Status, HttpStatus, DurationMs,
|
||||
ErrorMessage, ErrorDetail, RequestSummary, ResponseSummary, PayloadTruncated, Extra, ForwardState);
|
||||
|
||||
/// <summary>Writes every row of <paramref name="pages"/> to <paramref name="output"/> as one Parquet file.</summary>
|
||||
/// <param name="pages">Export rows, page by page.</param>
|
||||
/// <param name="output">The destination (e.g. the HTTP response body); written asynchronously only.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>The number of rows written.</returns>
|
||||
public static async Task<long> WriteAsync(
|
||||
IAsyncEnumerable<IReadOnlyList<AuditExportRow>> pages, Stream output, CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(pages);
|
||||
ArgumentNullException.ThrowIfNull(output);
|
||||
|
||||
// Non-seekable on purpose: the writer then tracks file offsets itself, so emptying the
|
||||
// buffer after each drain cannot corrupt the column-chunk offsets in the footer.
|
||||
var staging = new StagingStream();
|
||||
long total = 0;
|
||||
var buffer = new List<AuditExportRow>(RowGroupRows);
|
||||
await using (var writer = await ParquetWriter.CreateAsync(Schema, staging, cancellationToken: ct).ConfigureAwait(false))
|
||||
{
|
||||
await foreach (var page in pages.WithCancellation(ct).ConfigureAwait(false))
|
||||
{
|
||||
foreach (var row in page)
|
||||
{
|
||||
buffer.Add(row);
|
||||
if (buffer.Count == RowGroupRows)
|
||||
{
|
||||
total += await WriteGroupAsync(writer, buffer, ct).ConfigureAwait(false);
|
||||
await DrainAsync(staging, output, ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
total += await WriteGroupAsync(writer, buffer, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// The footer is written when the writer is disposed.
|
||||
await DrainAsync(staging, output, ct).ConfigureAwait(false);
|
||||
return total;
|
||||
}
|
||||
|
||||
private static async Task<int> WriteGroupAsync(ParquetWriter writer, List<AuditExportRow> rows, CancellationToken ct)
|
||||
{
|
||||
if (rows.Count == 0)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
using (var group = writer.CreateRowGroup())
|
||||
{
|
||||
await group.WriteAsync(EventId, Map(rows, r => r.EventId.ToString("D"))).ConfigureAwait(false);
|
||||
await group.WriteAsync<DateTime>(OccurredAtUtc, rows.Select(r => DateTime.SpecifyKind(r.OccurredAtUtc, DateTimeKind.Utc)).ToArray(), cancellationToken: ct).ConfigureAwait(false);
|
||||
await group.WriteAsync<DateTime>(IngestedAtUtc, rows.Select(r => r.IngestedAtUtc is { } i ? DateTime.SpecifyKind(i, DateTimeKind.Utc) : (DateTime?)null).ToArray(), cancellationToken: ct).ConfigureAwait(false);
|
||||
await group.WriteAsync(Channel, Map(rows, r => r.Channel.ToString())).ConfigureAwait(false);
|
||||
await group.WriteAsync(Kind, Map(rows, r => r.Kind.ToString())).ConfigureAwait(false);
|
||||
await group.WriteAsync(CorrelationId, Map(rows, r => r.CorrelationId?.ToString("D"))!).ConfigureAwait(false);
|
||||
await group.WriteAsync(ExecutionId, Map(rows, r => r.ExecutionId?.ToString("D"))!).ConfigureAwait(false);
|
||||
await group.WriteAsync(ParentExecutionId, Map(rows, r => r.ParentExecutionId?.ToString("D"))!).ConfigureAwait(false);
|
||||
await group.WriteAsync(SourceSiteId, Map(rows, r => r.SourceSiteId)!).ConfigureAwait(false);
|
||||
await group.WriteAsync(SourceNode, Map(rows, r => r.SourceNode)!).ConfigureAwait(false);
|
||||
await group.WriteAsync(SourceInstanceId, Map(rows, r => r.SourceInstanceId)!).ConfigureAwait(false);
|
||||
await group.WriteAsync(SourceScript, Map(rows, r => r.SourceScript)!).ConfigureAwait(false);
|
||||
await group.WriteAsync(Actor, Map(rows, r => r.Actor)!).ConfigureAwait(false);
|
||||
await group.WriteAsync(Target, Map(rows, r => r.Target)!).ConfigureAwait(false);
|
||||
await group.WriteAsync(Status, Map(rows, r => r.Status.ToString())).ConfigureAwait(false);
|
||||
await group.WriteAsync<int>(HttpStatus, rows.Select(r => r.HttpStatus).ToArray(), cancellationToken: ct).ConfigureAwait(false);
|
||||
await group.WriteAsync<int>(DurationMs, rows.Select(r => r.DurationMs).ToArray(), cancellationToken: ct).ConfigureAwait(false);
|
||||
await group.WriteAsync(ErrorMessage, Map(rows, r => r.ErrorMessage)!).ConfigureAwait(false);
|
||||
await group.WriteAsync(ErrorDetail, Map(rows, r => r.ErrorDetail)!).ConfigureAwait(false);
|
||||
await group.WriteAsync(RequestSummary, Map(rows, r => r.RequestSummary)!).ConfigureAwait(false);
|
||||
await group.WriteAsync(ResponseSummary, Map(rows, r => r.ResponseSummary)!).ConfigureAwait(false);
|
||||
await group.WriteAsync<bool>(PayloadTruncated, rows.Select(r => r.PayloadTruncated).ToArray(), cancellationToken: ct).ConfigureAwait(false);
|
||||
await group.WriteAsync(Extra, Map(rows, r => r.Extra)!).ConfigureAwait(false);
|
||||
await group.WriteAsync(ForwardState, Map(rows, r => r.ForwardState?.ToString())!).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var count = rows.Count;
|
||||
rows.Clear();
|
||||
return count;
|
||||
}
|
||||
|
||||
private static async Task DrainAsync(StagingStream staging, Stream output, CancellationToken ct)
|
||||
{
|
||||
var pending = staging.Pending;
|
||||
if (pending.Length == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await output.WriteAsync(pending, ct).ConfigureAwait(false);
|
||||
await output.FlushAsync(ct).ConfigureAwait(false);
|
||||
staging.Clear();
|
||||
}
|
||||
|
||||
private static string?[] Map(List<AuditExportRow> rows, Func<AuditExportRow, string?> select)
|
||||
=> rows.Select(select).ToArray();
|
||||
|
||||
private static DataField<string> Text(string name) => new(name, true);
|
||||
|
||||
private static DateTimeDataField Timestamp(string name, bool nullable)
|
||||
=> new(name, DateTimeFormat.Timestamp, isAdjustedToUTC: true, unit: DateTimeTimeUnit.Micros, isNullable: nullable);
|
||||
|
||||
/// <summary>
|
||||
/// Forward-only, non-seekable buffer the Parquet writer writes into (synchronously or
|
||||
/// not); <see cref="DrainAsync"/> moves its bytes to the real output asynchronously.
|
||||
/// </summary>
|
||||
private sealed class StagingStream : Stream
|
||||
{
|
||||
private readonly MemoryStream _buffer = new();
|
||||
|
||||
public ReadOnlyMemory<byte> Pending => _buffer.GetBuffer().AsMemory(0, (int)_buffer.Length);
|
||||
|
||||
public override bool CanRead => false;
|
||||
|
||||
public override bool CanSeek => false;
|
||||
|
||||
public override bool CanWrite => true;
|
||||
|
||||
public override long Length => throw new NotSupportedException();
|
||||
|
||||
public override long Position
|
||||
{
|
||||
get => throw new NotSupportedException();
|
||||
set => throw new NotSupportedException();
|
||||
}
|
||||
|
||||
public void Clear() => _buffer.SetLength(0);
|
||||
|
||||
public override void Write(byte[] buffer, int offset, int count) => _buffer.Write(buffer, offset, count);
|
||||
|
||||
public override void Write(ReadOnlySpan<byte> buffer) => _buffer.Write(buffer);
|
||||
|
||||
public override ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default)
|
||||
{
|
||||
_buffer.Write(buffer.Span);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
|
||||
{
|
||||
_buffer.Write(buffer, offset, count);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public override void Flush()
|
||||
{
|
||||
}
|
||||
|
||||
public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
|
||||
|
||||
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
|
||||
|
||||
public override void SetLength(long value) => throw new NotSupportedException();
|
||||
}
|
||||
}
|
||||
@@ -393,6 +393,11 @@ public class ManagementActor : ReceiveActor
|
||||
// spans every site.
|
||||
VerifyAuditChainCommand => OperationalAuditReaders,
|
||||
|
||||
// AuditLog Parquet archives (ScadaBridge#42): audit reads with verify-chain's gate —
|
||||
// OperationalAudit roles, site-scoped callers refused in the handler.
|
||||
ListAuditArchivesCommand => OperationalAuditReaders,
|
||||
VerifyAuditArchiveCommand => OperationalAuditReaders,
|
||||
|
||||
// Read-only queries -- any authenticated user
|
||||
_ => null
|
||||
};
|
||||
@@ -564,6 +569,8 @@ public class ManagementActor : ReceiveActor
|
||||
// Audit Log
|
||||
QueryAuditLogCommand cmd => await HandleQueryAuditLog(sp, cmd),
|
||||
VerifyAuditChainCommand cmd => await HandleVerifyAuditChain(sp, cmd, user),
|
||||
ListAuditArchivesCommand => await HandleListAuditArchives(sp, user),
|
||||
VerifyAuditArchiveCommand cmd => await HandleVerifyAuditArchive(sp, cmd, user),
|
||||
|
||||
// Health
|
||||
GetHealthSummaryCommand => HandleGetHealthSummary(sp),
|
||||
@@ -3256,6 +3263,59 @@ public class ManagementActor : ReceiveActor
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lists the AuditLog Parquet archives (ScadaBridge#42). Refused for a site-scoped,
|
||||
/// non-Administrator caller: an archive holds every site's rows.
|
||||
/// </summary>
|
||||
private static async Task<object?> HandleListAuditArchives(IServiceProvider sp, AuthenticatedUser user)
|
||||
{
|
||||
var archive = RequireAuditArchive(sp, user);
|
||||
return new AuditArchiveListResult(archive.IsEnabled, await archive.ListAsync());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies one AuditLog Parquet archive (ScadaBridge#42): files against the manifest,
|
||||
/// the chain recomputed from the archived rows, the chain end against the purge anchor.
|
||||
/// </summary>
|
||||
private static async Task<object?> HandleVerifyAuditArchive(
|
||||
IServiceProvider sp, VerifyAuditArchiveCommand cmd, AuthenticatedUser user)
|
||||
{
|
||||
var archive = RequireAuditArchive(sp, user);
|
||||
try
|
||||
{
|
||||
return await archive.VerifyAsync(cmd.Name);
|
||||
}
|
||||
catch (ArgumentException ex)
|
||||
{
|
||||
throw new ManagementCommandException(ex.Message, "INVALID_ARGUMENT");
|
||||
}
|
||||
catch (FileNotFoundException ex)
|
||||
{
|
||||
throw new ManagementCommandException(ex.Message, "NOT_FOUND");
|
||||
}
|
||||
}
|
||||
|
||||
private static IAuditLogArchive RequireAuditArchive(IServiceProvider sp, AuthenticatedUser user)
|
||||
{
|
||||
if (user.PermittedSiteIds.Length > 0
|
||||
&& !user.Roles.Contains(Roles.Administrator, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new SiteScopeViolationException(
|
||||
"Access denied: AuditLog archives hold every site's rows; they need an "
|
||||
+ "OperationalAudit role that is not scoped to specific sites.");
|
||||
}
|
||||
|
||||
var archive = sp.GetService<IAuditLogArchive>();
|
||||
if (archive is null || !archive.IsConfigured)
|
||||
{
|
||||
throw new ManagementCommandException(
|
||||
"No AuditLog archive directory is configured on this central node (AuditLog:Archive:Directory).",
|
||||
"ARCHIVE_NOT_CONFIGURED");
|
||||
}
|
||||
|
||||
return archive;
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Health handlers
|
||||
// ========================================================================
|
||||
|
||||
@@ -63,6 +63,9 @@ public static class ManagementEndpoints
|
||||
// AuditLog hash-chain verification (ScadaBridge#41) recomputes one hash per row of
|
||||
// a whole month — millions of rows on a busy central.
|
||||
typeof(VerifyAuditChainCommand),
|
||||
// AuditLog archive verification (ScadaBridge#42) hashes and re-chains a whole month's
|
||||
// Parquet archive.
|
||||
typeof(VerifyAuditArchiveCommand),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
|
||||
+1
@@ -12,6 +12,7 @@
|
||||
<InternalsVisibleTo Include="ZB.MOM.WW.ScadaBridge.ManagementService.Tests" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Parquet.Net" />
|
||||
<PackageReference Include="Akka" />
|
||||
<PackageReference Include="Akka.Cluster.Tools" />
|
||||
<PackageReference Include="ZB.MOM.WW.Configuration" />
|
||||
|
||||
Reference in New Issue
Block a user