feat(management): parquet audit export and audit archive list/verify (ScadaBridge#42)

GET /api/audit/export?format=parquet returned 501. It now streams the same 24
columns as csv/jsonl (AuditExportRow, CsvHeader order) as one Parquet file,
one 10,000-row row group at a time. Each row group is encoded into a small
non-seekable staging buffer and copied to the response asynchronously, because
Kestrel rejects the synchronous writes the Parquet writer issues.

New management commands, with CLI parity:
- ListAuditArchives -> `audit archive list` (AuditArchiveListResult: enabled
  flag + every manifest).
- VerifyAuditArchive(Name) -> `audit archive verify --name` (files, chain,
  purge anchor); exit 0 verified, 3 not verified, 1 error, 2 unauthorized.
Both carry verify-chain's gate: OperationalAudit roles (Administrator, Viewer),
site-scoped non-Administrator refused because an archive spans every site.
VerifyAuditArchive uses the long-running Ask timeout. Error codes
ARCHIVE_NOT_CONFIGURED, INVALID_ARGUMENT, NOT_FOUND.

RequiredRoleMatrixTests freezes both gates; ManagementSurfaceParityTests now
probes `audit archive list|verify` alongside `audit verify-chain`.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 17:00:58 -05:00
parent 7be4f19861
commit d1fc336e18
17 changed files with 982 additions and 42 deletions
@@ -0,0 +1,117 @@
using System.Globalization;
using System.Text.Json;
namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
/// <summary>
/// Helpers for the <c>audit archive list</c> / <c>audit archive verify</c> subcommands
/// (AuditLog Parquet cold archive, ScadaBridge#42). Both send a management command
/// (<c>ListAuditArchives</c>, <c>VerifyAuditArchive</c>); the archives live on the central
/// nodes' archive directory, not on the operator's machine.
/// </summary>
/// <remarks>
/// <c>verify</c> exit codes: <c>0</c> verified, <see cref="NotVerifiedExitCode"/> when the
/// files, the chain or the purge anchor disagree, <c>1</c> on a request error, <c>2</c> on an
/// authorization failure.
/// </remarks>
public static class AuditArchiveHelpers
{
/// <summary>Process exit code when an archive failed verification (tamper evidence).</summary>
public const int NotVerifiedExitCode = 3;
/// <summary>
/// Client deadline for <c>verify</c>: the server hashes and re-chains a whole month under
/// the long-running management Ask (5 min); the client waits a little longer.
/// </summary>
public static readonly TimeSpan VerifyRequestTimeout = TimeSpan.FromMinutes(6);
/// <summary>Renders a <c>ListAuditArchives</c> result.</summary>
/// <param name="json">The success body (an <c>AuditArchiveListResult</c>, camelCase).</param>
/// <param name="format">Resolved output format.</param>
/// <param name="output">Writer for the report.</param>
/// <returns>Always <c>0</c>.</returns>
public static int RenderList(string json, string format, TextWriter output)
{
if (!string.Equals(format, "table", StringComparison.OrdinalIgnoreCase))
{
output.WriteLine(json);
output.Flush();
return 0;
}
using var doc = JsonDocument.Parse(json);
var root = doc.RootElement;
var enabled = root.TryGetProperty("enabled", out var e) && e.ValueKind == JsonValueKind.True;
output.WriteLine($"Archiving: {(enabled ? "enabled" : "disabled")}");
if (!root.TryGetProperty("archives", out var archives) || archives.GetArrayLength() == 0)
{
output.WriteLine("No archives.");
output.Flush();
return 0;
}
output.WriteLine($"{"NAME",-42} {"PARTITION",-9} {"STATUS",-8} {"ROWS",12} {"CHAINED",12} {"BYTES",14}");
foreach (var a in archives.EnumerateArray())
{
output.WriteLine(string.Create(CultureInfo.InvariantCulture,
$"{Str(a, "name"),-42} {Str(a, "partition"),-9} {Str(a, "status"),-8} {Num(a, "rowCount"),12} {Num(a, "chainedRowCount"),12} {Num(a, "dataBytes"),14}"));
}
output.Flush();
return 0;
}
/// <summary>Renders a <c>VerifyAuditArchive</c> result and maps it to the exit code.</summary>
/// <param name="json">The success body (an <c>AuditArchiveVerificationResult</c>, camelCase).</param>
/// <param name="format">Resolved output format.</param>
/// <param name="output">Writer for the report.</param>
/// <returns><c>0</c> when verified, <see cref="NotVerifiedExitCode"/> otherwise.</returns>
public static int RenderVerify(string json, string format, TextWriter output)
{
using var doc = JsonDocument.Parse(json);
var root = doc.RootElement;
var verified = root.TryGetProperty("verified", out var v) && v.ValueKind == JsonValueKind.True;
if (!string.Equals(format, "table", StringComparison.OrdinalIgnoreCase))
{
output.WriteLine(json);
output.Flush();
return verified ? 0 : NotVerifiedExitCode;
}
output.WriteLine($"Archive : {Str(root, "name")}");
output.WriteLine($"Partition : {Str(root, "partition")}");
output.WriteLine($"Manifest : {Str(root, "manifestStatus")}");
output.WriteLine($"Verified : {(verified ? "yes" : "NO")}");
output.WriteLine($"Files intact: {(root.TryGetProperty("filesIntact", out var f) && f.ValueKind == JsonValueKind.True ? "yes" : "NO")}");
if (Str(root, "fileProblem") is { Length: > 0 } problem)
{
output.WriteLine($" problem : {problem}");
}
output.WriteLine($"Rows read : {Num(root, "rowsRead")}");
if (root.TryGetProperty("chains", out var chains) && chains.ValueKind == JsonValueKind.Array)
{
foreach (var c in chains.EnumerateArray())
{
var chain = c.GetProperty("chain");
output.WriteLine($"Chain {Str(chain, "partition")}: {Str(chain, "status")}, {Num(chain, "rowsVerified")} rows verified, "
+ $"{Num(chain, "tombstonesApplied")} tombstones, {Num(chain, "unchainedRows")} unchained; anchor {Str(c, "anchor")}");
output.WriteLine($" {Str(c, "anchorDetail")}");
if (chain.TryGetProperty("firstBreak", out var brk) && brk.ValueKind == JsonValueKind.Object)
{
output.WriteLine($" first break: chainSeq {Num(brk, "chainSeq")}, eventId {Str(brk, "eventId", "-")}, {Str(brk, "kind")} — {Str(brk, "detail")}");
}
}
}
output.Flush();
return verified ? 0 : NotVerifiedExitCode;
}
private static string Str(JsonElement e, string name, string missing = "")
=> e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() ?? missing : missing;
private static string Num(JsonElement e, string name, string missing = "0")
=> e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.Number ? v.GetInt64().ToString(CultureInfo.InvariantCulture) : missing;
}
@@ -9,7 +9,9 @@ namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
/// the centralized append-only Audit Log via the REST endpoints /// the centralized append-only Audit Log via the REST endpoints
/// (<c>GET /api/audit/query</c>, <c>GET /api/audit/export</c>, /// (<c>GET /api/audit/query</c>, <c>GET /api/audit/export</c>,
/// <c>GET /api/audit/tree</c>), plus <c>verify-chain</c>, which sends the /// <c>GET /api/audit/tree</c>), plus <c>verify-chain</c>, which sends the
/// <c>VerifyAuditChain</c> management command (hash-chain tamper evidence, ScadaBridge#41). /// <c>VerifyAuditChain</c> management command (hash-chain tamper evidence, ScadaBridge#41),
/// and <c>archive list</c> / <c>archive verify</c>, which send <c>ListAuditArchives</c> /
/// <c>VerifyAuditArchive</c> (Parquet cold archive, ScadaBridge#42).
/// </summary> /// </summary>
public static class AuditCommands public static class AuditCommands
{ {
@@ -31,6 +33,7 @@ public static class AuditCommands
command.Add(BuildTree(urlOption, formatOption, usernameOption, passwordOption)); command.Add(BuildTree(urlOption, formatOption, usernameOption, passwordOption));
command.Add(BuildVerifyChain(urlOption, formatOption, usernameOption, passwordOption)); command.Add(BuildVerifyChain(urlOption, formatOption, usernameOption, passwordOption));
command.Add(BuildBackfillSourceNode(urlOption, formatOption, usernameOption, passwordOption)); command.Add(BuildBackfillSourceNode(urlOption, formatOption, usernameOption, passwordOption));
command.Add(BuildArchive(urlOption, formatOption, usernameOption, passwordOption));
return command; return command;
} }
@@ -316,6 +319,49 @@ public static class AuditCommands
return cmd; return cmd;
} }
/// <summary>
/// Builds the <c>audit archive</c> group (ScadaBridge#42): <c>list</c> and <c>verify</c>
/// over the AuditLog Parquet cold archives on the central archive directory.
/// </summary>
private static Command BuildArchive(Option<string> urlOption, Option<string> formatOption, Option<string> usernameOption, Option<string> passwordOption)
{
var group = new Command("archive") { Description = "List and verify the AuditLog Parquet cold archives" };
var list = new Command("list") { Description = "List the AuditLog Parquet archives written by the retention purge" };
list.SetAction(async (ParseResult result) =>
{
var format = AuditCommandHelpers.ResolveFormat(result, formatOption);
return await CommandHelpers.ExecuteCommandAsync(
result, urlOption, formatOption, usernameOption, passwordOption,
new ListAuditArchivesCommand(),
onSuccess: json => AuditArchiveHelpers.RenderList(json, format, Console.Out));
});
group.Add(list);
var nameOption = new Option<string>("--name")
{
Description = "Archive name, as 'audit archive list' prints it (auditlog-YYYY-MM-yyyyMMddTHHmmssfffZ)",
Required = true,
};
var verify = new Command("verify")
{
Description = "Verify an archive's files, its hash chain and its purge anchor; exit 3 if it does not verify",
};
verify.Add(nameOption);
verify.SetAction(async (ParseResult result) =>
{
var format = AuditCommandHelpers.ResolveFormat(result, formatOption);
return await CommandHelpers.ExecuteCommandAsync(
result, urlOption, formatOption, usernameOption, passwordOption,
new VerifyAuditArchiveCommand(result.GetValue(nameOption)!),
timeout: AuditArchiveHelpers.VerifyRequestTimeout,
onSuccess: json => AuditArchiveHelpers.RenderVerify(json, format, Console.Out));
});
group.Add(verify);
return group;
}
/// <summary> /// <summary>
/// Builds the <c>audit backfill-source-node</c> sub-command. /// Builds the <c>audit backfill-source-node</c> sub-command.
/// Sets <c>SourceNode</c> on historical pre-feature rows whose <c>SourceNode IS NULL</c> /// Sets <c>SourceNode</c> on historical pre-feature rows whose <c>SourceNode IS NULL</c>
@@ -134,8 +134,8 @@ public static class AuditExportHelpers
/// Executes the export: GETs <c>/api/audit/export</c> and copies the response body /// Executes the export: GETs <c>/api/audit/export</c> and copies the response body
/// stream directly to <see cref="AuditExportArgs.Output"/>. The body is never fully /// stream directly to <see cref="AuditExportArgs.Output"/>. The body is never fully
/// buffered — the copy streams in fixed-size chunks via <see cref="Stream.ReadAsync(Memory{byte}, CancellationToken)"/>. /// buffered — the copy streams in fixed-size chunks via <see cref="Stream.ReadAsync(Memory{byte}, CancellationToken)"/>.
/// A <c>501 Not Implemented</c> (parquet not yet supported server-side) prints the /// A <c>501 Not Implemented</c> (a central older than ScadaBridge#42 does not serve
/// server message and returns a non-zero exit code. /// <c>format=parquet</c>) prints the server message and returns a non-zero exit code.
/// </summary> /// </summary>
/// <param name="client">The management HTTP client for API communication.</param> /// <param name="client">The management HTTP client for API communication.</param>
/// <param name="args">The export arguments containing filters and output file path.</param> /// <param name="args">The export arguments containing filters and output file path.</param>
@@ -31,3 +31,19 @@ public record AuditLogQueryResult(IReadOnlyList<AuditLogEntry> Entries, int Tota
/// </summary> /// </summary>
/// <param name="Month">The month to verify, <c>YYYY-MM</c>.</param> /// <param name="Month">The month to verify, <c>YYYY-MM</c>.</param>
public record VerifyAuditChainCommand(string Month); public record VerifyAuditChainCommand(string Month);
/// <summary>
/// Lists the AuditLog Parquet cold archives under the configured archive directory (audit
/// roadmap T2, ScadaBridge#42). Read-only; OperationalAudit roles, refused for a site-scoped
/// caller because an archive holds every site's rows.
/// </summary>
public record ListAuditArchivesCommand;
/// <summary>
/// Verifies one AuditLog Parquet archive (ScadaBridge#42): file integrity against its
/// manifest, the hash chain recomputed over the archived rows, and the chain end against the
/// checkpoint anchor the partition purge left. Read-only; same gate as
/// <see cref="ListAuditArchivesCommand"/>.
/// </summary>
/// <param name="Name">The archive name, as <c>audit archive list</c> reports it.</param>
public record VerifyAuditArchiveCommand(string Name);
@@ -22,8 +22,8 @@ namespace ZB.MOM.WW.ScadaBridge.ManagementService;
/// <list type="bullet"> /// <list type="bullet">
/// <item><c>GET /api/audit/query</c> — keyset-paged JSON page, gated on the /// <item><c>GET /api/audit/query</c> — keyset-paged JSON page, gated on the
/// <see cref="AuthorizationPolicies.OperationalAudit"/> permission.</item> /// <see cref="AuthorizationPolicies.OperationalAudit"/> permission.</item>
/// <item><c>GET /api/audit/export</c> — streamed bulk export (csv / jsonl; /// <item><c>GET /api/audit/export</c> — streamed bulk export (csv / jsonl /
/// parquet returns HTTP 501), gated on the /// parquet), gated on the
/// <see cref="AuthorizationPolicies.AuditExport"/> permission.</item> /// <see cref="AuthorizationPolicies.AuditExport"/> permission.</item>
/// <item><c>GET /api/audit/tree</c> — execution-chain tree rooted at the /// <item><c>GET /api/audit/tree</c> — execution-chain tree rooted at the
/// topmost ancestor of a given <c>executionId</c>, returned as a JSON array /// topmost ancestor of a given <c>executionId</c>, returned as a JSON array
@@ -47,11 +47,8 @@ namespace ZB.MOM.WW.ScadaBridge.ManagementService;
/// </para> /// </para>
/// ///
/// <para> /// <para>
/// <b>Parquet.</b> No Parquet library is referenced by the solution and /// <b>Parquet.</b> <c>format=parquet</c> (ScadaBridge#42) streams the same 24 columns as
/// <c>Component-AuditLog.md</c> explicitly defers Parquet archival to v1.x, so /// csv / jsonl as one Parquet file (<see cref="AuditExportParquet"/>), a row group at a time.
/// <c>format=parquet</c> returns <c>501 Not Implemented</c> with a guidance
/// message rather than a half-built binary stream. csv + jsonl are fully
/// implemented.
/// </para> /// </para>
/// </summary> /// </summary>
public static class AuditEndpoints public static class AuditEndpoints
@@ -65,6 +62,9 @@ public static class AuditEndpoints
/// <summary>Repository round-trip size used while streaming an export.</summary> /// <summary>Repository round-trip size used while streaming an export.</summary>
public const int ExportPageSize = 1000; public const int ExportPageSize = 1000;
/// <summary>Content type of a <c>format=parquet</c> export (ScadaBridge#42).</summary>
public const string ParquetContentType = "application/vnd.apache.parquet";
/// <summary> /// <summary>
/// Serializer for individual <see cref="AuditEvent"/> rows (query results + /// Serializer for individual <see cref="AuditEvent"/> rows (query results +
/// jsonl export lines). Drops null columns so a sparse audit row stays /// jsonl export lines). Drops null columns so a sparse audit row stays
@@ -185,10 +185,10 @@ public static class AuditEndpoints
// ───────────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────────
/// <summary> /// <summary>
/// Handles <c>GET /api/audit/export</c>: authenticates, checks the AuditExport permission, and streams the matching rows as CSV or JSONL. /// Handles <c>GET /api/audit/export</c>: authenticates, checks the AuditExport permission, and streams the matching rows as CSV, JSONL or Parquet.
/// </summary> /// </summary>
/// <param name="context">The HTTP context for the current request.</param> /// <param name="context">The HTTP context for the current request.</param>
/// <returns>A task that resolves to the HTTP result (streaming CSV/JSONL, 400, 401, 403, or 501).</returns> /// <returns>A task that resolves to the HTTP result (streaming CSV/JSONL/Parquet, 400, 401, or 403).</returns>
internal static async Task<IResult> HandleExport(HttpContext context) internal static async Task<IResult> HandleExport(HttpContext context)
{ {
var auth = await AuthenticateAsync(context); var auth = await AuthenticateAsync(context);
@@ -208,17 +208,7 @@ public static class AuditEndpoints
format = "csv"; format = "csv";
} }
if (format == "parquet") if (format != "csv" && format != "jsonl" && format != "parquet")
{
// Deferred to v1.x (Component-AuditLog.md §"Deferred"): no Parquet
// library is in the solution. Return a 501 with actionable guidance
// rather than a partial binary stream.
return Results.Json(
new { error = "Parquet export deferred to v1.x; use csv or jsonl.", code = "NOT_IMPLEMENTED" },
statusCode: 501);
}
if (format != "csv" && format != "jsonl")
{ {
return Results.Json( return Results.Json(
new { error = $"Unsupported export format '{format}'. Use csv, jsonl, or parquet.", code = "BAD_REQUEST" }, new { error = $"Unsupported export format '{format}'. Use csv, jsonl, or parquet.", code = "BAD_REQUEST" },
@@ -243,8 +233,12 @@ public static class AuditEndpoints
filter = restricted; filter = restricted;
var repo = context.RequestServices.GetRequiredService<IAuditLogRepository>(); var repo = context.RequestServices.GetRequiredService<IAuditLogRepository>();
var contentType = format == "csv" ? "text/csv; charset=utf-8" : "application/x-ndjson"; var (contentType, extension) = format switch
var extension = format == "csv" ? "csv" : "jsonl"; {
"csv" => ("text/csv; charset=utf-8", "csv"),
"parquet" => (ParquetContentType, "parquet"),
_ => ("application/x-ndjson", "jsonl"),
};
var fileName = $"audit-log-{DateTime.UtcNow:yyyyMMdd-HHmmss}.{extension}"; var fileName = $"audit-log-{DateTime.UtcNow:yyyyMMdd-HHmmss}.{extension}";
context.Response.ContentType = contentType; context.Response.ContentType = contentType;
@@ -256,6 +250,12 @@ public static class AuditEndpoints
{ {
await StreamCsvAsync(repo, filter, context.Response.Body, context.RequestAborted); await StreamCsvAsync(repo, filter, context.Response.Body, context.RequestAborted);
} }
else if (format == "parquet")
{
// ScadaBridge#42: same 24 columns as csv/jsonl, one row group per 10k rows.
await AuditExportParquet.WriteAsync(
ExportRowPagesAsync(repo, filter, context.RequestAborted), context.Response.Body, context.RequestAborted);
}
else else
{ {
await StreamJsonlAsync(repo, filter, context.Response.Body, context.RequestAborted); await StreamJsonlAsync(repo, filter, context.Response.Body, context.RequestAborted);
@@ -494,6 +494,18 @@ public static class AuditEndpoints
} }
} }
/// <summary>Projects <see cref="PageAsync"/> pages onto the flat export row shape.</summary>
private static async IAsyncEnumerable<IReadOnlyList<AuditExportRow>> ExportRowPagesAsync(
IAuditLogRepository repo,
AuditLogQueryFilter filter,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct)
{
await foreach (var page in PageAsync(repo, filter, ct))
{
yield return page.Select(AuditExportRow.From).ToList();
}
}
/// <summary> /// <summary>
/// Lazily yields full repository pages, advancing the keyset cursor until a /// Lazily yields full repository pages, advancing the keyset cursor until a
/// short page signals the end. Shared by the csv + jsonl streamers. /// short page signals the end. Shared by the csv + jsonl streamers.
@@ -0,0 +1,219 @@
using Parquet;
using Parquet.Schema;
namespace ZB.MOM.WW.ScadaBridge.ManagementService;
/// <summary>
/// <c>GET /api/audit/export?format=parquet</c> (ScadaBridge#42): the same 24 columns as the
/// csv / jsonl export (<see cref="AuditExportRow"/>, in <see cref="AuditEndpoints.CsvHeader"/>
/// order) as one Parquet file, one row group per <see cref="RowGroupRows"/> rows.
/// </summary>
/// <remarks>
/// <para>
/// Types: ids are <c>D</c>-format strings, timestamps are UTC Parquet timestamps
/// (microseconds), enums are their names, <c>HttpStatus</c>/<c>DurationMs</c> are int32,
/// <c>PayloadTruncated</c> is boolean, everything else is a UTF-8 string.
/// </para>
/// <para>
/// <b>Streaming.</b> Kestrel forbids synchronous writes to the response body and the Parquet
/// writer issues some, so each row group is encoded into a small in-memory buffer and then
/// copied to the response asynchronously. Memory is bounded by one row group, and the client
/// starts receiving bytes after the first one. A Parquet file's footer is written last, so a
/// response cut off mid-stream is not a readable file — the CLI treats it as a failed export.
/// </para>
/// <para>
/// This is the ad-hoc export. The retention archive (alog.md §15.2) is a different layout
/// that carries the hash-chain columns; see <c>AuditArchiveParquet</c>.
/// </para>
/// </remarks>
public static class AuditExportParquet
{
/// <summary>Rows per Parquet row group.</summary>
public const int RowGroupRows = 10_000;
private static readonly DataField<string> EventId = new(nameof(AuditExportRow.EventId), false);
private static readonly DateTimeDataField OccurredAtUtc = Timestamp(nameof(AuditExportRow.OccurredAtUtc), false);
private static readonly DateTimeDataField IngestedAtUtc = Timestamp(nameof(AuditExportRow.IngestedAtUtc), true);
private static readonly DataField<string> Channel = new(nameof(AuditExportRow.Channel), false);
private static readonly DataField<string> Kind = new(nameof(AuditExportRow.Kind), false);
private static readonly DataField<string> CorrelationId = Text(nameof(AuditExportRow.CorrelationId));
private static readonly DataField<string> ExecutionId = Text(nameof(AuditExportRow.ExecutionId));
private static readonly DataField<string> ParentExecutionId = Text(nameof(AuditExportRow.ParentExecutionId));
private static readonly DataField<string> SourceSiteId = Text(nameof(AuditExportRow.SourceSiteId));
private static readonly DataField<string> SourceNode = Text(nameof(AuditExportRow.SourceNode));
private static readonly DataField<string> SourceInstanceId = Text(nameof(AuditExportRow.SourceInstanceId));
private static readonly DataField<string> SourceScript = Text(nameof(AuditExportRow.SourceScript));
private static readonly DataField<string> Actor = Text(nameof(AuditExportRow.Actor));
private static readonly DataField<string> Target = Text(nameof(AuditExportRow.Target));
private static readonly DataField<string> Status = new(nameof(AuditExportRow.Status), false);
private static readonly DataField<int> HttpStatus = new(nameof(AuditExportRow.HttpStatus), true);
private static readonly DataField<int> DurationMs = new(nameof(AuditExportRow.DurationMs), true);
private static readonly DataField<string> ErrorMessage = Text(nameof(AuditExportRow.ErrorMessage));
private static readonly DataField<string> ErrorDetail = Text(nameof(AuditExportRow.ErrorDetail));
private static readonly DataField<string> RequestSummary = Text(nameof(AuditExportRow.RequestSummary));
private static readonly DataField<string> ResponseSummary = Text(nameof(AuditExportRow.ResponseSummary));
private static readonly DataField<bool> PayloadTruncated = new(nameof(AuditExportRow.PayloadTruncated), false);
private static readonly DataField<string> Extra = Text(nameof(AuditExportRow.Extra));
private static readonly DataField<string> ForwardState = Text(nameof(AuditExportRow.ForwardState));
/// <summary>The export schema; column names and order equal <see cref="AuditEndpoints.CsvHeader"/>.</summary>
public static ParquetSchema Schema { get; } = new(
EventId, OccurredAtUtc, IngestedAtUtc, Channel, Kind, CorrelationId, ExecutionId, ParentExecutionId,
SourceSiteId, SourceNode, SourceInstanceId, SourceScript, Actor, Target, Status, HttpStatus, DurationMs,
ErrorMessage, ErrorDetail, RequestSummary, ResponseSummary, PayloadTruncated, Extra, ForwardState);
/// <summary>Writes every row of <paramref name="pages"/> to <paramref name="output"/> as one Parquet file.</summary>
/// <param name="pages">Export rows, page by page.</param>
/// <param name="output">The destination (e.g. the HTTP response body); written asynchronously only.</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>The number of rows written.</returns>
public static async Task<long> WriteAsync(
IAsyncEnumerable<IReadOnlyList<AuditExportRow>> pages, Stream output, CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(pages);
ArgumentNullException.ThrowIfNull(output);
// Non-seekable on purpose: the writer then tracks file offsets itself, so emptying the
// buffer after each drain cannot corrupt the column-chunk offsets in the footer.
var staging = new StagingStream();
long total = 0;
var buffer = new List<AuditExportRow>(RowGroupRows);
await using (var writer = await ParquetWriter.CreateAsync(Schema, staging, cancellationToken: ct).ConfigureAwait(false))
{
await foreach (var page in pages.WithCancellation(ct).ConfigureAwait(false))
{
foreach (var row in page)
{
buffer.Add(row);
if (buffer.Count == RowGroupRows)
{
total += await WriteGroupAsync(writer, buffer, ct).ConfigureAwait(false);
await DrainAsync(staging, output, ct).ConfigureAwait(false);
}
}
}
total += await WriteGroupAsync(writer, buffer, ct).ConfigureAwait(false);
}
// The footer is written when the writer is disposed.
await DrainAsync(staging, output, ct).ConfigureAwait(false);
return total;
}
private static async Task<int> WriteGroupAsync(ParquetWriter writer, List<AuditExportRow> rows, CancellationToken ct)
{
if (rows.Count == 0)
{
return 0;
}
using (var group = writer.CreateRowGroup())
{
await group.WriteAsync(EventId, Map(rows, r => r.EventId.ToString("D"))).ConfigureAwait(false);
await group.WriteAsync<DateTime>(OccurredAtUtc, rows.Select(r => DateTime.SpecifyKind(r.OccurredAtUtc, DateTimeKind.Utc)).ToArray(), cancellationToken: ct).ConfigureAwait(false);
await group.WriteAsync<DateTime>(IngestedAtUtc, rows.Select(r => r.IngestedAtUtc is { } i ? DateTime.SpecifyKind(i, DateTimeKind.Utc) : (DateTime?)null).ToArray(), cancellationToken: ct).ConfigureAwait(false);
await group.WriteAsync(Channel, Map(rows, r => r.Channel.ToString())).ConfigureAwait(false);
await group.WriteAsync(Kind, Map(rows, r => r.Kind.ToString())).ConfigureAwait(false);
await group.WriteAsync(CorrelationId, Map(rows, r => r.CorrelationId?.ToString("D"))!).ConfigureAwait(false);
await group.WriteAsync(ExecutionId, Map(rows, r => r.ExecutionId?.ToString("D"))!).ConfigureAwait(false);
await group.WriteAsync(ParentExecutionId, Map(rows, r => r.ParentExecutionId?.ToString("D"))!).ConfigureAwait(false);
await group.WriteAsync(SourceSiteId, Map(rows, r => r.SourceSiteId)!).ConfigureAwait(false);
await group.WriteAsync(SourceNode, Map(rows, r => r.SourceNode)!).ConfigureAwait(false);
await group.WriteAsync(SourceInstanceId, Map(rows, r => r.SourceInstanceId)!).ConfigureAwait(false);
await group.WriteAsync(SourceScript, Map(rows, r => r.SourceScript)!).ConfigureAwait(false);
await group.WriteAsync(Actor, Map(rows, r => r.Actor)!).ConfigureAwait(false);
await group.WriteAsync(Target, Map(rows, r => r.Target)!).ConfigureAwait(false);
await group.WriteAsync(Status, Map(rows, r => r.Status.ToString())).ConfigureAwait(false);
await group.WriteAsync<int>(HttpStatus, rows.Select(r => r.HttpStatus).ToArray(), cancellationToken: ct).ConfigureAwait(false);
await group.WriteAsync<int>(DurationMs, rows.Select(r => r.DurationMs).ToArray(), cancellationToken: ct).ConfigureAwait(false);
await group.WriteAsync(ErrorMessage, Map(rows, r => r.ErrorMessage)!).ConfigureAwait(false);
await group.WriteAsync(ErrorDetail, Map(rows, r => r.ErrorDetail)!).ConfigureAwait(false);
await group.WriteAsync(RequestSummary, Map(rows, r => r.RequestSummary)!).ConfigureAwait(false);
await group.WriteAsync(ResponseSummary, Map(rows, r => r.ResponseSummary)!).ConfigureAwait(false);
await group.WriteAsync<bool>(PayloadTruncated, rows.Select(r => r.PayloadTruncated).ToArray(), cancellationToken: ct).ConfigureAwait(false);
await group.WriteAsync(Extra, Map(rows, r => r.Extra)!).ConfigureAwait(false);
await group.WriteAsync(ForwardState, Map(rows, r => r.ForwardState?.ToString())!).ConfigureAwait(false);
}
var count = rows.Count;
rows.Clear();
return count;
}
private static async Task DrainAsync(StagingStream staging, Stream output, CancellationToken ct)
{
var pending = staging.Pending;
if (pending.Length == 0)
{
return;
}
await output.WriteAsync(pending, ct).ConfigureAwait(false);
await output.FlushAsync(ct).ConfigureAwait(false);
staging.Clear();
}
private static string?[] Map(List<AuditExportRow> rows, Func<AuditExportRow, string?> select)
=> rows.Select(select).ToArray();
private static DataField<string> Text(string name) => new(name, true);
private static DateTimeDataField Timestamp(string name, bool nullable)
=> new(name, DateTimeFormat.Timestamp, isAdjustedToUTC: true, unit: DateTimeTimeUnit.Micros, isNullable: nullable);
/// <summary>
/// Forward-only, non-seekable buffer the Parquet writer writes into (synchronously or
/// not); <see cref="DrainAsync"/> moves its bytes to the real output asynchronously.
/// </summary>
private sealed class StagingStream : Stream
{
private readonly MemoryStream _buffer = new();
public ReadOnlyMemory<byte> Pending => _buffer.GetBuffer().AsMemory(0, (int)_buffer.Length);
public override bool CanRead => false;
public override bool CanSeek => false;
public override bool CanWrite => true;
public override long Length => throw new NotSupportedException();
public override long Position
{
get => throw new NotSupportedException();
set => throw new NotSupportedException();
}
public void Clear() => _buffer.SetLength(0);
public override void Write(byte[] buffer, int offset, int count) => _buffer.Write(buffer, offset, count);
public override void Write(ReadOnlySpan<byte> buffer) => _buffer.Write(buffer);
public override ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default)
{
_buffer.Write(buffer.Span);
return ValueTask.CompletedTask;
}
public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
{
_buffer.Write(buffer, offset, count);
return Task.CompletedTask;
}
public override void Flush()
{
}
public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask;
public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
}
}
@@ -393,6 +393,11 @@ public class ManagementActor : ReceiveActor
// spans every site. // spans every site.
VerifyAuditChainCommand => OperationalAuditReaders, VerifyAuditChainCommand => OperationalAuditReaders,
// AuditLog Parquet archives (ScadaBridge#42): audit reads with verify-chain's gate —
// OperationalAudit roles, site-scoped callers refused in the handler.
ListAuditArchivesCommand => OperationalAuditReaders,
VerifyAuditArchiveCommand => OperationalAuditReaders,
// Read-only queries -- any authenticated user // Read-only queries -- any authenticated user
_ => null _ => null
}; };
@@ -564,6 +569,8 @@ public class ManagementActor : ReceiveActor
// Audit Log // Audit Log
QueryAuditLogCommand cmd => await HandleQueryAuditLog(sp, cmd), QueryAuditLogCommand cmd => await HandleQueryAuditLog(sp, cmd),
VerifyAuditChainCommand cmd => await HandleVerifyAuditChain(sp, cmd, user), VerifyAuditChainCommand cmd => await HandleVerifyAuditChain(sp, cmd, user),
ListAuditArchivesCommand => await HandleListAuditArchives(sp, user),
VerifyAuditArchiveCommand cmd => await HandleVerifyAuditArchive(sp, cmd, user),
// Health // Health
GetHealthSummaryCommand => HandleGetHealthSummary(sp), GetHealthSummaryCommand => HandleGetHealthSummary(sp),
@@ -3256,6 +3263,59 @@ public class ManagementActor : ReceiveActor
} }
} }
/// <summary>
/// Lists the AuditLog Parquet archives (ScadaBridge#42). Refused for a site-scoped,
/// non-Administrator caller: an archive holds every site's rows.
/// </summary>
private static async Task<object?> HandleListAuditArchives(IServiceProvider sp, AuthenticatedUser user)
{
var archive = RequireAuditArchive(sp, user);
return new AuditArchiveListResult(archive.IsEnabled, await archive.ListAsync());
}
/// <summary>
/// Verifies one AuditLog Parquet archive (ScadaBridge#42): files against the manifest,
/// the chain recomputed from the archived rows, the chain end against the purge anchor.
/// </summary>
private static async Task<object?> HandleVerifyAuditArchive(
IServiceProvider sp, VerifyAuditArchiveCommand cmd, AuthenticatedUser user)
{
var archive = RequireAuditArchive(sp, user);
try
{
return await archive.VerifyAsync(cmd.Name);
}
catch (ArgumentException ex)
{
throw new ManagementCommandException(ex.Message, "INVALID_ARGUMENT");
}
catch (FileNotFoundException ex)
{
throw new ManagementCommandException(ex.Message, "NOT_FOUND");
}
}
private static IAuditLogArchive RequireAuditArchive(IServiceProvider sp, AuthenticatedUser user)
{
if (user.PermittedSiteIds.Length > 0
&& !user.Roles.Contains(Roles.Administrator, StringComparer.OrdinalIgnoreCase))
{
throw new SiteScopeViolationException(
"Access denied: AuditLog archives hold every site's rows; they need an "
+ "OperationalAudit role that is not scoped to specific sites.");
}
var archive = sp.GetService<IAuditLogArchive>();
if (archive is null || !archive.IsConfigured)
{
throw new ManagementCommandException(
"No AuditLog archive directory is configured on this central node (AuditLog:Archive:Directory).",
"ARCHIVE_NOT_CONFIGURED");
}
return archive;
}
// ======================================================================== // ========================================================================
// Health handlers // Health handlers
// ======================================================================== // ========================================================================
@@ -63,6 +63,9 @@ public static class ManagementEndpoints
// AuditLog hash-chain verification (ScadaBridge#41) recomputes one hash per row of // AuditLog hash-chain verification (ScadaBridge#41) recomputes one hash per row of
// a whole month — millions of rows on a busy central. // a whole month — millions of rows on a busy central.
typeof(VerifyAuditChainCommand), typeof(VerifyAuditChainCommand),
// AuditLog archive verification (ScadaBridge#42) hashes and re-chains a whole month's
// Parquet archive.
typeof(VerifyAuditArchiveCommand),
]; ];
/// <summary> /// <summary>
@@ -12,6 +12,7 @@
<InternalsVisibleTo Include="ZB.MOM.WW.ScadaBridge.ManagementService.Tests" /> <InternalsVisibleTo Include="ZB.MOM.WW.ScadaBridge.ManagementService.Tests" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<PackageReference Include="Parquet.Net" />
<PackageReference Include="Akka" /> <PackageReference Include="Akka" />
<PackageReference Include="Akka.Cluster.Tools" /> <PackageReference Include="Akka.Cluster.Tools" />
<PackageReference Include="ZB.MOM.WW.Configuration" /> <PackageReference Include="ZB.MOM.WW.Configuration" />
@@ -69,7 +69,7 @@ public class CommandTreeTests
Assert.Contains("bundle", groupNames); Assert.Contains("bundle", groupNames);
} }
/// <summary>Verifies that the audit command group exposes the query, export and verify-chain subcommands.</summary> /// <summary>Verifies that the audit command group exposes the query, export, verify-chain and archive subcommands.</summary>
[Fact] [Fact]
public void AuditCommandGroup_HasQueryExportAndVerifyChain() public void AuditCommandGroup_HasQueryExportAndVerifyChain()
{ {
@@ -80,6 +80,7 @@ public class CommandTreeTests
Assert.Contains("query", subNames); Assert.Contains("query", subNames);
Assert.Contains("export", subNames); Assert.Contains("export", subNames);
Assert.Contains("verify-chain", subNames); Assert.Contains("verify-chain", subNames);
Assert.Contains("archive", subNames);
} }
/// <summary>Verifies that the bundle command group exposes the export, preview and import subcommands.</summary> /// <summary>Verifies that the bundle command group exposes the export, preview and import subcommands.</summary>
@@ -0,0 +1,136 @@
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Text.Json;
using ZB.MOM.WW.ScadaBridge.CLI.Commands;
namespace ZB.MOM.WW.ScadaBridge.CLI.Tests.Commands;
/// <summary>
/// Tests for <c>scadabridge audit archive list</c> / <c>verify</c> (ScadaBridge#42): each
/// sends its management command (<c>ListAuditArchives</c>, <c>VerifyAuditArchive</c>), and
/// <c>verify</c> maps the verdict onto the exit code (0 verified, 3 not verified).
/// </summary>
[Collection("Console")]
public class AuditArchiveCommandTests
{
private const string Name = "auditlog-2025-10-20261109T120000123Z";
private const string ListBody =
"{\"enabled\":true,\"archives\":[{\"name\":\"" + Name + "\",\"partition\":\"2025-10\",\"status\":\"Purged\"," +
"\"rowCount\":11,\"chainedRowCount\":10,\"dataBytes\":4096}]}";
private const string VerifiedBody =
"{\"name\":\"" + Name + "\",\"partition\":\"2025-10\",\"manifestStatus\":\"Purged\",\"verified\":true,\"filesIntact\":true," +
"\"rowsRead\":11,\"chains\":[{\"chain\":{\"partition\":\"2025-10\",\"status\":\"Intact\",\"rowsVerified\":10," +
"\"tombstonesApplied\":2,\"unchainedRows\":1},\"anchor\":\"Matches\",\"anchorDetail\":\"ends at the anchor\"}]}";
private const string MismatchBody =
"{\"name\":\"" + Name + "\",\"partition\":\"2025-10\",\"manifestStatus\":\"Purged\",\"verified\":false,\"filesIntact\":true," +
"\"rowsRead\":11,\"chains\":[{\"chain\":{\"partition\":\"2025-10\",\"status\":\"Intact\",\"rowsVerified\":10," +
"\"tombstonesApplied\":0,\"unchainedRows\":0},\"anchor\":\"Mismatch\",\"anchorDetail\":\"hash differs\"}]}";
/// <summary>The audit group carries the archive group with list + verify.</summary>
[Fact]
public void AuditGroup_HasArchiveListAndVerify()
{
var root = AuditCommandTestHarness.BuildRoot();
var archive = root.Subcommands.Single(c => c.Name == "audit").Subcommands.Single(c => c.Name == "archive");
Assert.Equal(["list", "verify"], archive.Subcommands.Select(c => c.Name).Order());
}
/// <summary><c>list</c> sends ListAuditArchives and prints the archives.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task List_SendsListAuditArchives_AndPrintsTheArchives()
{
var (exit, output, request) = await RunAsync(ListBody, "audit", "archive", "list", "--format", "table");
Assert.Equal(0, exit);
Assert.Contains(Name, output);
Assert.Contains("Archiving: enabled", output);
using var payload = JsonDocument.Parse(request);
Assert.Equal("ListAuditArchives", payload.RootElement.GetProperty("command").GetString());
}
/// <summary><c>verify</c> sends VerifyAuditArchive with the name and maps the verdict to the exit code.</summary>
/// <param name="body">The verification result the fake endpoint serves.</param>
/// <param name="format">Output format.</param>
/// <param name="expectedExit">Expected exit code.</param>
/// <param name="expectedText">Text the report must contain.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
[Theory]
[InlineData(VerifiedBody, "table", 0, "anchor Matches")]
[InlineData(VerifiedBody, "json", 0, "\"verified\":true")]
[InlineData(MismatchBody, "table", 3, "Verified : NO")]
[InlineData(MismatchBody, "json", 3, "\"anchor\":\"Mismatch\"")]
public async Task Verify_SendsVerifyAuditArchive_AndMapsVerdictToExitCode(
string body, string format, int expectedExit, string expectedText)
{
var (exit, output, request) = await RunAsync(body, "audit", "archive", "verify", "--name", Name, "--format", format);
Assert.Equal(expectedExit, exit);
Assert.Contains(expectedText, output);
using var payload = JsonDocument.Parse(request);
Assert.Equal("VerifyAuditArchive", payload.RootElement.GetProperty("command").GetString());
var name = payload.RootElement.GetProperty("payload").EnumerateObject()
.First(p => string.Equals(p.Name, "name", StringComparison.OrdinalIgnoreCase)).Value.GetString();
Assert.Equal(Name, name);
}
/// <summary><c>verify</c> without <c>--name</c> is a usage error.</summary>
[Fact]
public void Verify_MissingName_ProducesRequiredFlagError()
{
var root = AuditCommandTestHarness.BuildRoot();
var (exit, _, err) = AuditCommandTestHarness.Invoke(root, "audit", "archive", "verify");
Assert.NotEqual(0, exit);
Assert.Contains("--name", err);
}
private static async Task<(int Exit, string Output, string Request)> RunAsync(string body, params string[] args)
{
var listener = new TcpListener(IPAddress.Loopback, 0);
listener.Start();
try
{
var port = ((IPEndPoint)listener.LocalEndpoint).Port;
string? requestBody = null;
var server = Task.Run(async () =>
{
using var client = await listener.AcceptTcpClientAsync();
await using var stream = client.GetStream();
using var reader = new StreamReader(stream, Encoding.UTF8, leaveOpen: true);
var length = 0;
while (await reader.ReadLineAsync() is { Length: > 0 } line)
if (line.StartsWith("Content-Length:", StringComparison.OrdinalIgnoreCase))
length = int.Parse(line[15..].Trim());
var buffer = new char[length];
var read = 0;
while (read < length)
{
var count = await reader.ReadAsync(buffer.AsMemory(read));
if (count == 0) throw new EndOfStreamException();
read += count;
}
requestBody = new string(buffer);
var bytes = Encoding.UTF8.GetBytes(body);
var headers = Encoding.ASCII.GetBytes(
$"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {bytes.Length}\r\nConnection: close\r\n\r\n");
await stream.WriteAsync(headers);
await stream.WriteAsync(bytes);
});
var root = AuditCommandTestHarness.BuildRoot();
var (exit, output, _) = AuditCommandTestHarness.Invoke(
root, [.. args, "--url", $"http://127.0.0.1:{port}", "--username", "u", "--password", "p"]);
await server.WaitAsync(TimeSpan.FromSeconds(5));
return (exit, output, requestBody!);
}
finally
{
listener.Stop();
}
}
}
@@ -27,14 +27,14 @@ public class AuditCommandsScaffoldTests
Assert.False(string.IsNullOrWhiteSpace(audit.Description)); Assert.False(string.IsNullOrWhiteSpace(audit.Description));
} }
/// <summary>Verifies that the audit group exposes the query, export, tree, verify-chain and backfill-source-node subcommands.</summary> /// <summary>Verifies that the audit group exposes the query, export, tree, verify-chain, backfill-source-node and archive subcommands.</summary>
[Fact] [Fact]
public void Audit_HasFiveSubcommands_QueryExportTreeVerifyChainBackfillSourceNode() public void Audit_HasSixSubcommands_QueryExportTreeVerifyChainBackfillSourceNodeArchive()
{ {
var audit = BuildAudit(); var audit = BuildAudit();
var names = audit.Subcommands.Select(c => c.Name).OrderBy(n => n).ToArray(); var names = audit.Subcommands.Select(c => c.Name).OrderBy(n => n).ToArray();
Assert.Equal( Assert.Equal(
new[] { "backfill-source-node", "export", "query", "tree", "verify-chain" }, new[] { "archive", "backfill-source-node", "export", "query", "tree", "verify-chain" },
names); names);
} }
@@ -62,8 +62,11 @@ public class AuditCommandsScaffoldTests
[Fact] [Fact]
public void Audit_EveryLeafCommand_HasAnAction() public void Audit_EveryLeafCommand_HasAnAction()
{ {
// `archive` is a group (list / verify, ScadaBridge#42): check its leaves, not it.
var audit = BuildAudit(); var audit = BuildAudit();
Assert.All(audit.Subcommands, sub => var leaves = audit.Subcommands.SelectMany(c => c.Subcommands.Count == 0 ? [c] : c.Subcommands).ToList();
Assert.Equal(7, leaves.Count);
Assert.All(leaves, sub =>
Assert.True(sub.Action != null, $"Leaf command '{sub.Name}' has no action.")); Assert.True(sub.Action != null, $"Leaf command '{sub.Name}' has no action."));
} }
} }
@@ -44,6 +44,9 @@ public class ManagementSurfaceParityTests
Assert.Contains("ListSharedScripts", sent.Keys); Assert.Contains("ListSharedScripts", sent.Keys);
} }
private static readonly HashSet<string> AuditManagementLeaves =
["audit verify-chain", "audit archive list", "audit archive verify"];
private static async Task<ConcurrentDictionary<string, string>> ProbeProductionTree(string? omitGroup = null) private static async Task<ConcurrentDictionary<string, string>> ProbeProductionTree(string? omitGroup = null)
{ {
var root = CliApplication.BuildRoot(); var root = CliApplication.BuildRoot();
@@ -93,9 +96,10 @@ public class ManagementSurfaceParityTests
foreach (var (leaf, path) in Leaves(root, [])) foreach (var (leaf, path) in Leaves(root, []))
{ {
// Separate REST and persistent SignalR surfaces, not management contracts. // Separate REST and persistent SignalR surfaces, not management contracts.
// `audit verify-chain` is the one audit leaf that IS a management contract // `audit verify-chain` (VerifyAuditChain, ScadaBridge#41) and `audit archive
// (VerifyAuditChain, ScadaBridge#41), so it stays in the probe. // list|verify` (ListAuditArchives / VerifyAuditArchive, ScadaBridge#42) are the
if (path.Length == 0 || (path[0] == "audit" && string.Join(' ', path) != "audit verify-chain") // audit leaves that ARE management contracts, so they stay in the probe.
if (path.Length == 0 || (path[0] == "audit" && !AuditManagementLeaves.Contains(string.Join(' ', path)))
|| path[0] == omitGroup || path[0] == omitGroup
|| string.Join(' ', path) == "debug stream") continue; || string.Join(' ', path) == "debug stream") continue;
currentPath = string.Join(' ', path); currentPath = string.Join(' ', path);
@@ -0,0 +1,147 @@
using System.Text.Json;
using Akka.Actor;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
using NSubstitute;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Management;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
namespace ZB.MOM.WW.ScadaBridge.ManagementService.Tests;
/// <summary>
/// The <c>ListAuditArchives</c> / <c>VerifyAuditArchive</c> management commands
/// (ScadaBridge#42): role gate, site-scope refusal, an unconfigured archive directory, bad
/// names, and the results reaching the wire with enums as names.
/// </summary>
public class AuditArchiveCommandTests : TestKit
{
private const string Name = "auditlog-2025-10-20261109T120000123Z";
private readonly IAuditLogArchive _archive = Substitute.For<IAuditLogArchive>();
/// <summary>Configures the archive substitute as a configured, enabled archive.</summary>
public AuditArchiveCommandTests()
{
_archive.IsConfigured.Returns(true);
_archive.IsEnabled.Returns(true);
}
private IActorRef CreateActor(bool registerArchive = true)
{
var services = new ServiceCollection().AddTestConfigurationTransaction();
if (registerArchive)
{
services.AddScoped(_ => _archive);
}
return Sys.ActorOf(Props.Create(() => new ManagementActor(
services.BuildServiceProvider(), NullLogger<ManagementActor>.Instance)));
}
private static ManagementEnvelope Envelope(object command, string[] sites, params string[] roles) =>
new(new AuthenticatedUser("u", "U", roles, sites), command, Guid.NewGuid().ToString("N"));
private static AuditArchiveManifest Manifest() => new(
1, Name, new DateTime(2025, 10, 1, 0, 0, 0, DateTimeKind.Utc), "2025-10",
new DateTime(2026, 11, 9, 12, 0, 0, DateTimeKind.Utc), AuditArchiveStatus.Purged,
new DateTime(2026, 11, 9, 12, 0, 1, DateTimeKind.Utc), 10, 9, 1, null, null,
Name + ".parquet", "ab", 1234, Name + ".tombstones.parquet", "cd", 0,
[new AuditArchiveChainState("2025-10", 9, new string('0', 64), null, null, 0)]);
/// <summary>A Viewer lists the archives; status renders as a name.</summary>
[Fact]
public void Viewer_ListsArchives()
{
_archive.ListAsync(Arg.Any<CancellationToken>()).Returns([Manifest()]);
CreateActor().Tell(Envelope(new ListAuditArchivesCommand(), [], "Viewer"));
var response = ExpectMsg<ManagementSuccess>(TimeSpan.FromSeconds(5));
using var json = JsonDocument.Parse(response.JsonData);
Assert.True(json.RootElement.GetProperty("enabled").GetBoolean());
var archive = json.RootElement.GetProperty("archives")[0];
Assert.Equal(Name, archive.GetProperty("name").GetString());
Assert.Equal("Purged", archive.GetProperty("status").GetString());
}
/// <summary>Verification results reach the wire with the anchor and chain status as names.</summary>
[Fact]
public void Viewer_GetsVerificationResult()
{
var chain = new AuditChainVerificationResult("2025-10", AuditChainStatus.Intact, 9, 0, null, 9, 1, null);
_archive.VerifyAsync(Name, Arg.Any<CancellationToken>()).Returns(new AuditArchiveVerificationResult(
Name, "2025-10", AuditArchiveStatus.Purged, true, true, null, 10,
[new AuditArchiveChainVerification(chain, AuditArchiveAnchorStatus.Matches, "ends at the anchor")]));
CreateActor().Tell(Envelope(new VerifyAuditArchiveCommand(Name), [], "Viewer"));
var response = ExpectMsg<ManagementSuccess>(TimeSpan.FromSeconds(5));
using var json = JsonDocument.Parse(response.JsonData);
Assert.True(json.RootElement.GetProperty("verified").GetBoolean());
var c = json.RootElement.GetProperty("chains")[0];
Assert.Equal("Matches", c.GetProperty("anchor").GetString());
Assert.Equal("Intact", c.GetProperty("chain").GetProperty("status").GetString());
}
/// <summary>Designer/Deployer without an OperationalAudit role are refused both commands.</summary>
[Fact]
public void NonAuditRole_IsUnauthorized()
{
var actor = CreateActor();
actor.Tell(Envelope(new ListAuditArchivesCommand(), [], "Designer", "Deployer"));
ExpectMsg<ManagementUnauthorized>(TimeSpan.FromSeconds(5));
actor.Tell(Envelope(new VerifyAuditArchiveCommand(Name), [], "Designer", "Deployer"));
ExpectMsg<ManagementUnauthorized>(TimeSpan.FromSeconds(5));
_archive.DidNotReceiveWithAnyArgs().ListAsync(default);
_archive.DidNotReceiveWithAnyArgs().VerifyAsync(default!, default);
}
/// <summary>A site-scoped Viewer is refused: an archive holds every site's rows.</summary>
[Fact]
public void SiteScopedViewer_IsRefused()
{
CreateActor().Tell(Envelope(new VerifyAuditArchiveCommand(Name), ["3"], "Viewer"));
var response = ExpectMsg<ManagementUnauthorized>(TimeSpan.FromSeconds(5));
Assert.Contains("every site", response.Message);
_archive.DidNotReceiveWithAnyArgs().VerifyAsync(default!, default);
}
/// <summary>No archive directory configured is an explicit error code, not an empty list.</summary>
[Fact]
public void UnconfiguredArchive_IsArchiveNotConfigured()
{
_archive.IsConfigured.Returns(false);
CreateActor().Tell(Envelope(new ListAuditArchivesCommand(), [], "Administrator"));
var response = ExpectMsg<ManagementError>(TimeSpan.FromSeconds(5));
Assert.Equal("ARCHIVE_NOT_CONFIGURED", response.ErrorCode);
}
/// <summary>A malformed name is INVALID_ARGUMENT; an unknown one is NOT_FOUND.</summary>
[Fact]
public void BadNames_MapToCallerErrors()
{
_archive.VerifyAsync("../etc", Arg.Any<CancellationToken>())
.Returns<Task<AuditArchiveVerificationResult>>(_ => throw new ArgumentException("not an archive name"));
_archive.VerifyAsync(Name, Arg.Any<CancellationToken>())
.Returns<Task<AuditArchiveVerificationResult>>(_ => throw new FileNotFoundException("no archive"));
var actor = CreateActor();
actor.Tell(Envelope(new VerifyAuditArchiveCommand("../etc"), [], "Administrator"));
Assert.Equal("INVALID_ARGUMENT", ExpectMsg<ManagementError>(TimeSpan.FromSeconds(5)).ErrorCode);
actor.Tell(Envelope(new VerifyAuditArchiveCommand(Name), [], "Administrator"));
Assert.Equal("NOT_FOUND", ExpectMsg<ManagementError>(TimeSpan.FromSeconds(5)).ErrorCode);
}
/// <summary>Archive verification runs under the long-running Ask timeout.</summary>
[Fact]
public void VerifyAuditArchive_UsesLongRunningAskTimeout()
{
Assert.Equal(TimeSpan.FromMinutes(5),
ManagementEndpoints.ResolveAskTimeout(new ManagementServiceOptions(), typeof(VerifyAuditArchiveCommand)));
}
}
@@ -506,21 +506,43 @@ public class AuditEndpointsTests
Assert.Equal(propertyNames, AuditEndpoints.CsvHeader.Split(',')); Assert.Equal(propertyNames, AuditEndpoints.CsvHeader.Split(','));
} }
/// <summary>Verifies that a Parquet export returns 501.</summary> /// <summary>
/// <c>format=parquet</c> (ScadaBridge#42; it used to be a 501) streams one readable Parquet
/// file whose columns are the csv/jsonl export's columns, in order, holding every row.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns> /// <returns>A task that represents the asynchronous operation.</returns>
[Fact] [Fact]
public async Task Export_Parquet_Returns501() public async Task Export_Parquet_StreamsReadableParquet_WithTheCsvColumns()
{ {
// Parquet archival is deferred to v1.x (Component-AuditLog.md) — no var (client, _, host) = await BuildHostAsync(
// library is referenced, so the endpoint returns 501 with guidance. roles: new[] { "Administrator" },
var (client, _, host) = await BuildHostAsync(roles: new[] { "Administrator" }); queryPages: new[]
{
(IReadOnlyList<AuditEvent>)new[]
{
SampleEvent(Guid.Parse("cccccccc-0000-0000-0000-000000000001")),
SampleEvent(Guid.Parse("cccccccc-0000-0000-0000-000000000002")),
},
(IReadOnlyList<AuditEvent>)Array.Empty<AuditEvent>(),
});
using (host) using (host)
{ {
var response = await client.SendAsync(Get("/api/audit/export?format=parquet")); var response = await client.SendAsync(Get("/api/audit/export?format=Parquet"));
Assert.Equal(HttpStatusCode.NotImplemented, response.StatusCode); Assert.Equal(HttpStatusCode.OK, response.StatusCode);
var body = await response.Content.ReadAsStringAsync(); Assert.Equal(AuditEndpoints.ParquetContentType, response.Content.Headers.ContentType!.MediaType);
Assert.Contains("Parquet export deferred to v1.x", body); Assert.Contains(".parquet", response.Content.Headers.ContentDisposition?.ToString()
?? response.Headers.GetValues("Content-Disposition").FirstOrDefault()
?? response.Content.Headers.GetValues("Content-Disposition").First());
var bytes = await response.Content.ReadAsByteArrayAsync();
await using var reader = await Parquet.ParquetReader.CreateAsync(new MemoryStream(bytes));
Assert.Equal(AuditEndpoints.CsvHeader.Split(','), reader.Schema.GetDataFields().Select(f => f.Name));
using var group = reader.OpenRowGroupReader(0);
Assert.Equal(2, group.RowCount);
var ids = new string?[2];
await group.ReadAsync(reader.Schema.GetDataFields()[0], ids!);
Assert.Equal(new[] { "cccccccc-0000-0000-0000-000000000001", "cccccccc-0000-0000-0000-000000000002" }, ids);
} }
} }
@@ -0,0 +1,150 @@
using Parquet;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.ManagementService.Tests;
/// <summary>
/// <see cref="AuditExportParquet"/> (ScadaBridge#42): the Parquet export round-trips every
/// one of the 24 export columns, splits into row groups, and never writes synchronously to
/// the output (Kestrel rejects synchronous response writes).
/// </summary>
public class AuditExportParquetTests
{
/// <summary>Rows written and read back are equal, column for column, across row groups.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task RoundTrip_EveryColumn_AcrossRowGroups()
{
var rows = Enumerable.Range(0, AuditExportParquet.RowGroupRows + 7).Select(Row).ToList();
var output = new AsyncOnlyStream();
var written = await AuditExportParquet.WriteAsync(Pages(rows, 1000), output);
Assert.Equal(rows.Count, written);
await using var reader = await ParquetReader.CreateAsync(new MemoryStream(output.ToArray()));
Assert.Equal(2, reader.RowGroupCount);
var back = new List<AuditExportRow>();
var fields = reader.Schema.GetDataFields();
for (var g = 0; g < reader.RowGroupCount; g++)
{
using var group = reader.OpenRowGroupReader(g);
var n = (int)group.RowCount;
async Task<string?[]> S(int i) { var a = new string?[n]; await group.ReadAsync(fields[i], a!); return a; }
async Task<T?[]> V<T>(int i) where T : struct { var a = new T?[n]; await group.ReadAsync<T>(fields[i], a); return a; }
var id = await S(0); var occurred = await V<DateTime>(1); var ingested = await V<DateTime>(2);
var channel = await S(3); var kind = await S(4); var corr = await S(5); var exec = await S(6); var parent = await S(7);
var site = await S(8); var node = await S(9); var inst = await S(10); var script = await S(11); var actor = await S(12);
var target = await S(13); var status = await S(14); var http = await V<int>(15); var dur = await V<int>(16);
var errMsg = await S(17); var errDetail = await S(18); var req = await S(19); var resp = await S(20);
var trunc = await V<bool>(21); var extra = await S(22); var fwd = await S(23);
for (var i = 0; i < n; i++)
{
back.Add(new AuditExportRow
{
EventId = Guid.Parse(id[i]!),
OccurredAtUtc = occurred[i]!.Value,
IngestedAtUtc = ingested[i],
Channel = Enum.Parse<AuditChannel>(channel[i]!),
Kind = Enum.Parse<AuditKind>(kind[i]!),
CorrelationId = corr[i] is { } c ? Guid.Parse(c) : null,
ExecutionId = exec[i] is { } e ? Guid.Parse(e) : null,
ParentExecutionId = parent[i] is { } p ? Guid.Parse(p) : null,
SourceSiteId = site[i],
SourceNode = node[i],
SourceInstanceId = inst[i],
SourceScript = script[i],
Actor = actor[i],
Target = target[i],
Status = Enum.Parse<AuditStatus>(status[i]!),
HttpStatus = http[i],
DurationMs = dur[i],
ErrorMessage = errMsg[i],
ErrorDetail = errDetail[i],
RequestSummary = req[i],
ResponseSummary = resp[i],
PayloadTruncated = trunc[i]!.Value,
Extra = extra[i],
ForwardState = fwd[i] is { } f ? Enum.Parse<AuditForwardState>(f) : null,
});
}
}
Assert.Equal(rows, back);
}
/// <summary>An export with no rows is still a valid (empty) Parquet file.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
public async Task NoRows_IsAValidEmptyFile()
{
var output = new AsyncOnlyStream();
Assert.Equal(0, await AuditExportParquet.WriteAsync(Pages([], 10), output));
await using var reader = await ParquetReader.CreateAsync(new MemoryStream(output.ToArray()));
Assert.Equal(0, reader.RowGroupCount);
Assert.Equal(24, reader.Schema.GetDataFields().Length);
}
private static AuditExportRow Row(int i) => new()
{
EventId = Guid.NewGuid(),
// Microsecond precision: the export timestamp column is TIMESTAMP(MICROS).
OccurredAtUtc = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc).AddTicks(i * 10L),
IngestedAtUtc = i % 3 == 0 ? null : new DateTime(2026, 5, 20, 12, 0, 1, DateTimeKind.Utc),
Channel = (AuditChannel)(i % Enum.GetValues<AuditChannel>().Length),
Kind = AuditKind.ApiCall,
CorrelationId = i % 2 == 0 ? Guid.NewGuid() : null,
ExecutionId = Guid.NewGuid(),
ParentExecutionId = i % 5 == 0 ? Guid.NewGuid() : null,
SourceSiteId = i % 4 == 0 ? null : "plant-a",
SourceNode = "node-a",
SourceInstanceId = "Plant1.Boiler",
SourceScript = i % 2 == 0 ? "OnTick" : null,
Actor = i % 7 == 0 ? string.Empty : "api-key-1",
Target = "erp",
Status = AuditStatus.Delivered,
HttpStatus = i % 2 == 0 ? 200 : null,
DurationMs = i,
ErrorMessage = i % 9 == 0 ? "boom, \"quoted\"\nline" : null,
ErrorDetail = null,
RequestSummary = "{\"ü\":\"日本\"}",
ResponseSummary = null,
PayloadTruncated = i % 2 == 1,
Extra = null,
ForwardState = i % 11 == 0 ? AuditForwardState.Forwarded : null,
};
private static async IAsyncEnumerable<IReadOnlyList<AuditExportRow>> Pages(List<AuditExportRow> rows, int size)
{
for (var i = 0; i < rows.Count; i += size)
{
await Task.Yield();
yield return rows.Skip(i).Take(size).ToList();
}
}
/// <summary>A response-body stand-in that fails any synchronous write, like Kestrel.</summary>
private sealed class AsyncOnlyStream : Stream
{
private readonly MemoryStream _inner = new();
public byte[] ToArray() => _inner.ToArray();
public override bool CanRead => false;
public override bool CanSeek => false;
public override bool CanWrite => true;
public override long Length => throw new NotSupportedException();
public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); }
public override void Flush() => throw new InvalidOperationException("Synchronous operations are disallowed.");
public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask;
public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
public override void SetLength(long value) => throw new NotSupportedException();
public override void Write(byte[] buffer, int offset, int count) => throw new InvalidOperationException("Synchronous operations are disallowed.");
public override ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default)
{
_inner.Write(buffer.Span);
return ValueTask.CompletedTask;
}
}
}
@@ -171,6 +171,9 @@ public class RequiredRoleMatrixTests
// AuditLog hash-chain verification (ScadaBridge#41) carries audit query's // AuditLog hash-chain verification (ScadaBridge#41) carries audit query's
// OperationalAudit permission (alog.md §15.1); written out by hand to keep the freeze. // OperationalAudit permission (alog.md §15.1); written out by hand to keep the freeze.
["VerifyAuditChain"] = [Roles.Administrator, Roles.Viewer], ["VerifyAuditChain"] = [Roles.Administrator, Roles.Viewer],
// AuditLog Parquet archives (ScadaBridge#42): same audit-read gate as verify-chain.
["ListAuditArchives"] = [Roles.Administrator, Roles.Viewer],
["VerifyAuditArchive"] = [Roles.Administrator, Roles.Viewer],
// ---- Two-person secured write ---------------------------------------------- // ---- Two-person secured write ----------------------------------------------
["SubmitSecuredWrite"] = [Roles.Operator], ["SubmitSecuredWrite"] = [Roles.Operator],