feat(management): parquet audit export and audit archive list/verify (ScadaBridge#42)
GET /api/audit/export?format=parquet returned 501. It now streams the same 24 columns as csv/jsonl (AuditExportRow, CsvHeader order) as one Parquet file, one 10,000-row row group at a time. Each row group is encoded into a small non-seekable staging buffer and copied to the response asynchronously, because Kestrel rejects the synchronous writes the Parquet writer issues. New management commands, with CLI parity: - ListAuditArchives -> `audit archive list` (AuditArchiveListResult: enabled flag + every manifest). - VerifyAuditArchive(Name) -> `audit archive verify --name` (files, chain, purge anchor); exit 0 verified, 3 not verified, 1 error, 2 unauthorized. Both carry verify-chain's gate: OperationalAudit roles (Administrator, Viewer), site-scoped non-Administrator refused because an archive spans every site. VerifyAuditArchive uses the long-running Ask timeout. Error codes ARCHIVE_NOT_CONFIGURED, INVALID_ARGUMENT, NOT_FOUND. RequiredRoleMatrixTests freezes both gates; ManagementSurfaceParityTests now probes `audit archive list|verify` alongside `audit verify-chain`. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
@@ -69,7 +69,7 @@ public class CommandTreeTests
|
||||
Assert.Contains("bundle", groupNames);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the audit command group exposes the query, export and verify-chain subcommands.</summary>
|
||||
/// <summary>Verifies that the audit command group exposes the query, export, verify-chain and archive subcommands.</summary>
|
||||
[Fact]
|
||||
public void AuditCommandGroup_HasQueryExportAndVerifyChain()
|
||||
{
|
||||
@@ -80,6 +80,7 @@ public class CommandTreeTests
|
||||
Assert.Contains("query", subNames);
|
||||
Assert.Contains("export", subNames);
|
||||
Assert.Contains("verify-chain", subNames);
|
||||
Assert.Contains("archive", subNames);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the bundle command group exposes the export, preview and import subcommands.</summary>
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using ZB.MOM.WW.ScadaBridge.CLI.Commands;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.CLI.Tests.Commands;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <c>scadabridge audit archive list</c> / <c>verify</c> (ScadaBridge#42): each
|
||||
/// sends its management command (<c>ListAuditArchives</c>, <c>VerifyAuditArchive</c>), and
|
||||
/// <c>verify</c> maps the verdict onto the exit code (0 verified, 3 not verified).
|
||||
/// </summary>
|
||||
[Collection("Console")]
|
||||
public class AuditArchiveCommandTests
|
||||
{
|
||||
private const string Name = "auditlog-2025-10-20261109T120000123Z";
|
||||
|
||||
private const string ListBody =
|
||||
"{\"enabled\":true,\"archives\":[{\"name\":\"" + Name + "\",\"partition\":\"2025-10\",\"status\":\"Purged\"," +
|
||||
"\"rowCount\":11,\"chainedRowCount\":10,\"dataBytes\":4096}]}";
|
||||
|
||||
private const string VerifiedBody =
|
||||
"{\"name\":\"" + Name + "\",\"partition\":\"2025-10\",\"manifestStatus\":\"Purged\",\"verified\":true,\"filesIntact\":true," +
|
||||
"\"rowsRead\":11,\"chains\":[{\"chain\":{\"partition\":\"2025-10\",\"status\":\"Intact\",\"rowsVerified\":10," +
|
||||
"\"tombstonesApplied\":2,\"unchainedRows\":1},\"anchor\":\"Matches\",\"anchorDetail\":\"ends at the anchor\"}]}";
|
||||
|
||||
private const string MismatchBody =
|
||||
"{\"name\":\"" + Name + "\",\"partition\":\"2025-10\",\"manifestStatus\":\"Purged\",\"verified\":false,\"filesIntact\":true," +
|
||||
"\"rowsRead\":11,\"chains\":[{\"chain\":{\"partition\":\"2025-10\",\"status\":\"Intact\",\"rowsVerified\":10," +
|
||||
"\"tombstonesApplied\":0,\"unchainedRows\":0},\"anchor\":\"Mismatch\",\"anchorDetail\":\"hash differs\"}]}";
|
||||
|
||||
/// <summary>The audit group carries the archive group with list + verify.</summary>
|
||||
[Fact]
|
||||
public void AuditGroup_HasArchiveListAndVerify()
|
||||
{
|
||||
var root = AuditCommandTestHarness.BuildRoot();
|
||||
var archive = root.Subcommands.Single(c => c.Name == "audit").Subcommands.Single(c => c.Name == "archive");
|
||||
Assert.Equal(["list", "verify"], archive.Subcommands.Select(c => c.Name).Order());
|
||||
}
|
||||
|
||||
/// <summary><c>list</c> sends ListAuditArchives and prints the archives.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task List_SendsListAuditArchives_AndPrintsTheArchives()
|
||||
{
|
||||
var (exit, output, request) = await RunAsync(ListBody, "audit", "archive", "list", "--format", "table");
|
||||
|
||||
Assert.Equal(0, exit);
|
||||
Assert.Contains(Name, output);
|
||||
Assert.Contains("Archiving: enabled", output);
|
||||
using var payload = JsonDocument.Parse(request);
|
||||
Assert.Equal("ListAuditArchives", payload.RootElement.GetProperty("command").GetString());
|
||||
}
|
||||
|
||||
/// <summary><c>verify</c> sends VerifyAuditArchive with the name and maps the verdict to the exit code.</summary>
|
||||
/// <param name="body">The verification result the fake endpoint serves.</param>
|
||||
/// <param name="format">Output format.</param>
|
||||
/// <param name="expectedExit">Expected exit code.</param>
|
||||
/// <param name="expectedText">Text the report must contain.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Theory]
|
||||
[InlineData(VerifiedBody, "table", 0, "anchor Matches")]
|
||||
[InlineData(VerifiedBody, "json", 0, "\"verified\":true")]
|
||||
[InlineData(MismatchBody, "table", 3, "Verified : NO")]
|
||||
[InlineData(MismatchBody, "json", 3, "\"anchor\":\"Mismatch\"")]
|
||||
public async Task Verify_SendsVerifyAuditArchive_AndMapsVerdictToExitCode(
|
||||
string body, string format, int expectedExit, string expectedText)
|
||||
{
|
||||
var (exit, output, request) = await RunAsync(body, "audit", "archive", "verify", "--name", Name, "--format", format);
|
||||
|
||||
Assert.Equal(expectedExit, exit);
|
||||
Assert.Contains(expectedText, output);
|
||||
using var payload = JsonDocument.Parse(request);
|
||||
Assert.Equal("VerifyAuditArchive", payload.RootElement.GetProperty("command").GetString());
|
||||
var name = payload.RootElement.GetProperty("payload").EnumerateObject()
|
||||
.First(p => string.Equals(p.Name, "name", StringComparison.OrdinalIgnoreCase)).Value.GetString();
|
||||
Assert.Equal(Name, name);
|
||||
}
|
||||
|
||||
/// <summary><c>verify</c> without <c>--name</c> is a usage error.</summary>
|
||||
[Fact]
|
||||
public void Verify_MissingName_ProducesRequiredFlagError()
|
||||
{
|
||||
var root = AuditCommandTestHarness.BuildRoot();
|
||||
var (exit, _, err) = AuditCommandTestHarness.Invoke(root, "audit", "archive", "verify");
|
||||
|
||||
Assert.NotEqual(0, exit);
|
||||
Assert.Contains("--name", err);
|
||||
}
|
||||
|
||||
private static async Task<(int Exit, string Output, string Request)> RunAsync(string body, params string[] args)
|
||||
{
|
||||
var listener = new TcpListener(IPAddress.Loopback, 0);
|
||||
listener.Start();
|
||||
try
|
||||
{
|
||||
var port = ((IPEndPoint)listener.LocalEndpoint).Port;
|
||||
string? requestBody = null;
|
||||
var server = Task.Run(async () =>
|
||||
{
|
||||
using var client = await listener.AcceptTcpClientAsync();
|
||||
await using var stream = client.GetStream();
|
||||
using var reader = new StreamReader(stream, Encoding.UTF8, leaveOpen: true);
|
||||
var length = 0;
|
||||
while (await reader.ReadLineAsync() is { Length: > 0 } line)
|
||||
if (line.StartsWith("Content-Length:", StringComparison.OrdinalIgnoreCase))
|
||||
length = int.Parse(line[15..].Trim());
|
||||
var buffer = new char[length];
|
||||
var read = 0;
|
||||
while (read < length)
|
||||
{
|
||||
var count = await reader.ReadAsync(buffer.AsMemory(read));
|
||||
if (count == 0) throw new EndOfStreamException();
|
||||
read += count;
|
||||
}
|
||||
requestBody = new string(buffer);
|
||||
var bytes = Encoding.UTF8.GetBytes(body);
|
||||
var headers = Encoding.ASCII.GetBytes(
|
||||
$"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {bytes.Length}\r\nConnection: close\r\n\r\n");
|
||||
await stream.WriteAsync(headers);
|
||||
await stream.WriteAsync(bytes);
|
||||
});
|
||||
|
||||
var root = AuditCommandTestHarness.BuildRoot();
|
||||
var (exit, output, _) = AuditCommandTestHarness.Invoke(
|
||||
root, [.. args, "--url", $"http://127.0.0.1:{port}", "--username", "u", "--password", "p"]);
|
||||
await server.WaitAsync(TimeSpan.FromSeconds(5));
|
||||
return (exit, output, requestBody!);
|
||||
}
|
||||
finally
|
||||
{
|
||||
listener.Stop();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -27,14 +27,14 @@ public class AuditCommandsScaffoldTests
|
||||
Assert.False(string.IsNullOrWhiteSpace(audit.Description));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the audit group exposes the query, export, tree, verify-chain and backfill-source-node subcommands.</summary>
|
||||
/// <summary>Verifies that the audit group exposes the query, export, tree, verify-chain, backfill-source-node and archive subcommands.</summary>
|
||||
[Fact]
|
||||
public void Audit_HasFiveSubcommands_QueryExportTreeVerifyChainBackfillSourceNode()
|
||||
public void Audit_HasSixSubcommands_QueryExportTreeVerifyChainBackfillSourceNodeArchive()
|
||||
{
|
||||
var audit = BuildAudit();
|
||||
var names = audit.Subcommands.Select(c => c.Name).OrderBy(n => n).ToArray();
|
||||
Assert.Equal(
|
||||
new[] { "backfill-source-node", "export", "query", "tree", "verify-chain" },
|
||||
new[] { "archive", "backfill-source-node", "export", "query", "tree", "verify-chain" },
|
||||
names);
|
||||
}
|
||||
|
||||
@@ -62,8 +62,11 @@ public class AuditCommandsScaffoldTests
|
||||
[Fact]
|
||||
public void Audit_EveryLeafCommand_HasAnAction()
|
||||
{
|
||||
// `archive` is a group (list / verify, ScadaBridge#42): check its leaves, not it.
|
||||
var audit = BuildAudit();
|
||||
Assert.All(audit.Subcommands, sub =>
|
||||
var leaves = audit.Subcommands.SelectMany(c => c.Subcommands.Count == 0 ? [c] : c.Subcommands).ToList();
|
||||
Assert.Equal(7, leaves.Count);
|
||||
Assert.All(leaves, sub =>
|
||||
Assert.True(sub.Action != null, $"Leaf command '{sub.Name}' has no action."));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,6 +44,9 @@ public class ManagementSurfaceParityTests
|
||||
Assert.Contains("ListSharedScripts", sent.Keys);
|
||||
}
|
||||
|
||||
private static readonly HashSet<string> AuditManagementLeaves =
|
||||
["audit verify-chain", "audit archive list", "audit archive verify"];
|
||||
|
||||
private static async Task<ConcurrentDictionary<string, string>> ProbeProductionTree(string? omitGroup = null)
|
||||
{
|
||||
var root = CliApplication.BuildRoot();
|
||||
@@ -93,9 +96,10 @@ public class ManagementSurfaceParityTests
|
||||
foreach (var (leaf, path) in Leaves(root, []))
|
||||
{
|
||||
// Separate REST and persistent SignalR surfaces, not management contracts.
|
||||
// `audit verify-chain` is the one audit leaf that IS a management contract
|
||||
// (VerifyAuditChain, ScadaBridge#41), so it stays in the probe.
|
||||
if (path.Length == 0 || (path[0] == "audit" && string.Join(' ', path) != "audit verify-chain")
|
||||
// `audit verify-chain` (VerifyAuditChain, ScadaBridge#41) and `audit archive
|
||||
// list|verify` (ListAuditArchives / VerifyAuditArchive, ScadaBridge#42) are the
|
||||
// audit leaves that ARE management contracts, so they stay in the probe.
|
||||
if (path.Length == 0 || (path[0] == "audit" && !AuditManagementLeaves.Contains(string.Join(' ', path)))
|
||||
|| path[0] == omitGroup
|
||||
|| string.Join(' ', path) == "debug stream") continue;
|
||||
currentPath = string.Join(' ', path);
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
using System.Text.Json;
|
||||
using Akka.Actor;
|
||||
using Akka.TestKit.Xunit2;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using NSubstitute;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Management;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ManagementService.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// The <c>ListAuditArchives</c> / <c>VerifyAuditArchive</c> management commands
|
||||
/// (ScadaBridge#42): role gate, site-scope refusal, an unconfigured archive directory, bad
|
||||
/// names, and the results reaching the wire with enums as names.
|
||||
/// </summary>
|
||||
public class AuditArchiveCommandTests : TestKit
|
||||
{
|
||||
private const string Name = "auditlog-2025-10-20261109T120000123Z";
|
||||
|
||||
private readonly IAuditLogArchive _archive = Substitute.For<IAuditLogArchive>();
|
||||
|
||||
/// <summary>Configures the archive substitute as a configured, enabled archive.</summary>
|
||||
public AuditArchiveCommandTests()
|
||||
{
|
||||
_archive.IsConfigured.Returns(true);
|
||||
_archive.IsEnabled.Returns(true);
|
||||
}
|
||||
|
||||
private IActorRef CreateActor(bool registerArchive = true)
|
||||
{
|
||||
var services = new ServiceCollection().AddTestConfigurationTransaction();
|
||||
if (registerArchive)
|
||||
{
|
||||
services.AddScoped(_ => _archive);
|
||||
}
|
||||
|
||||
return Sys.ActorOf(Props.Create(() => new ManagementActor(
|
||||
services.BuildServiceProvider(), NullLogger<ManagementActor>.Instance)));
|
||||
}
|
||||
|
||||
private static ManagementEnvelope Envelope(object command, string[] sites, params string[] roles) =>
|
||||
new(new AuthenticatedUser("u", "U", roles, sites), command, Guid.NewGuid().ToString("N"));
|
||||
|
||||
private static AuditArchiveManifest Manifest() => new(
|
||||
1, Name, new DateTime(2025, 10, 1, 0, 0, 0, DateTimeKind.Utc), "2025-10",
|
||||
new DateTime(2026, 11, 9, 12, 0, 0, DateTimeKind.Utc), AuditArchiveStatus.Purged,
|
||||
new DateTime(2026, 11, 9, 12, 0, 1, DateTimeKind.Utc), 10, 9, 1, null, null,
|
||||
Name + ".parquet", "ab", 1234, Name + ".tombstones.parquet", "cd", 0,
|
||||
[new AuditArchiveChainState("2025-10", 9, new string('0', 64), null, null, 0)]);
|
||||
|
||||
/// <summary>A Viewer lists the archives; status renders as a name.</summary>
|
||||
[Fact]
|
||||
public void Viewer_ListsArchives()
|
||||
{
|
||||
_archive.ListAsync(Arg.Any<CancellationToken>()).Returns([Manifest()]);
|
||||
|
||||
CreateActor().Tell(Envelope(new ListAuditArchivesCommand(), [], "Viewer"));
|
||||
|
||||
var response = ExpectMsg<ManagementSuccess>(TimeSpan.FromSeconds(5));
|
||||
using var json = JsonDocument.Parse(response.JsonData);
|
||||
Assert.True(json.RootElement.GetProperty("enabled").GetBoolean());
|
||||
var archive = json.RootElement.GetProperty("archives")[0];
|
||||
Assert.Equal(Name, archive.GetProperty("name").GetString());
|
||||
Assert.Equal("Purged", archive.GetProperty("status").GetString());
|
||||
}
|
||||
|
||||
/// <summary>Verification results reach the wire with the anchor and chain status as names.</summary>
|
||||
[Fact]
|
||||
public void Viewer_GetsVerificationResult()
|
||||
{
|
||||
var chain = new AuditChainVerificationResult("2025-10", AuditChainStatus.Intact, 9, 0, null, 9, 1, null);
|
||||
_archive.VerifyAsync(Name, Arg.Any<CancellationToken>()).Returns(new AuditArchiveVerificationResult(
|
||||
Name, "2025-10", AuditArchiveStatus.Purged, true, true, null, 10,
|
||||
[new AuditArchiveChainVerification(chain, AuditArchiveAnchorStatus.Matches, "ends at the anchor")]));
|
||||
|
||||
CreateActor().Tell(Envelope(new VerifyAuditArchiveCommand(Name), [], "Viewer"));
|
||||
|
||||
var response = ExpectMsg<ManagementSuccess>(TimeSpan.FromSeconds(5));
|
||||
using var json = JsonDocument.Parse(response.JsonData);
|
||||
Assert.True(json.RootElement.GetProperty("verified").GetBoolean());
|
||||
var c = json.RootElement.GetProperty("chains")[0];
|
||||
Assert.Equal("Matches", c.GetProperty("anchor").GetString());
|
||||
Assert.Equal("Intact", c.GetProperty("chain").GetProperty("status").GetString());
|
||||
}
|
||||
|
||||
/// <summary>Designer/Deployer without an OperationalAudit role are refused both commands.</summary>
|
||||
[Fact]
|
||||
public void NonAuditRole_IsUnauthorized()
|
||||
{
|
||||
var actor = CreateActor();
|
||||
actor.Tell(Envelope(new ListAuditArchivesCommand(), [], "Designer", "Deployer"));
|
||||
ExpectMsg<ManagementUnauthorized>(TimeSpan.FromSeconds(5));
|
||||
actor.Tell(Envelope(new VerifyAuditArchiveCommand(Name), [], "Designer", "Deployer"));
|
||||
ExpectMsg<ManagementUnauthorized>(TimeSpan.FromSeconds(5));
|
||||
_archive.DidNotReceiveWithAnyArgs().ListAsync(default);
|
||||
_archive.DidNotReceiveWithAnyArgs().VerifyAsync(default!, default);
|
||||
}
|
||||
|
||||
/// <summary>A site-scoped Viewer is refused: an archive holds every site's rows.</summary>
|
||||
[Fact]
|
||||
public void SiteScopedViewer_IsRefused()
|
||||
{
|
||||
CreateActor().Tell(Envelope(new VerifyAuditArchiveCommand(Name), ["3"], "Viewer"));
|
||||
|
||||
var response = ExpectMsg<ManagementUnauthorized>(TimeSpan.FromSeconds(5));
|
||||
Assert.Contains("every site", response.Message);
|
||||
_archive.DidNotReceiveWithAnyArgs().VerifyAsync(default!, default);
|
||||
}
|
||||
|
||||
/// <summary>No archive directory configured is an explicit error code, not an empty list.</summary>
|
||||
[Fact]
|
||||
public void UnconfiguredArchive_IsArchiveNotConfigured()
|
||||
{
|
||||
_archive.IsConfigured.Returns(false);
|
||||
|
||||
CreateActor().Tell(Envelope(new ListAuditArchivesCommand(), [], "Administrator"));
|
||||
|
||||
var response = ExpectMsg<ManagementError>(TimeSpan.FromSeconds(5));
|
||||
Assert.Equal("ARCHIVE_NOT_CONFIGURED", response.ErrorCode);
|
||||
}
|
||||
|
||||
/// <summary>A malformed name is INVALID_ARGUMENT; an unknown one is NOT_FOUND.</summary>
|
||||
[Fact]
|
||||
public void BadNames_MapToCallerErrors()
|
||||
{
|
||||
_archive.VerifyAsync("../etc", Arg.Any<CancellationToken>())
|
||||
.Returns<Task<AuditArchiveVerificationResult>>(_ => throw new ArgumentException("not an archive name"));
|
||||
_archive.VerifyAsync(Name, Arg.Any<CancellationToken>())
|
||||
.Returns<Task<AuditArchiveVerificationResult>>(_ => throw new FileNotFoundException("no archive"));
|
||||
|
||||
var actor = CreateActor();
|
||||
actor.Tell(Envelope(new VerifyAuditArchiveCommand("../etc"), [], "Administrator"));
|
||||
Assert.Equal("INVALID_ARGUMENT", ExpectMsg<ManagementError>(TimeSpan.FromSeconds(5)).ErrorCode);
|
||||
actor.Tell(Envelope(new VerifyAuditArchiveCommand(Name), [], "Administrator"));
|
||||
Assert.Equal("NOT_FOUND", ExpectMsg<ManagementError>(TimeSpan.FromSeconds(5)).ErrorCode);
|
||||
}
|
||||
|
||||
/// <summary>Archive verification runs under the long-running Ask timeout.</summary>
|
||||
[Fact]
|
||||
public void VerifyAuditArchive_UsesLongRunningAskTimeout()
|
||||
{
|
||||
Assert.Equal(TimeSpan.FromMinutes(5),
|
||||
ManagementEndpoints.ResolveAskTimeout(new ManagementServiceOptions(), typeof(VerifyAuditArchiveCommand)));
|
||||
}
|
||||
}
|
||||
@@ -506,21 +506,43 @@ public class AuditEndpointsTests
|
||||
Assert.Equal(propertyNames, AuditEndpoints.CsvHeader.Split(','));
|
||||
}
|
||||
|
||||
/// <summary>Verifies that a Parquet export returns 501.</summary>
|
||||
/// <summary>
|
||||
/// <c>format=parquet</c> (ScadaBridge#42; it used to be a 501) streams one readable Parquet
|
||||
/// file whose columns are the csv/jsonl export's columns, in order, holding every row.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Export_Parquet_Returns501()
|
||||
public async Task Export_Parquet_StreamsReadableParquet_WithTheCsvColumns()
|
||||
{
|
||||
// Parquet archival is deferred to v1.x (Component-AuditLog.md) — no
|
||||
// library is referenced, so the endpoint returns 501 with guidance.
|
||||
var (client, _, host) = await BuildHostAsync(roles: new[] { "Administrator" });
|
||||
var (client, _, host) = await BuildHostAsync(
|
||||
roles: new[] { "Administrator" },
|
||||
queryPages: new[]
|
||||
{
|
||||
(IReadOnlyList<AuditEvent>)new[]
|
||||
{
|
||||
SampleEvent(Guid.Parse("cccccccc-0000-0000-0000-000000000001")),
|
||||
SampleEvent(Guid.Parse("cccccccc-0000-0000-0000-000000000002")),
|
||||
},
|
||||
(IReadOnlyList<AuditEvent>)Array.Empty<AuditEvent>(),
|
||||
});
|
||||
using (host)
|
||||
{
|
||||
var response = await client.SendAsync(Get("/api/audit/export?format=parquet"));
|
||||
var response = await client.SendAsync(Get("/api/audit/export?format=Parquet"));
|
||||
|
||||
Assert.Equal(HttpStatusCode.NotImplemented, response.StatusCode);
|
||||
var body = await response.Content.ReadAsStringAsync();
|
||||
Assert.Contains("Parquet export deferred to v1.x", body);
|
||||
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
|
||||
Assert.Equal(AuditEndpoints.ParquetContentType, response.Content.Headers.ContentType!.MediaType);
|
||||
Assert.Contains(".parquet", response.Content.Headers.ContentDisposition?.ToString()
|
||||
?? response.Headers.GetValues("Content-Disposition").FirstOrDefault()
|
||||
?? response.Content.Headers.GetValues("Content-Disposition").First());
|
||||
|
||||
var bytes = await response.Content.ReadAsByteArrayAsync();
|
||||
await using var reader = await Parquet.ParquetReader.CreateAsync(new MemoryStream(bytes));
|
||||
Assert.Equal(AuditEndpoints.CsvHeader.Split(','), reader.Schema.GetDataFields().Select(f => f.Name));
|
||||
using var group = reader.OpenRowGroupReader(0);
|
||||
Assert.Equal(2, group.RowCount);
|
||||
var ids = new string?[2];
|
||||
await group.ReadAsync(reader.Schema.GetDataFields()[0], ids!);
|
||||
Assert.Equal(new[] { "cccccccc-0000-0000-0000-000000000001", "cccccccc-0000-0000-0000-000000000002" }, ids);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
using Parquet;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ManagementService.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="AuditExportParquet"/> (ScadaBridge#42): the Parquet export round-trips every
|
||||
/// one of the 24 export columns, splits into row groups, and never writes synchronously to
|
||||
/// the output (Kestrel rejects synchronous response writes).
|
||||
/// </summary>
|
||||
public class AuditExportParquetTests
|
||||
{
|
||||
/// <summary>Rows written and read back are equal, column for column, across row groups.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RoundTrip_EveryColumn_AcrossRowGroups()
|
||||
{
|
||||
var rows = Enumerable.Range(0, AuditExportParquet.RowGroupRows + 7).Select(Row).ToList();
|
||||
var output = new AsyncOnlyStream();
|
||||
|
||||
var written = await AuditExportParquet.WriteAsync(Pages(rows, 1000), output);
|
||||
|
||||
Assert.Equal(rows.Count, written);
|
||||
await using var reader = await ParquetReader.CreateAsync(new MemoryStream(output.ToArray()));
|
||||
Assert.Equal(2, reader.RowGroupCount);
|
||||
var back = new List<AuditExportRow>();
|
||||
var fields = reader.Schema.GetDataFields();
|
||||
for (var g = 0; g < reader.RowGroupCount; g++)
|
||||
{
|
||||
using var group = reader.OpenRowGroupReader(g);
|
||||
var n = (int)group.RowCount;
|
||||
async Task<string?[]> S(int i) { var a = new string?[n]; await group.ReadAsync(fields[i], a!); return a; }
|
||||
async Task<T?[]> V<T>(int i) where T : struct { var a = new T?[n]; await group.ReadAsync<T>(fields[i], a); return a; }
|
||||
var id = await S(0); var occurred = await V<DateTime>(1); var ingested = await V<DateTime>(2);
|
||||
var channel = await S(3); var kind = await S(4); var corr = await S(5); var exec = await S(6); var parent = await S(7);
|
||||
var site = await S(8); var node = await S(9); var inst = await S(10); var script = await S(11); var actor = await S(12);
|
||||
var target = await S(13); var status = await S(14); var http = await V<int>(15); var dur = await V<int>(16);
|
||||
var errMsg = await S(17); var errDetail = await S(18); var req = await S(19); var resp = await S(20);
|
||||
var trunc = await V<bool>(21); var extra = await S(22); var fwd = await S(23);
|
||||
for (var i = 0; i < n; i++)
|
||||
{
|
||||
back.Add(new AuditExportRow
|
||||
{
|
||||
EventId = Guid.Parse(id[i]!),
|
||||
OccurredAtUtc = occurred[i]!.Value,
|
||||
IngestedAtUtc = ingested[i],
|
||||
Channel = Enum.Parse<AuditChannel>(channel[i]!),
|
||||
Kind = Enum.Parse<AuditKind>(kind[i]!),
|
||||
CorrelationId = corr[i] is { } c ? Guid.Parse(c) : null,
|
||||
ExecutionId = exec[i] is { } e ? Guid.Parse(e) : null,
|
||||
ParentExecutionId = parent[i] is { } p ? Guid.Parse(p) : null,
|
||||
SourceSiteId = site[i],
|
||||
SourceNode = node[i],
|
||||
SourceInstanceId = inst[i],
|
||||
SourceScript = script[i],
|
||||
Actor = actor[i],
|
||||
Target = target[i],
|
||||
Status = Enum.Parse<AuditStatus>(status[i]!),
|
||||
HttpStatus = http[i],
|
||||
DurationMs = dur[i],
|
||||
ErrorMessage = errMsg[i],
|
||||
ErrorDetail = errDetail[i],
|
||||
RequestSummary = req[i],
|
||||
ResponseSummary = resp[i],
|
||||
PayloadTruncated = trunc[i]!.Value,
|
||||
Extra = extra[i],
|
||||
ForwardState = fwd[i] is { } f ? Enum.Parse<AuditForwardState>(f) : null,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Assert.Equal(rows, back);
|
||||
}
|
||||
|
||||
/// <summary>An export with no rows is still a valid (empty) Parquet file.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task NoRows_IsAValidEmptyFile()
|
||||
{
|
||||
var output = new AsyncOnlyStream();
|
||||
Assert.Equal(0, await AuditExportParquet.WriteAsync(Pages([], 10), output));
|
||||
await using var reader = await ParquetReader.CreateAsync(new MemoryStream(output.ToArray()));
|
||||
Assert.Equal(0, reader.RowGroupCount);
|
||||
Assert.Equal(24, reader.Schema.GetDataFields().Length);
|
||||
}
|
||||
|
||||
private static AuditExportRow Row(int i) => new()
|
||||
{
|
||||
EventId = Guid.NewGuid(),
|
||||
// Microsecond precision: the export timestamp column is TIMESTAMP(MICROS).
|
||||
OccurredAtUtc = new DateTime(2026, 5, 20, 12, 0, 0, DateTimeKind.Utc).AddTicks(i * 10L),
|
||||
IngestedAtUtc = i % 3 == 0 ? null : new DateTime(2026, 5, 20, 12, 0, 1, DateTimeKind.Utc),
|
||||
Channel = (AuditChannel)(i % Enum.GetValues<AuditChannel>().Length),
|
||||
Kind = AuditKind.ApiCall,
|
||||
CorrelationId = i % 2 == 0 ? Guid.NewGuid() : null,
|
||||
ExecutionId = Guid.NewGuid(),
|
||||
ParentExecutionId = i % 5 == 0 ? Guid.NewGuid() : null,
|
||||
SourceSiteId = i % 4 == 0 ? null : "plant-a",
|
||||
SourceNode = "node-a",
|
||||
SourceInstanceId = "Plant1.Boiler",
|
||||
SourceScript = i % 2 == 0 ? "OnTick" : null,
|
||||
Actor = i % 7 == 0 ? string.Empty : "api-key-1",
|
||||
Target = "erp",
|
||||
Status = AuditStatus.Delivered,
|
||||
HttpStatus = i % 2 == 0 ? 200 : null,
|
||||
DurationMs = i,
|
||||
ErrorMessage = i % 9 == 0 ? "boom, \"quoted\"\nline" : null,
|
||||
ErrorDetail = null,
|
||||
RequestSummary = "{\"ü\":\"日本\"}",
|
||||
ResponseSummary = null,
|
||||
PayloadTruncated = i % 2 == 1,
|
||||
Extra = null,
|
||||
ForwardState = i % 11 == 0 ? AuditForwardState.Forwarded : null,
|
||||
};
|
||||
|
||||
private static async IAsyncEnumerable<IReadOnlyList<AuditExportRow>> Pages(List<AuditExportRow> rows, int size)
|
||||
{
|
||||
for (var i = 0; i < rows.Count; i += size)
|
||||
{
|
||||
await Task.Yield();
|
||||
yield return rows.Skip(i).Take(size).ToList();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>A response-body stand-in that fails any synchronous write, like Kestrel.</summary>
|
||||
private sealed class AsyncOnlyStream : Stream
|
||||
{
|
||||
private readonly MemoryStream _inner = new();
|
||||
|
||||
public byte[] ToArray() => _inner.ToArray();
|
||||
|
||||
public override bool CanRead => false;
|
||||
public override bool CanSeek => false;
|
||||
public override bool CanWrite => true;
|
||||
public override long Length => throw new NotSupportedException();
|
||||
public override long Position { get => throw new NotSupportedException(); set => throw new NotSupportedException(); }
|
||||
public override void Flush() => throw new InvalidOperationException("Synchronous operations are disallowed.");
|
||||
public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException();
|
||||
public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException();
|
||||
public override void SetLength(long value) => throw new NotSupportedException();
|
||||
public override void Write(byte[] buffer, int offset, int count) => throw new InvalidOperationException("Synchronous operations are disallowed.");
|
||||
public override ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationToken cancellationToken = default)
|
||||
{
|
||||
_inner.Write(buffer.Span);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -171,6 +171,9 @@ public class RequiredRoleMatrixTests
|
||||
// AuditLog hash-chain verification (ScadaBridge#41) carries audit query's
|
||||
// OperationalAudit permission (alog.md §15.1); written out by hand to keep the freeze.
|
||||
["VerifyAuditChain"] = [Roles.Administrator, Roles.Viewer],
|
||||
// AuditLog Parquet archives (ScadaBridge#42): same audit-read gate as verify-chain.
|
||||
["ListAuditArchives"] = [Roles.Administrator, Roles.Viewer],
|
||||
["VerifyAuditArchive"] = [Roles.Administrator, Roles.Viewer],
|
||||
|
||||
// ---- Two-person secured write ----------------------------------------------
|
||||
["SubmitSecuredWrite"] = [Roles.Operator],
|
||||
|
||||
Reference in New Issue
Block a user