feat(audit): off-box chain-head export, verify-heads, maintenance login (ScadaBridge#107)

Item 1 (owner decision: no signing). AuditLog:HashChain:HeadExport (default off): both centrals periodically read every month's chain head (LastSeq, LastHash, purge anchor, chained rows, tombstones; one snapshot under the chain lock SHARED) and append the changed heads to a sink the DB login cannot write - File (append-only JSONL per node per day, local or UNC) or Syslog (RFC 5424 over UDP / TCP octet counting). `audit verify-heads --path` reads those records back (file or syslog-collector lines) and the new VerifyAuditChainHeads command checks each against the database: Matches / Extended / Purged, or Mismatch / Regressed / MissingHead / MissingLink / CountMismatch (exit 3). AuditChainHeadStoreTests forges a chain with the production hasher: verify-chain says Intact, verify-heads says Mismatch. Item 3. AuditLog:MaintenanceConnectionString (${secret:}-capable, unset = today's single login) runs the chain job, chain verify / head export, the partition switch-out + archive and the per-channel purge on AuditMaintenanceContext (keyed audit-maintenance repository/archive, purge actor prefers them). AuditMaintenanceLoginTests runs the hardened setup on SQL Server: writer (datareader/datawriter/audit_writer) ingests and is refused chaining and deleting; the chainer+purger login does the full cycle. Two-login grants documented in Component-AuditLog.md. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-09 23:16:05 -05:00
parent 5e7d5389d4
commit c4d7ec9b96
40 changed files with 2769 additions and 43 deletions
@@ -27,14 +27,14 @@ public class AuditCommandsScaffoldTests
        Assert.False(string.IsNullOrWhiteSpace(audit.Description));
    }

-    /// <summary>Verifies that the audit group exposes the query, export, tree, verify-chain, backfill-source-node and archive subcommands.</summary>
+    /// <summary>Verifies that the audit group exposes the query, export, tree, verify-chain, verify-heads, backfill-source-node and archive subcommands.</summary>
    [Fact]
-    public void Audit_HasSixSubcommands_QueryExportTreeVerifyChainBackfillSourceNodeArchive()
+    public void Audit_HasSevenSubcommands_QueryExportTreeVerifyChainVerifyHeadsBackfillSourceNodeArchive()
    {
        var audit = BuildAudit();
        var names = audit.Subcommands.Select(c => c.Name).OrderBy(n => n).ToArray();
        Assert.Equal(
-            new[] { "archive", "backfill-source-node", "export", "query", "tree", "verify-chain" },
+            new[] { "archive", "backfill-source-node", "export", "query", "tree", "verify-chain", "verify-heads" },
            names);
    }

@@ -65,7 +65,7 @@ public class AuditCommandsScaffoldTests
        // `archive` is a group (list / verify, ScadaBridge#42): check its leaves, not it.
        var audit = BuildAudit();
        var leaves = audit.Subcommands.SelectMany(c => c.Subcommands.Count == 0 ? [c] : c.Subcommands).ToList();
-        Assert.Equal(7, leaves.Count);
+        Assert.Equal(8, leaves.Count); // + verify-heads (ScadaBridge#107)
        Assert.All(leaves, sub =>
            Assert.True(sub.Action != null, $"Leaf command '{sub.Name}' has no action."));
    }
@@ -0,0 +1,158 @@
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using System.Text.Json;
+using ZB.MOM.WW.ScadaBridge.CLI.Commands;
+
+namespace ZB.MOM.WW.ScadaBridge.CLI.Tests.Commands;
+
+/// <summary>
+/// Tests for <c>scadabridge audit verify-heads</c> (ScadaBridge#107 item 1): reading off-box
+/// exported chain heads (file-sink lines and syslog-collector lines), collapsing duplicates,
+/// sending <c>VerifyAuditChainHeads</c>, and mapping the verdict to the exit code.
+/// </summary>
+[Collection("Console")]
+public sealed class AuditVerifyHeadsCommandTests : IDisposable
+{
+    private readonly DirectoryInfo _dir = Directory.CreateTempSubdirectory("verify-heads-");
+
+    /// <inheritdoc />
+    public void Dispose() => _dir.Delete(recursive: true);
+
+    private static string Line(string month, long seq, char hash, string exportedAt = "2026-05-02T00:00:00Z", string node = "central-a") =>
+        "{\"v\":1,\"month\":\"" + month + "\",\"lastSeq\":" + seq + ",\"lastHash\":\"" + new string(hash, 64)
+        + "\",\"purgedThroughSeq\":null,\"purgedThroughHash\":null,\"chainedRows\":" + seq + ",\"tombstones\":0,"
+        + "\"headUpdatedAtUtc\":\"2026-05-02T00:00:00Z\",\"exportedAtUtc\":\"" + exportedAt + "\",\"node\":\"" + node + "\"}";
+
+    /// <summary>
+    /// File-sink lines and RFC 5424 syslog lines both parse; junk is skipped; the same head
+    /// exported by both nodes (and re-exported) collapses to one record.
+    /// </summary>
+    [Fact]
+    public void ReadRecords_ParsesFileAndSyslogLines_SkipsJunk_AndCollapsesDuplicates()
+    {
+        Directory.CreateDirectory(Path.Combine(_dir.FullName, "2026"));
+        File.WriteAllLines(Path.Combine(_dir.FullName, "2026", "audit-chain-heads-2026-05-02-central-a.jsonl"),
+        [
+            Line("2026-05", 10, 'a'),
+            Line("2026-05", 12, 'b'),
+            "not json at all",
+            "",
+        ]);
+        File.WriteAllLines(Path.Combine(_dir.FullName, "syslog.log"),
+        [
+            "Nov 10 03:00:00 collector <109>1 2026-05-02T00:00:00.000Z central-b scadabridge 42 chainhead - "
+                + Line("2026-05", 12, 'b', "2026-05-02T01:00:00Z", "central-b"),
+            "<109>1 2026-05-02T00:00:00.000Z central-b scadabridge 42 chainhead - " + Line("2026-04", 7, 'c'),
+        ]);
+        File.WriteAllText(Path.Combine(_dir.FullName, "ignored.csv"), Line("2026-03", 1, 'd'));
+
+        var read = AuditVerifyHeadsHelpers.ReadRecords(_dir.FullName);
+
+        Assert.Equal(2, read.Files);
+        Assert.Equal(4, read.LinesRead);
+        Assert.Equal(1, read.LinesSkipped);
+        Assert.Equal(["2026-04/7", "2026-05/10", "2026-05/12"], read.Records.Select(r => $"{r.Month}/{r.LastSeq}"));
+        // The duplicate kept is the latest export.
+        Assert.Equal("central-b", read.Records.Single(r => r.LastSeq == 12).Node);
+    }
+
+    /// <summary>A record missing its hash or sequence is not a record.</summary>
+    /// <param name="line">The line.</param>
+    [Theory]
+    [InlineData("{\"v\":1,\"month\":\"2026-05\",\"lastSeq\":0,\"lastHash\":\"aa\"}")]
+    [InlineData("{\"v\":1,\"month\":\"2026-05\",\"lastSeq\":3}")]
+    [InlineData("{\"month\":\"2026-05\",\"lastSeq\":3,\"lastHash\":\"aa\"}")]
+    [InlineData("{ broken")]
+    public void TryParseLine_RejectsIncompleteRecords(string line)
+    {
+        Assert.Null(AuditVerifyHeadsHelpers.TryParseLine(line));
+    }
+
+    /// <summary>
+    /// Round trip: the request carries <c>VerifyAuditChainHeads</c> with the distinct records,
+    /// and the verdict drives the exit code (0 verified, 3 not).
+    /// </summary>
+    /// <param name="verified">Whether the fake server reports verified.</param>
+    /// <param name="format">Output format.</param>
+    /// <param name="expectedExit">Expected exit code.</param>
+    /// <param name="expectedText">Text the report must contain.</param>
+    /// <returns>A task that represents the asynchronous operation.</returns>
+    [Theory]
+    [InlineData(true, "table", 0, "Verified        : yes")]
+    [InlineData(false, "table", 3, "Mismatch")]
+    [InlineData(false, "json", 3, "\"verified\":false")]
+    public async Task VerifyHeads_SendsTheRecords_AndMapsVerdictToExitCode(bool verified, string format, int expectedExit, string expectedText)
+    {
+        var file = Path.Combine(_dir.FullName, "heads.jsonl");
+        File.WriteAllLines(file, [Line("2026-05", 12, 'b'), Line("2026-05", 12, 'b', "2026-05-02T02:00:00Z")]);
+        var body = "{\"verified\":" + (verified ? "true" : "false") + ",\"recordsChecked\":1,\"checks\":[{\"month\":\"2026-05\","
+            + "\"exportedSeq\":12,\"exportedAtUtc\":\"2026-05-02T02:00:00Z\",\"node\":\"central-a\",\"status\":\""
+            + (verified ? "Matches" : "Mismatch") + "\",\"databaseSeq\":12,\"detail\":\"d\"}]}";
+
+        var listener = new TcpListener(IPAddress.Loopback, 0);
+        listener.Start();
+        try
+        {
+            var port = ((IPEndPoint)listener.LocalEndpoint).Port;
+            string? requestBody = null;
+            var server = Task.Run(async () =>
+            {
+                using var client = await listener.AcceptTcpClientAsync();
+                await using var stream = client.GetStream();
+                using var reader = new StreamReader(stream, Encoding.UTF8, leaveOpen: true);
+                var length = 0;
+                while (await reader.ReadLineAsync() is { Length: > 0 } line)
+                    if (line.StartsWith("Content-Length:", StringComparison.OrdinalIgnoreCase))
+                        length = int.Parse(line[15..].Trim());
+                var buffer = new char[length];
+                var read = 0;
+                while (read < length)
+                {
+                    var count = await reader.ReadAsync(buffer.AsMemory(read));
+                    if (count == 0) throw new EndOfStreamException();
+                    read += count;
+                }
+                requestBody = new string(buffer);
+                var bytes = Encoding.UTF8.GetBytes(body);
+                var headers = Encoding.ASCII.GetBytes(
+                    $"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {bytes.Length}\r\nConnection: close\r\n\r\n");
+                await stream.WriteAsync(headers);
+                await stream.WriteAsync(bytes);
+            });
+
+            var root = AuditCommandTestHarness.BuildRoot();
+            var (exit, output, _) = AuditCommandTestHarness.Invoke(
+                root, "audit", "verify-heads", "--path", file,
+                "--url", $"http://127.0.0.1:{port}", "--username", "u", "--password", "p", "--format", format);
+            await server.WaitAsync(TimeSpan.FromSeconds(5));
+
+            Assert.Equal(expectedExit, exit);
+            Assert.Contains(expectedText, output);
+            using var payload = JsonDocument.Parse(requestBody!);
+            Assert.Equal("VerifyAuditChainHeads", payload.RootElement.GetProperty("command").GetString());
+            var records = payload.RootElement.GetProperty("payload").EnumerateObject()
+                .First(p => string.Equals(p.Name, "records", StringComparison.OrdinalIgnoreCase)).Value;
+            Assert.Equal(1, records.GetArrayLength());
+            Assert.Equal(new string('b', 64), records[0].EnumerateObject()
+                .First(p => string.Equals(p.Name, "lastHash", StringComparison.OrdinalIgnoreCase)).Value.GetString());
+        }
+        finally
+        {
+            listener.Stop();
+        }
+    }
+
+    /// <summary>No records found is an input error, and nothing is sent.</summary>
+    [Fact]
+    public void VerifyHeads_NoRecords_ExitsOneWithoutContactingTheServer()
+    {
+        var file = Path.Combine(_dir.FullName, "empty.jsonl");
+        File.WriteAllText(file, "nothing here\n");
+        var root = AuditCommandTestHarness.BuildRoot();
+        var (exit, _, _) = AuditCommandTestHarness.Invoke(
+            root, "audit", "verify-heads", "--path", file, "--url", "http://127.0.0.1:1", "--username", "u", "--password", "p");
+
+        Assert.Equal(1, exit);
+    }
+}
@@ -45,7 +45,7 @@ public class ManagementSurfaceParityTests
    }

    private static readonly HashSet<string> AuditManagementLeaves =
-        ["audit verify-chain", "audit archive list", "audit archive verify"];
+        ["audit verify-chain", "audit verify-heads", "audit archive list", "audit archive verify"];

    private static async Task<ConcurrentDictionary<string, string>> ProbeProductionTree(string? omitGroup = null)
    {
@@ -56,6 +56,12 @@ public class ManagementSurfaceParityTests
        await File.WriteAllTextAsync(file, "{}");
        var csv = Path.Combine(directory.FullName, "native.csv");
        await File.WriteAllTextAsync(csv, "SourceName,Connection,SourceReference,Filter\nSource,Connection,Path,\n");
+        // `audit verify-heads` (ScadaBridge#107) reads exported chain heads before it sends anything.
+        HeadsFile = Path.Combine(directory.FullName, "heads.jsonl");
+        await File.WriteAllTextAsync(HeadsFile,
+            "{\"v\":1,\"month\":\"2026-05\",\"lastSeq\":3,\"lastHash\":\"" + new string('a', 64)
+            + "\",\"purgedThroughSeq\":null,\"purgedThroughHash\":null,\"chainedRows\":3,\"tombstones\":0,"
+            + "\"headUpdatedAtUtc\":\"2026-05-02T00:00:00Z\",\"exportedAtUtc\":\"2026-05-02T00:00:00Z\",\"node\":\"central-a\"}\n");
        using var stop = new CancellationTokenSource(TimeSpan.FromSeconds(30));
        var listener = new TcpListener(IPAddress.Loopback, 0);
        listener.Start();
@@ -96,7 +102,8 @@ public class ManagementSurfaceParityTests
            foreach (var (leaf, path) in Leaves(root, []))
            {
                // Separate REST and persistent SignalR surfaces, not management contracts.
-                // `audit verify-chain` (VerifyAuditChain, ScadaBridge#41) and `audit archive
+                // `audit verify-chain` (VerifyAuditChain, ScadaBridge#41), `audit verify-heads`
+                // (VerifyAuditChainHeads, ScadaBridge#107) and `audit archive
                // list|verify` (ListAuditArchives / VerifyAuditArchive, ScadaBridge#42) are the
                // audit leaves that ARE management contracts, so they stay in the probe.
                if (path.Length == 0 || (path[0] == "audit" && !AuditManagementLeaves.Contains(string.Join(' ', path)))
@@ -135,8 +142,11 @@ public class ManagementSurfaceParityTests
            foreach (var entry in Leaves(child, [.. path, child.Name])) yield return entry;
    }

+    private static string HeadsFile = "";
+
    private static string ValueFor(Option option, string path, string file, string csv)
    {
+        if (path == "audit verify-heads" && option.Name == "--path") return HeadsFile;
        var type = option.GetType().GetGenericArguments().FirstOrDefault();
        type = Nullable.GetUnderlyingType(type ?? typeof(string)) ?? type;
        if (type == typeof(int) || type == typeof(long) || type == typeof(double)) return "1";