feat(audit)!: remove the AuditLog hash chain, head export and two-login maintenance hardening

Owner decision 2026-11-15, at IT security's request, pending a family-wide general solution.

Removed: the chaining service, hasher, verifier, chain head/tombstone entities and store, the #139
chain-start cutoff, AuditLog:HashChain (incl. ChainBacklog and HeadExport with its File/Syslog sinks
and export service), the audit-chain-head-export health check, the VerifyAuditChain /
VerifyAuditChainHeads management commands and the `audit verify-chain` / `audit verify-heads` CLI
commands, and the two-login plumbing (AuditLog:MaintenanceConnectionString, AuditMaintenanceContext,
the keyed audit-maintenance repository/archive). Purge, archive, retention, partition roll-forward
and backfill-source-node all run on the runtime connection again, as before #107.

Parquet archive (#42) kept, chain-free: format v2 (ten stored columns, no tombstone file, manifest
without chain state), the switch guard is the row count only (51003 gone), verify = data-file size,
SHA-256 and row count, and superseded-stale cleanup proves coverage by EventId. Format-v1 archives
still list, verify and are cleaned up (their tombstone file too).

Migration RemoveAuditLogHashChain drops the chain indexes, the scadabridge_audit_chainer role
(members first), both chain tables, ChainSeq/RowHash and the purger's AuditMaintenanceGrants grants;
writer/purger roles stay. Down restores them. Every chain-column reference is EXEC-deferred for the
idempotent script (#126); FlipAuditLogClusteredKey's two chain index statements get the same
deferral, behaviour unchanged, so re-running the idempotent script on a migrated database works.
The partition-switch staging table adds the chain columns only when the live table has them, so the
new binary purges either schema. Deploy gate: old binaries map the chain columns, so the migration
ships with the new binaries, both centrals stopped.

A leftover AuditLog:HashChain section or MaintenanceConnectionString (DEV/QA still carry
AuditLog__HashChain__Enabled=true) is ignored with one startup warning (RetiredAuditSettingsWarning).

Claude-Session: https://claude.ai/code/session_01GT67AFKC45MBz1wPZnVbX8
This commit is contained in:
Joseph Doherty
2026-11-15 17:37:33 -05:00
parent 125db03232
commit ed2ed0a367
82 changed files with 3203 additions and 7603 deletions
@@ -11,16 +11,16 @@ namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
/// </summary>
/// <remarks>
/// <c>verify</c> exit codes: <c>0</c> verified, <see cref="NotVerifiedExitCode"/> when the
/// files, the chain or the purge anchor disagree, <c>1</c> on a request error, <c>2</c> on an
/// data file disagrees with its manifest, <c>1</c> on a request error, <c>2</c> on an
/// authorization failure.
/// </remarks>
public static class AuditArchiveHelpers
{
/// <summary>Process exit code when an archive failed verification (tamper evidence).</summary>
/// <summary>Process exit code when an archive failed verification.</summary>
public const int NotVerifiedExitCode = 3;
/// <summary>
/// Client deadline for <c>verify</c>: the server hashes and re-chains a whole month under
/// Client deadline for <c>verify</c>: the server hashes and reads a whole month under
/// the long-running management Ask (5 min); the client waits a little longer.
/// </summary>
public static readonly TimeSpan VerifyRequestTimeout = TimeSpan.FromMinutes(6);
@@ -50,11 +50,11 @@ public static class AuditArchiveHelpers
return 0;
}
output.WriteLine($"{"NAME",-42} {"PARTITION",-9} {"STATUS",-8} {"ROWS",12} {"CHAINED",12} {"BYTES",14}");
output.WriteLine($"{"NAME",-42} {"PARTITION",-9} {"STATUS",-8} {"ROWS",12} {"BYTES",14}");
foreach (var a in archives.EnumerateArray())
{
output.WriteLine(string.Create(CultureInfo.InvariantCulture,
$"{Str(a, "name"),-42} {Str(a, "partition"),-9} {Str(a, "status"),-8} {Num(a, "rowCount"),12} {Num(a, "chainedRowCount"),12} {Num(a, "dataBytes"),14}"));
$"{Str(a, "name"),-42} {Str(a, "partition"),-9} {Str(a, "status"),-8} {Num(a, "rowCount"),12} {Num(a, "dataBytes"),14}"));
}
output.Flush();
@@ -90,20 +90,6 @@ public static class AuditArchiveHelpers
}
output.WriteLine($"Rows read : {Num(root, "rowsRead")}");
if (root.TryGetProperty("chains", out var chains) && chains.ValueKind == JsonValueKind.Array)
{
foreach (var c in chains.EnumerateArray())
{
var chain = c.GetProperty("chain");
output.WriteLine($"Chain {Str(chain, "partition")}: {Str(chain, "status")}, {Num(chain, "rowsVerified")} rows verified, "
+ $"{Num(chain, "tombstonesApplied")} tombstones, {Num(chain, "unchainedRows")} unchained; anchor {Str(c, "anchor")}");
output.WriteLine($" {Str(c, "anchorDetail")}");
if (chain.TryGetProperty("firstBreak", out var brk) && brk.ValueKind == JsonValueKind.Object)
{
output.WriteLine($" first break: chainSeq {Num(brk, "chainSeq")}, eventId {Str(brk, "eventId", "-")}, {Str(brk, "kind")} — {Str(brk, "detail")}");
}
}
}
output.Flush();
return verified ? 0 : NotVerifiedExitCode;
@@ -8,17 +8,14 @@ namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
/// The <c>scadabridge audit</c> command group. Provides read access to
/// the centralized append-only Audit Log via the REST endpoints
/// (<c>GET /api/audit/query</c>, <c>GET /api/audit/export</c>,
/// <c>GET /api/audit/tree</c>), plus <c>verify-chain</c>, which sends the
/// <c>VerifyAuditChain</c> management command (hash-chain tamper evidence, ScadaBridge#41),
/// <c>verify-heads</c>, which checks off-box exported chain heads with
/// <c>VerifyAuditChainHeads</c> (ScadaBridge#107), and <c>archive list</c> / <c>archive verify</c>
/// <c>GET /api/audit/tree</c>), plus <c>archive list</c> / <c>archive verify</c>
/// / <c>archive cleanup</c>, which send <c>ListAuditArchives</c> / <c>VerifyAuditArchive</c>
/// (Parquet cold archive, ScadaBridge#42) / <c>CleanupAuditArchives</c> (ScadaBridge#109).
/// </summary>
public static class AuditCommands
{
/// <summary>
/// Builds the <c>audit</c> command group with query, export, tree, and verify-chain
/// Builds the <c>audit</c> command group with query, export, tree and archive
/// sub-commands.
/// </summary>
/// <param name="urlOption">Global <c>--url</c> option for the management API endpoint.</param>
@@ -33,8 +30,6 @@ public static class AuditCommands
command.Add(BuildQuery(urlOption, formatOption, usernameOption, passwordOption));
command.Add(BuildExport(urlOption, formatOption, usernameOption, passwordOption));
command.Add(BuildTree(urlOption, formatOption, usernameOption, passwordOption));
command.Add(BuildVerifyChain(urlOption, formatOption, usernameOption, passwordOption));
command.Add(BuildVerifyHeads(urlOption, formatOption, usernameOption, passwordOption));
command.Add(BuildBackfillSourceNode(urlOption, formatOption, usernameOption, passwordOption));
command.Add(BuildArchive(urlOption, formatOption, usernameOption, passwordOption));
@@ -293,87 +288,10 @@ public static class AuditCommands
return cmd;
}
private static Command BuildVerifyChain(Option<string> urlOption, Option<string> formatOption, Option<string> usernameOption, Option<string> passwordOption)
{
var monthOption = new Option<string>("--month") { Description = "UTC calendar month to verify (YYYY-MM)", Required = true };
var cmd = new Command("verify-chain")
{
Description = "Verify the audit log hash chain for a month; exit 3 and report the first break if it was tampered with",
};
cmd.Add(monthOption);
cmd.SetAction(async (ParseResult result) =>
{
var month = result.GetValue(monthOption)!;
if (!AuditVerifyChainHelpers.IsValidMonth(month))
{
OutputFormatter.WriteError(
$"Invalid month '{month}'. Expected YYYY-MM (e.g. 2026-05).", "INVALID_ARGUMENT");
return 1;
}
var format = AuditCommandHelpers.ResolveFormat(result, formatOption);
return await CommandHelpers.ExecuteCommandAsync(
result, urlOption, formatOption, usernameOption, passwordOption,
new VerifyAuditChainCommand(month),
timeout: AuditVerifyChainHelpers.RequestTimeout,
onSuccess: json => AuditVerifyChainHelpers.Render(json, format, Console.Out));
});
return cmd;
}
/// <summary>
/// Builds the <c>audit archive</c> group (ScadaBridge#42): <c>list</c> and <c>verify</c>
/// over the AuditLog Parquet cold archives on the central archive directory.
/// Builds the <c>audit archive</c> group (ScadaBridge#42): <c>list</c>, <c>verify</c> and
/// <c>cleanup</c> over the AuditLog Parquet cold archives on the central archive directory.
/// </summary>
private static Command BuildVerifyHeads(Option<string> urlOption, Option<string> formatOption, Option<string> usernameOption, Option<string> passwordOption)
{
var pathOption = new Option<string>("--path")
{
Description = "Export file, or directory of *.jsonl/*.log/*.txt export files (file sink or syslog collector output)",
Required = true,
};
var cmd = new Command("verify-heads")
{
Description = "Check AuditLog chain heads exported off-box against the database chain; exit 3 if any no longer matches",
};
cmd.Add(pathOption);
cmd.SetAction(async (ParseResult result) =>
{
AuditVerifyHeadsHelpers.ReadResult read;
try
{
read = AuditVerifyHeadsHelpers.ReadRecords(result.GetValue(pathOption)!);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
OutputFormatter.WriteError(ex.Message, "INVALID_ARGUMENT");
return 1;
}
if (read.Records.Count == 0)
{
OutputFormatter.WriteError(
$"No chain-head records found in {read.Files} file(s) ({read.LinesSkipped} unparsable line(s)).",
"INVALID_ARGUMENT");
return 1;
}
if (read.LinesSkipped > 0)
{
Console.Error.WriteLine($"warning: skipped {read.LinesSkipped} line(s) without a chain-head record.");
}
var format = AuditCommandHelpers.ResolveFormat(result, formatOption);
return await CommandHelpers.ExecuteCommandAsync(
result, urlOption, formatOption, usernameOption, passwordOption,
new VerifyAuditChainHeadsCommand(read.Records),
timeout: AuditVerifyHeadsHelpers.RequestTimeout,
onSuccess: json => AuditVerifyHeadsHelpers.Render(json, format, Console.Out));
});
return cmd;
}
private static Command BuildArchive(Option<string> urlOption, Option<string> formatOption, Option<string> usernameOption, Option<string> passwordOption)
{
var group = new Command("archive") { Description = "List, verify and clean up the AuditLog Parquet cold archives" };
@@ -396,7 +314,7 @@ public static class AuditCommands
};
var verify = new Command("verify")
{
Description = "Verify an archive's files, its hash chain and its purge anchor; exit 3 if it does not verify",
Description = "Verify an archive's data file against its manifest; exit 3 if it does not verify",
};
verify.Add(nameOption);
verify.SetAction(async (ParseResult result) =>
@@ -1,88 +0,0 @@
using System.Globalization;
using System.Text.Json;
namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
/// <summary>
/// Helpers for the <c>audit verify-chain</c> subcommand (ScadaBridge#41). The command sends
/// the <c>VerifyAuditChain</c> management command for one UTC calendar month and reports
/// whether that month's AuditLog hash chain is intact, or the first break (chain partition,
/// <c>ChainSeq</c>, <c>EventId</c>, kind).
/// </summary>
/// <remarks>
/// Exit codes: <c>0</c> intact (or nothing chained yet), <see cref="BrokenExitCode"/> when
/// the chain is broken, <c>1</c> on a request error, <c>2</c> on an authorization failure.
/// </remarks>
public static class AuditVerifyChainHelpers
{
/// <summary>Process exit code when verification found a break (tamper evidence).</summary>
public const int BrokenExitCode = 3;
/// <summary>
/// Client deadline. The server bounds the call with the long-running management Ask
/// (5 min); the client waits a little longer so the server's answer always arrives.
/// </summary>
public static readonly TimeSpan RequestTimeout = TimeSpan.FromMinutes(6);
/// <summary>
/// Returns true if <paramref name="month"/> is a well-formed <c>YYYY-MM</c> value
/// with a real month (01-12). A malformed month (e.g. <c>2026-13</c>) is rejected.
/// </summary>
/// <param name="month">The month string to validate in YYYY-MM format.</param>
/// <returns><c>true</c> if the string is a well-formed YYYY-MM value with a real month; otherwise <c>false</c>.</returns>
public static bool IsValidMonth(string? month)
=> !string.IsNullOrWhiteSpace(month)
&& DateTime.TryParseExact(month, "yyyy-MM", CultureInfo.InvariantCulture,
DateTimeStyles.None, out _);
/// <summary>
/// Renders a <c>VerifyAuditChain</c> result and maps it to the process exit code.
/// <c>json</c> prints the server body verbatim; <c>table</c> prints a short report.
/// </summary>
/// <param name="json">The success body (an <c>AuditChainVerificationResult</c>, camelCase).</param>
/// <param name="format">Resolved output format.</param>
/// <param name="output">Writer for the report.</param>
/// <returns><c>0</c> when intact / not chained, <see cref="BrokenExitCode"/> when broken.</returns>
public static int Render(string json, string format, TextWriter output)
{
using var doc = JsonDocument.Parse(json);
var root = doc.RootElement;
var status = root.TryGetProperty("status", out var s) ? s.GetString() ?? "" : "";
var broken = string.Equals(status, "Broken", StringComparison.OrdinalIgnoreCase);
if (!string.Equals(format, "table", StringComparison.OrdinalIgnoreCase))
{
output.WriteLine(json);
output.Flush();
return broken ? BrokenExitCode : 0;
}
output.WriteLine($"Partition : {Str(root, "partition")}");
output.WriteLine($"Status : {status}");
output.WriteLine($"Rows verified : {Num(root, "rowsVerified")}");
output.WriteLine($"Tombstones applied: {Num(root, "tombstonesApplied")}");
output.WriteLine($"Purged through : {Num(root, "purgedThroughSeq", "-")}");
output.WriteLine($"Head sequence : {Num(root, "headSeq", "-")}");
output.WriteLine($"Unchained rows : {Num(root, "unchainedRows")}");
output.WriteLine($"Chain from : {Str(root, "chainFromUtc", "- (whole table in scope)")}");
output.WriteLine($"Pre-cutoff rows : {Num(root, "preCutoffRows")}");
if (root.TryGetProperty("firstBreak", out var brk) && brk.ValueKind == JsonValueKind.Object)
{
output.WriteLine("First break:");
output.WriteLine($" partition : {Str(brk, "partition")}");
output.WriteLine($" chainSeq : {Num(brk, "chainSeq")}");
output.WriteLine($" eventId : {Str(brk, "eventId", "-")}");
output.WriteLine($" kind : {Str(brk, "kind")}");
output.WriteLine($" detail : {Str(brk, "detail")}");
}
output.Flush();
return broken ? BrokenExitCode : 0;
}
private static string Str(JsonElement e, string name, string missing = "")
=> e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() ?? missing : missing;
private static string Num(JsonElement e, string name, string missing = "0")
=> e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.Number ? v.GetInt64().ToString(CultureInfo.InvariantCulture) : missing;
}
@@ -1,160 +0,0 @@
using System.Globalization;
using System.Text.Json;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
namespace ZB.MOM.WW.ScadaBridge.CLI.Commands;
/// <summary>
/// Helpers for <c>audit verify-heads</c> (ScadaBridge#107 item 1): read AuditLog chain heads
/// exported off-box (the <c>AuditLog:HashChain:HeadExport</c> JSON lines — straight from the
/// file sink, or from a syslog collector's log), send them in a
/// <c>VerifyAuditChainHeads</c> management command, and render the verdicts.
/// </summary>
/// <remarks>
/// Exit codes: <c>0</c> every exported head is consistent with the database,
/// <see cref="FailedExitCode"/> at least one is not (tamper evidence), <c>1</c> on a request
/// or input error, <c>2</c> on an authorization failure.
/// </remarks>
public static class AuditVerifyHeadsHelpers
{
/// <summary>Process exit code when a head check failed.</summary>
public const int FailedExitCode = 3;
/// <summary>Client deadline (the server bounds the call with the long-running Ask).</summary>
public static readonly TimeSpan RequestTimeout = TimeSpan.FromMinutes(6);
private static readonly JsonSerializerOptions ReadOptions = new(JsonSerializerDefaults.Web);
/// <summary>The result of reading export files.</summary>
/// <param name="Records">Distinct records (by month, sequence, hash), latest export kept.</param>
/// <param name="LinesRead">Lines that held a head record.</param>
/// <param name="LinesSkipped">Non-empty lines without a parsable record.</param>
/// <param name="Files">Files read.</param>
public sealed record ReadResult(IReadOnlyList<AuditChainHeadRecord> Records, int LinesRead, int LinesSkipped, int Files);
/// <summary>
/// Reads every record from <paramref name="path"/> — one file, or every <c>*.jsonl</c> /
/// <c>*.log</c> / <c>*.txt</c> file below a directory. Each line's JSON object is taken from
/// its first <c>{</c> to its last <c>}</c>, so syslog-prefixed lines parse too.
/// </summary>
/// <param name="path">File or directory.</param>
/// <returns>The records read.</returns>
/// <exception cref="FileNotFoundException">The path does not exist.</exception>
public static ReadResult ReadRecords(string path)
{
string[] files;
if (File.Exists(path))
{
files = [path];
}
else if (Directory.Exists(path))
{
files = Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories)
.Where(f => f.EndsWith(".jsonl", StringComparison.OrdinalIgnoreCase)
|| f.EndsWith(".log", StringComparison.OrdinalIgnoreCase)
|| f.EndsWith(".txt", StringComparison.OrdinalIgnoreCase))
.Order(StringComparer.Ordinal)
.ToArray();
}
else
{
throw new FileNotFoundException($"'{path}' is neither a file nor a directory.", path);
}
var records = new List<AuditChainHeadRecord>();
int read = 0, skipped = 0;
foreach (var file in files)
{
foreach (var line in File.ReadLines(file))
{
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
var record = TryParseLine(line);
if (record is null)
{
skipped++;
continue;
}
read++;
records.Add(record);
}
}
var distinct = records
.GroupBy(r => (r.Month, r.LastSeq, Hash: r.LastHash.ToLowerInvariant()))
.Select(g => g.OrderByDescending(r => r.ExportedAtUtc).First())
.OrderBy(r => r.Month, StringComparer.Ordinal)
.ThenBy(r => r.LastSeq)
.ToList();
return new ReadResult(distinct, read, skipped, files.Length);
}
/// <summary>Parses one export line; null when it holds no valid record.</summary>
/// <param name="line">The line.</param>
/// <returns>The record, or null.</returns>
public static AuditChainHeadRecord? TryParseLine(string line)
{
var start = line.IndexOf('{');
var end = line.LastIndexOf('}');
if (start < 0 || end <= start)
{
return null;
}
try
{
var record = JsonSerializer.Deserialize<AuditChainHeadRecord>(line.AsSpan(start, end - start + 1), ReadOptions);
return record is { V: >= 1, Month.Length: 7, LastHash.Length: > 0 } && record.LastSeq > 0 ? record : null;
}
catch (JsonException)
{
return null;
}
}
/// <summary>Renders a <c>VerifyAuditChainHeads</c> result and maps it to the exit code.</summary>
/// <param name="json">Success body (an <c>AuditChainHeadVerificationResult</c>, camelCase).</param>
/// <param name="format">Resolved output format.</param>
/// <param name="output">Writer for the report.</param>
/// <returns><c>0</c> when verified, <see cref="FailedExitCode"/> otherwise.</returns>
public static int Render(string json, string format, TextWriter output)
{
using var doc = JsonDocument.Parse(json);
var root = doc.RootElement;
var verified = root.TryGetProperty("verified", out var v) && v.ValueKind == JsonValueKind.True;
if (!string.Equals(format, "table", StringComparison.OrdinalIgnoreCase))
{
output.WriteLine(json);
output.Flush();
return verified ? 0 : FailedExitCode;
}
output.WriteLine($"Verified : {(verified ? "yes" : "NO")}");
output.WriteLine($"Heads checked : {Num(root, "recordsChecked")}");
if (root.TryGetProperty("checks", out var checks) && checks.ValueKind == JsonValueKind.Array)
{
output.WriteLine($"{"MONTH",-8} {"EXPORTED SEQ",12} {"DB SEQ",8} {"STATUS",-13} DETAIL");
foreach (var c in checks.EnumerateArray())
{
output.WriteLine(string.Create(CultureInfo.InvariantCulture,
$"{Str(c, "month"),-8} {Num(c, "exportedSeq"),12} {Num(c, "databaseSeq", "-"),8} {Str(c, "status"),-13} {Str(c, "detail")}"));
}
}
output.Flush();
return verified ? 0 : FailedExitCode;
}
private static string Str(JsonElement e, string name)
=> e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() ?? "" : "";
private static string Num(JsonElement e, string name, string missing = "0")
=> e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.Number
? v.GetInt64().ToString(CultureInfo.InvariantCulture)
: missing;
}