feat(audit): off-box chain-head export, verify-heads, maintenance login (ScadaBridge#107)

Item 1 (owner decision: no signing). AuditLog:HashChain:HeadExport (default
off): both centrals periodically read every month's chain head (LastSeq,
LastHash, purge anchor, chained rows, tombstones; one snapshot under the
chain lock SHARED) and append the changed heads to a sink the DB login cannot
write - File (append-only JSONL per node per day, local or UNC) or Syslog
(RFC 5424 over UDP / TCP octet counting). `audit verify-heads --path` reads
those records back (file or syslog-collector lines) and the new
VerifyAuditChainHeads command checks each against the database: Matches /
Extended / Purged, or Mismatch / Regressed / MissingHead / MissingLink /
CountMismatch (exit 3). AuditChainHeadStoreTests forges a chain with the
production hasher: verify-chain says Intact, verify-heads says Mismatch.

Item 3. AuditLog:MaintenanceConnectionString (${secret:}-capable, unset =
today's single login) runs the chain job, chain verify / head export, the
partition switch-out + archive and the per-channel purge on
AuditMaintenanceContext (keyed audit-maintenance repository/archive, purge
actor prefers them). AuditMaintenanceLoginTests runs the hardened setup on
SQL Server: writer (datareader/datawriter/audit_writer) ingests and is
refused chaining and deleting; the chainer+purger login does the full cycle.
Two-login grants documented in Component-AuditLog.md.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
2026-11-09 23:16:05 -05:00
parent 5e7d5389d4
commit c4d7ec9b96
40 changed files with 2769 additions and 43 deletions
@@ -0,0 +1,139 @@
using System.Globalization;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
/// <summary>
/// Pure comparison of one exported chain head (ScadaBridge#107 item 1) with the database's
/// current state for that month. Kept free of I/O so every verdict is unit-testable; the
/// database side supplies the head snapshot and, when needed, the link stored at the
/// exported sequence number.
/// </summary>
public static class AuditChainHeadComparer
{
/// <summary>What the database stores at one chain sequence number.</summary>
/// <param name="RowHash">The <c>RowHash</c> of the live row carrying the sequence, or null.</param>
/// <param name="TombstoneHash">The <c>RowHash</c> of a per-channel purge tombstone carrying it, or null.</param>
public readonly record struct StoredLink(byte[]? RowHash, byte[]? TombstoneHash);
/// <summary>Compares one exported head with the database.</summary>
/// <param name="record">The exported head.</param>
/// <param name="head">The database head for the record's month, or null when there is none.</param>
/// <param name="linkAt">Reads the stored link at a sequence number (only called when the chain has grown).</param>
/// <returns>The verdict.</returns>
public static async Task<AuditChainHeadCheck> CompareAsync(
AuditChainHeadRecord record, AuditChainHeadSnapshot? head, Func<long, Task<StoredLink>> linkAt)
{
ArgumentNullException.ThrowIfNull(record);
ArgumentNullException.ThrowIfNull(linkAt);
AuditChainHeadCheck Result(AuditChainHeadStatus status, string detail) =>
new(record.Month, record.LastSeq, record.ExportedAtUtc, record.Node, status, head?.LastSeq, detail);
if (!TryParseHex(record.LastHash, out var exported))
{
return Result(AuditChainHeadStatus.Mismatch, $"Exported hash '{record.LastHash}' is not 32 bytes of hex.");
}
if (head is null)
{
return Result(AuditChainHeadStatus.MissingHead,
$"{record.Month} has no chain head in the database, but {record.LastSeq} links were exported for it.");
}
if (head.LastSeq < record.LastSeq)
{
return Result(AuditChainHeadStatus.Regressed,
$"Database head is at sequence {head.LastSeq}, below the exported {record.LastSeq}: links were removed and the head rolled back.");
}
if (head.LastSeq == record.LastSeq)
{
if (!exported.AsSpan().SequenceEqual(head.LastHash))
{
return Result(AuditChainHeadStatus.Mismatch,
$"Database head hash at sequence {head.LastSeq} is {ToHex(head.LastHash)}, exported {record.LastHash}: the chain was rewritten.");
}
// No link was added since the export. Unless a partition purge set (or moved) the
// anchor since, rows + tombstones are invariant: the per-channel purge turns a row
// into a tombstone, nothing else may remove a chained row.
if (head.PurgedThroughSeq == record.PurgedThroughSeq
&& head.ChainedRows + head.Tombstones != record.ChainedRows + record.Tombstones)
{
return Result(AuditChainHeadStatus.CountMismatch,
$"Chained rows + tombstones are {head.ChainedRows} + {head.Tombstones} in the database, {record.ChainedRows} + {record.Tombstones} exported.");
}
return Result(AuditChainHeadStatus.Matches, "Database head equals the exported head.");
}
// The chain has grown: the link at the exported sequence must still carry the exported hash.
if (head.PurgedThroughSeq is { } anchor && record.LastSeq <= anchor)
{
if (record.LastSeq < anchor)
{
return Result(AuditChainHeadStatus.Purged,
$"Sequence {record.LastSeq} was removed by a partition purge (anchor at {anchor}); verify the month's archive.");
}
return head.PurgedThroughHash is { } anchorHash && exported.AsSpan().SequenceEqual(anchorHash)
? Result(AuditChainHeadStatus.Extended, $"Exported head is the partition-purge anchor at {anchor}.")
: Result(AuditChainHeadStatus.Mismatch, $"Partition-purge anchor at {anchor} does not carry the exported hash.");
}
var link = await linkAt(record.LastSeq).ConfigureAwait(false);
var stored = link.RowHash ?? link.TombstoneHash;
if (stored is null)
{
return Result(AuditChainHeadStatus.MissingLink,
$"No row, tombstone or anchor carries sequence {record.LastSeq}: a chained row was deleted outside the purge paths.");
}
return exported.AsSpan().SequenceEqual(stored)
? Result(AuditChainHeadStatus.Extended,
$"Chain grew to {head.LastSeq}; the {(link.RowHash is null ? "tombstone" : "row")} at {record.LastSeq} still carries the exported hash.")
: Result(AuditChainHeadStatus.Mismatch,
$"The {(link.RowHash is null ? "tombstone" : "row")} at sequence {record.LastSeq} carries {ToHex(stored)}, exported {record.LastHash}: the chain was rewritten.");
}
/// <summary>Lowercase hex of a hash.</summary>
/// <param name="bytes">The bytes.</param>
/// <returns>Hex string.</returns>
public static string ToHex(byte[] bytes) => Convert.ToHexString(bytes).ToLowerInvariant();
/// <summary>Parses a <c>YYYY-MM</c> month into its first instant (UTC).</summary>
/// <param name="month">The month text.</param>
/// <param name="monthStart">The parsed month.</param>
/// <returns>True when well-formed.</returns>
public static bool TryParseMonth(string? month, out DateTime monthStart)
{
if (DateTime.TryParseExact(month, "yyyy-MM", CultureInfo.InvariantCulture, DateTimeStyles.None, out var parsed))
{
monthStart = new DateTime(parsed.Year, parsed.Month, 1, 0, 0, 0, DateTimeKind.Utc);
return true;
}
monthStart = default;
return false;
}
private static bool TryParseHex(string? hex, out byte[] bytes)
{
bytes = [];
if (hex is null || hex.Length != AuditChainHasher.HashLength * 2)
{
return false;
}
try
{
bytes = Convert.FromHexString(hex);
return true;
}
catch (FormatException)
{
return false;
}
}
}
@@ -45,7 +45,7 @@ public static class AuditChainLock
/// those two columns; the ingest writer role keeps its table-wide DENY UPDATE.
/// </para>
/// </remarks>
public sealed class AuditLogHashChain : IAuditLogHashChain
public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore
{
// 4 parameters per row; 200 rows = 800 parameters, far under SQL Server's 2,100 cap.
private const int UpdateChunkRows = 200;
@@ -273,6 +273,151 @@ public sealed class AuditLogHashChain : IAuditLogHashChain
}
}
/// <inheritdoc />
public async Task<IReadOnlyList<AuditChainHeadSnapshot>> ReadHeadsAsync(CancellationToken ct = default)
{
var conn = _context.Database.GetDbConnection();
var openedHere = await OpenAsync(conn, ct).ConfigureAwait(false);
try
{
await using var tx = await conn.BeginTransactionAsync(IsolationLevel.ReadCommitted, ct).ConfigureAwait(false);
await AcquireSharedOrThrowAsync(conn, tx, ct).ConfigureAwait(false);
var heads = await ReadHeadsCoreAsync(conn, tx, ct).ConfigureAwait(false);
await tx.CommitAsync(ct).ConfigureAwait(false);
return heads;
}
finally
{
if (openedHere)
{
await conn.CloseAsync().ConfigureAwait(false);
}
}
}
/// <inheritdoc />
public async Task<AuditChainHeadVerificationResult> VerifyExportedHeadsAsync(
IReadOnlyList<AuditChainHeadRecord> records, CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(records);
// Collapse duplicates (both centrals export, and every unchanged re-export repeats a
// record): one check per distinct (month, sequence, hash) claim.
var distinct = records
.GroupBy(r => (r.Month, r.LastSeq, Hash: r.LastHash?.ToLowerInvariant()))
.Select(g => g.OrderByDescending(r => r.ExportedAtUtc).First())
.ToList();
var conn = _context.Database.GetDbConnection();
var openedHere = await OpenAsync(conn, ct).ConfigureAwait(false);
try
{
await using var tx = await conn.BeginTransactionAsync(IsolationLevel.ReadCommitted, ct).ConfigureAwait(false);
await AcquireSharedOrThrowAsync(conn, tx, ct).ConfigureAwait(false);
var heads = (await ReadHeadsCoreAsync(conn, tx, ct).ConfigureAwait(false))
.ToDictionary(h => AuditChainHasher.PartitionLabel(h.Month), StringComparer.Ordinal);
var checks = new List<AuditChainHeadCheck>(distinct.Count);
foreach (var record in distinct)
{
if (!AuditChainHeadComparer.TryParseMonth(record.Month, out var month))
{
checks.Add(new AuditChainHeadCheck(record.Month ?? "", record.LastSeq, record.ExportedAtUtc, record.Node,
AuditChainHeadStatus.Mismatch, null, $"Exported month '{record.Month}' is not YYYY-MM."));
continue;
}
heads.TryGetValue(AuditChainHasher.PartitionLabel(month), out var head);
checks.Add(await AuditChainHeadComparer.CompareAsync(
record, head, seq => ReadStoredLinkAsync(conn, tx, month, seq, ct)).ConfigureAwait(false));
}
await tx.CommitAsync(ct).ConfigureAwait(false);
var ordered = checks
.OrderByDescending(c => c.IsFailure)
.ThenBy(c => c.Month, StringComparer.Ordinal)
.ThenBy(c => c.ExportedSeq)
.ToList();
return new AuditChainHeadVerificationResult(ordered.All(c => !c.IsFailure), ordered.Count, ordered);
}
finally
{
if (openedHere)
{
await conn.CloseAsync().ConfigureAwait(false);
}
}
}
private async Task AcquireSharedOrThrowAsync(DbConnection conn, DbTransaction tx, CancellationToken ct)
{
if (!await TryAcquireLockAsync(conn, tx, "Shared", VerifyLockTimeout, ct).ConfigureAwait(false))
{
await tx.RollbackAsync(ct).ConfigureAwait(false);
throw new TimeoutException(
$"The AuditLog hash chain was busy (chaining or purge in progress) for {VerifyLockTimeout.TotalSeconds:0}s; retry.");
}
}
private async Task<IReadOnlyList<AuditChainHeadSnapshot>> ReadHeadsCoreAsync(DbConnection conn, DbTransaction tx, CancellationToken ct)
{
// Per month: the head row, the chained rows still present (the filtered ChainSeq index,
// partition-eliminated by the month range) and the tombstones. The head table holds one
// row per month ever chained, so this is a few dozen rows.
var heads = new List<AuditChainHeadSnapshot>();
await using var cmd = Command(conn, tx, @"
SELECT h.ChainMonth, h.LastSeq, h.LastHash, h.PurgedThroughSeq, h.PurgedThroughHash, h.UpdatedAtUtc,
(SELECT COUNT_BIG(*) FROM dbo.AuditLog a
WHERE a.ChainSeq IS NOT NULL
AND a.OccurredAtUtc >= CAST(h.ChainMonth AS datetime2(7))
AND a.OccurredAtUtc < DATEADD(MONTH, 1, CAST(h.ChainMonth AS datetime2(7)))) AS ChainedRows,
(SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone t WHERE t.ChainMonth = h.ChainMonth) AS Tombstones
FROM dbo.AuditLogChainHead h
ORDER BY h.ChainMonth;");
cmd.CommandTimeout = VerifyCommandTimeoutSeconds;
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
while (await reader.ReadAsync(ct).ConfigureAwait(false))
{
heads.Add(new AuditChainHeadSnapshot(
DateTime.SpecifyKind(reader.GetDateTime(0), DateTimeKind.Utc),
reader.GetInt64(1),
(byte[])reader.GetValue(2),
reader.IsDBNull(3) ? null : reader.GetInt64(3),
reader.IsDBNull(4) ? null : (byte[])reader.GetValue(4),
reader.GetInt64(6),
reader.GetInt64(7),
DateTime.SpecifyKind(reader.GetDateTime(5), DateTimeKind.Utc)));
}
return heads;
}
private async Task<AuditChainHeadComparer.StoredLink> ReadStoredLinkAsync(
DbConnection conn, DbTransaction tx, DateTime month, long seq, CancellationToken ct)
{
byte[]? rowHash;
await using (var cmd = Command(conn, tx,
"SELECT TOP (1) RowHash FROM dbo.AuditLog WHERE ChainSeq = @seq AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd;"))
{
cmd.CommandTimeout = VerifyCommandTimeoutSeconds;
cmd.Parameters.Add(new SqlParameter("@seq", SqlDbType.BigInt) { Value = seq });
AddMonthRange(cmd, month);
rowHash = await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false) as byte[];
}
byte[]? tombstoneHash = null;
if (rowHash is null)
{
await using var cmd = Command(conn, tx,
"SELECT RowHash FROM dbo.AuditLogChainTombstone WHERE ChainMonth = @month AND ChainSeq = @seq;");
AddDate(cmd, "@month", month);
cmd.Parameters.Add(new SqlParameter("@seq", SqlDbType.BigInt) { Value = seq });
tombstoneHash = await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false) as byte[];
}
return new AuditChainHeadComparer.StoredLink(rowHash, tombstoneHash);
}
/// <summary>
/// Takes the chain application lock inside <paramref name="tx"/> (released at commit /
/// rollback). Returns false when the lock could not be granted within <paramref name="timeout"/>.
@@ -0,0 +1,82 @@
using Microsoft.AspNetCore.DataProtection;
using Microsoft.EntityFrameworkCore;
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
/// <summary>
/// The connection the AuditLog maintenance paths run on — the hash-chain job and both purge
/// paths (partition switch-out with its archive, per-channel DELETE) — ScadaBridge#107 item 3.
/// </summary>
/// <remarks>
/// <para>
/// With <c>AuditLog:MaintenanceConnectionString</c> unset (the default), this is the scope's
/// ordinary <see cref="ScadaBridgeDbContext"/>: one login for everything, exactly as before.
/// With it set, the maintenance paths get their OWN context on that connection string, so a
/// DBA can give the runtime (writer) login only <c>scadabridge_audit_writer</c> — whose DENYs
/// stop it writing chain state or deleting audit rows — and a separate maintenance login the
/// <c>scadabridge_audit_chainer</c> + <c>scadabridge_audit_purger</c> roles. The string is read
/// after the pre-host <c>${secret:}</c> expansion, so it may (and in production should) be a
/// secret reference.
/// </para>
/// <para>Scoped: an owned context is disposed with the scope; a borrowed one is left to DI.</para>
/// </remarks>
public sealed class AuditMaintenanceContext : IAsyncDisposable, IDisposable
{
/// <summary>DI key of the maintenance-login <c>IAuditLogRepository</c> and <c>IAuditLogArchive</c>.</summary>
public const string ServiceKey = "audit-maintenance";
private readonly bool _owned;
private AuditMaintenanceContext(ScadaBridgeDbContext context, bool owned)
{
Context = context;
_owned = owned;
}
/// <summary>The context to run maintenance SQL on.</summary>
public ScadaBridgeDbContext Context { get; }
/// <summary>True when <see cref="Context"/> is on the separate maintenance connection string.</summary>
public bool UsesSeparateLogin => _owned;
/// <summary>Wraps the scope's ordinary context (no separate maintenance login configured).</summary>
/// <param name="context">The scope's context.</param>
/// <returns>The wrapper.</returns>
public static AuditMaintenanceContext Shared(ScadaBridgeDbContext context)
=> new(context ?? throw new ArgumentNullException(nameof(context)), owned: false);
/// <summary>Opens an owned context on the maintenance connection string.</summary>
/// <param name="connectionString">The maintenance connection string.</param>
/// <param name="protectionProvider">Data protection for secret-bearing columns (same as the runtime context).</param>
/// <returns>The wrapper; disposing it disposes the context.</returns>
public static AuditMaintenanceContext Separate(string connectionString, IDataProtectionProvider? protectionProvider)
{
ArgumentException.ThrowIfNullOrWhiteSpace(connectionString);
var options = new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer(connectionString, sql => sql.EnableRetryOnFailure(
maxRetryCount: 5, maxRetryDelay: TimeSpan.FromSeconds(30), errorNumbersToAdd: null))
.ConfigureWarnings(w => w.Ignore(
Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning))
.Options;
var context = protectionProvider is null
? new ScadaBridgeDbContext(options)
: new ScadaBridgeDbContext(options, protectionProvider);
return new AuditMaintenanceContext(context, owned: true);
}
/// <inheritdoc />
public ValueTask DisposeAsync() => _owned ? Context.DisposeAsync() : ValueTask.CompletedTask;
/// <inheritdoc />
public void Dispose()
{
if (_owned)
{
Context.Dispose();
}
}
}
/// <summary>The configured maintenance connection string (null = use the runtime connection).</summary>
/// <param name="ConnectionString">The connection string, or null.</param>
public sealed record AuditMaintenanceConnection(string? ConnectionString);
@@ -20,8 +20,14 @@ public static class ServiceCollectionExtensions
/// </summary>
/// <param name="services">The service collection to register into.</param>
/// <param name="connectionString">SQL Server connection string for the central configuration database.</param>
/// <param name="auditMaintenanceConnectionString">
/// Optional separate connection string for the AuditLog maintenance paths — hash-chain job
/// and both purge paths (<c>AuditLog:MaintenanceConnectionString</c>, ScadaBridge#107 item 3).
/// Null or blank runs them on <paramref name="connectionString"/> as before.
/// </param>
/// <returns>The same <paramref name="services"/> collection, for chaining.</returns>
public static IServiceCollection AddConfigurationDatabase(this IServiceCollection services, string connectionString)
public static IServiceCollection AddConfigurationDatabase(
this IServiceCollection services, string connectionString, string? auditMaintenanceConnectionString = null)
{
// The DbContext is constructed via the (options, IDataProtectionProvider) overload so
// secret-bearing configuration columns are encrypted at rest. AddDataProtection below
@@ -111,9 +117,31 @@ public static class ServiceCollectionExtensions
// fresh scope on each tick (mirrors AuditLogPurgeActor / AuditLogIngestActor).
services.AddScoped<IPartitionMaintenance, AuditLogPartitionMaintenance>();
// AuditLog maintenance connection (ScadaBridge#107 item 3): the chain job and both purge
// paths run on AuditMaintenanceContext — the scope's own context unless a separate
// maintenance login is configured, in which case an owned context on that string.
services.AddSingleton(new AuditMaintenanceConnection(
string.IsNullOrWhiteSpace(auditMaintenanceConnectionString) ? null : auditMaintenanceConnectionString));
services.AddScoped(sp =>
{
var maintenance = sp.GetRequiredService<AuditMaintenanceConnection>().ConnectionString;
return maintenance is null
? AuditMaintenanceContext.Shared(sp.GetRequiredService<ScadaBridgeDbContext>())
: AuditMaintenanceContext.Separate(maintenance, sp.GetService<IDataProtectionProvider>());
});
services.AddKeyedScoped<IAuditLogRepository>(AuditMaintenanceContext.ServiceKey, (sp, _) => new AuditLogRepository(
sp.GetRequiredService<AuditMaintenanceContext>().Context,
sp.GetService<Microsoft.Extensions.Logging.ILogger<AuditLogRepository>>()));
// AuditLog hash-chain tamper evidence (ScadaBridge#41): chained by the central
// AuditLogHashChainService, verified by the VerifyAuditChain management command.
services.AddScoped<IAuditLogHashChain, AuditLogHashChain>();
// AuditLogHashChainService, verified by the VerifyAuditChain management command; the
// same instance reads/verifies the off-box head export (#107 item 1). Both on the
// maintenance connection.
services.AddScoped(sp => new AuditLogHashChain(
sp.GetRequiredService<AuditMaintenanceContext>().Context,
sp.GetService<Microsoft.Extensions.Logging.ILogger<AuditLogHashChain>>()));
services.AddScoped<IAuditLogHashChain>(sp => sp.GetRequiredService<AuditLogHashChain>());
services.AddScoped<IAuditChainHeadStore>(sp => sp.GetRequiredService<AuditLogHashChain>());
// AuditLog Parquet cold archive (ScadaBridge#42): written by the purge actor before a
// partition switch, listed/verified by the ListAuditArchives / VerifyAuditArchive
@@ -123,6 +151,12 @@ public static class ServiceCollectionExtensions
sp.GetRequiredService<ScadaBridgeDbContext>(),
sp.GetService<Microsoft.Extensions.Options.IOptions<AuditLogArchiveOptions>>()?.Value ?? new AuditLogArchiveOptions(),
sp.GetService<Microsoft.Extensions.Logging.ILogger<AuditLogPartitionArchive>>()));
// The purge actor's archive-then-purge runs on the maintenance connection (#107 item 3);
// list/verify from the management surface stay on the runtime connection (reads only).
services.AddKeyedScoped<IAuditLogArchive>(AuditMaintenanceContext.ServiceKey, (sp, _) => new AuditLogPartitionArchive(
sp.GetRequiredService<AuditMaintenanceContext>().Context,
sp.GetService<Microsoft.Extensions.Options.IOptions<AuditLogArchiveOptions>>()?.Value ?? new AuditLogArchiveOptions(),
sp.GetService<Microsoft.Extensions.Logging.ILogger<AuditLogPartitionArchive>>()));
// Opt-in config-audit retention (ScadaBridge:ConfigAudit:RetentionDays, default 0 = off).
// Options binding to configuration is done by the host; without it the defaults apply