feat(audit): hash-chain cutoff - chain only rows written from first enablement, no backlog (#139)

Owner decision on #107 (2026-11-13): enabling AuditLog:HashChain must not
chain the historical backlog. main chained every unchained row oldest-first.

- The first chaining pass on a pristine database (no head, no chained row)
  writes a chain-start anchor: AuditLogChainTombstone (cutoff month,
  ChainSeq 0), PurgedAtUtc = DB SYSUTCDATETIME(), RowHash =
  SHA-256("ScadaBridge.AuditLog.ChainCutoff.v1" || YYYY-MM || ticks).
  Written once under the exclusive chain lock, so both centrals agree
  whatever their clocks/config. No migration: existing table + grants.
- Only rows with OccurredAtUtc >= cutoff are chained. The anchor hash is
  the cutoff month's genesis, binding the cutoff into the chain.
- AuditLog:HashChain:ChainBacklog (default false) restores chain-everything,
  read only on the very first enablement. Legacy chains (heads, no anchor)
  keep chaining everything.
- verify-chain splits unchained rows into UnchainedRows vs PreCutoffRows
  (+ ChainFromUtc); new breaks ChainedBeforeCutoff (row re-dated before the
  cutoff, or cutoff moved forward) and CutoffAnchorInvalid (anchor hash /
  month / multiplicity). Archive verify uses the anchor from its own
  tombstone file or the DB.
- Partition switch-out keeps the anchor (deletes ChainSeq > 0 only); the
  head store / export / verify-heads tombstone count excludes it.
- CLI verify-chain table prints Chain from + Pre-cutoff rows.

Tests: AuditLogHashChainCutoffTests (7, MSSQL), 6 verifier unit tests,
service/CLI/management wire tests; 9 mutations all killed.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
2026-11-13 03:14:03 -05:00
parent 712c5117ee
commit 52da02a76a
21 changed files with 1208 additions and 49 deletions
@@ -15,10 +15,14 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// them chains at a time, so a dual-active pair still produces one gap-free chain per month.
/// </para>
/// <para>
/// Enabling it on an existing database chains the historical backlog oldest month first,
/// <see cref="BatchSize"/> × <see cref="MaxBatchesPerTick"/> rows per tick, so the first
/// ticks run long. Rows chained late are only protected from the moment they are chained.
/// Run <c>audit backfill-source-node</c> BEFORE enabling: a chained row's SourceNode is frozen.
/// <b>Chain start (ScadaBridge#139).</b> Enabling it on an existing database does NOT chain the
/// historical backlog: the first chaining pass records a chain-start anchor in the database at
/// the database's current UTC time, and only rows dated from then on are chained. Older rows
/// stay unchained and are reported by <c>audit verify-chain</c> as pre-cutoff rows, never as a
/// break. The anchor is written once and is the same for both central nodes. Set
/// <see cref="ChainBacklog"/> to chain the backlog instead (only effective on the very first
/// enablement). Run <c>audit backfill-source-node</c> BEFORE enabling either way: a chained
/// row's SourceNode is frozen.
/// </para>
/// </remarks>
public sealed class AuditLogHashChainOptions
@@ -26,6 +30,15 @@ public sealed class AuditLogHashChainOptions
/// <summary>Whether the chaining job runs (default false).</summary>
public bool Enabled { get; set; }
/// <summary>
/// Chain the historical backlog on first enablement instead of starting the chain at the
/// enablement instant (default false — owner decision on ScadaBridge#107, 2026-11-13: no
/// backlog chaining). Only read while nothing has been chained yet: once the database holds
/// a chain-start anchor or any chained row, the decision is fixed and this value is ignored.
/// Set it the same on both central nodes; if they differ, whichever chains first decides.
/// </summary>
public bool ChainBacklog { get; set; }
/// <summary>Seconds between chaining ticks (default 60).</summary>
public int IntervalSeconds { get; set; } = 60;
@@ -92,7 +92,7 @@ public sealed class AuditLogHashChainService : IHostedService, IDisposable
var chain = scope.ServiceProvider.GetRequiredService<IAuditLogHashChain>();
for (var i = 0; i < options.MaxBatchesPerTick && !ct.IsCancellationRequested; i++)
{
var result = await chain.ChainPendingAsync(options.BatchSize, ct).ConfigureAwait(false);
var result = await chain.ChainPendingAsync(options.BatchSize, options.ChainBacklog, ct).ConfigureAwait(false);
total += result.RowsChained;
if (result.LockBusy || result.RowsChained == 0)
{
@@ -64,6 +64,8 @@ public static class AuditVerifyChainHelpers
output.WriteLine($"Purged through : {Num(root, "purgedThroughSeq", "-")}");
output.WriteLine($"Head sequence : {Num(root, "headSeq", "-")}");
output.WriteLine($"Unchained rows : {Num(root, "unchainedRows")}");
output.WriteLine($"Chain from : {Str(root, "chainFromUtc", "- (whole table in scope)")}");
output.WriteLine($"Pre-cutoff rows : {Num(root, "preCutoffRows")}");
if (root.TryGetProperty("firstBreak", out var brk) && brk.ValueKind == JsonValueKind.Object)
{
output.WriteLine("First break:");
@@ -31,10 +31,22 @@ public interface IAuditLogHashChain
/// Returns immediately with <see cref="AuditChainBatchResult.LockBusy"/> set when another
/// chainer holds the chain lock.
/// </summary>
/// <remarks>
/// <b>Chain start (ScadaBridge#139).</b> On a pristine database (nothing ever chained) and
/// with <paramref name="chainBacklog"/> false, the first pass records a chain-start anchor
/// at the database's current UTC time; from then on only rows dated at or after it are
/// chained, whatever either node passes later. With <paramref name="chainBacklog"/> true on
/// that first pass, no anchor is written and every row is chained, oldest month first. A
/// database that already has a chain and no anchor keeps chaining every row.
/// </remarks>
/// <param name="batchSize">Maximum rows to chain in this pass.</param>
/// <param name="chainBacklog">
/// Only consulted on a pristine database: true chains the historical backlog instead of
/// starting the chain now.
/// </param>
/// <param name="ct">Cancellation token.</param>
/// <returns>How many rows were chained, and whether the lock was busy.</returns>
Task<AuditChainBatchResult> ChainPendingAsync(int batchSize, CancellationToken ct = default);
Task<AuditChainBatchResult> ChainPendingAsync(int batchSize, bool chainBacklog = false, CancellationToken ct = default);
/// <summary>
/// Verifies one month's chain end to end and reports the first break.
@@ -46,6 +46,20 @@ public enum AuditChainBreakKind
/// deleted from the tail, or the head itself was altered.
/// </summary>
HeadMismatch,
/// <summary>
/// A chained row is dated before the chain cutoff (ScadaBridge#139). Rows before the cutoff
/// are never chained, so the row's <c>OccurredAtUtc</c> was moved back (to hide it among the
/// unprotected pre-cutoff rows) or the cutoff itself was moved forward.
/// </summary>
ChainedBeforeCutoff,
/// <summary>
/// The chain-start (cutoff) anchor is inconsistent (ScadaBridge#139): its stored hash does not
/// match its cutoff, it sits in the wrong month, or more than one anchor exists. The cutoff
/// that decides which rows are in scope cannot be trusted.
/// </summary>
CutoffAnchorInvalid,
}
/// <summary>
@@ -80,10 +94,20 @@ public sealed record AuditChainBreak(
/// </param>
/// <param name="HeadSeq">The last sequence number the chain head records, or null when the month has no head.</param>
/// <param name="UnchainedRows">
/// Rows in the month the chaining job has not reached yet (written since the last tick, or
/// written while chaining was disabled). They are not covered by the chain.
/// Rows in the month, at or after the chain cutoff, that the chaining job has not reached yet
/// (written since the last tick, or written while chaining was disabled). They are not covered
/// by the chain yet.
/// </param>
/// <param name="FirstBreak">The first break, or null when <see cref="Status"/> is not <see cref="AuditChainStatus.Broken"/>.</param>
/// <param name="PreCutoffRows">
/// Rows in the month dated before the chain cutoff (ScadaBridge#139). They are out of scope by
/// design — the chain covers only rows written from its first enablement on — so they are
/// never chained and never a break. Zero when the chain has no cutoff.
/// </param>
/// <param name="ChainFromUtc">
/// The chain cutoff: the instant the chain was first enabled without backlog chaining. Null
/// when the chain covers the whole table (legacy / <c>ChainBacklog</c> enablement).
/// </param>
public sealed record AuditChainVerificationResult(
string Partition,
AuditChainStatus Status,
@@ -92,7 +116,9 @@ public sealed record AuditChainVerificationResult(
long? PurgedThroughSeq,
long? HeadSeq,
long UnchainedRows,
AuditChainBreak? FirstBreak);
AuditChainBreak? FirstBreak,
long PreCutoffRows = 0,
DateTime? ChainFromUtc = null);
/// <summary>Outcome of one chaining pass over unchained AuditLog rows.</summary>
/// <param name="RowsChained">Rows linked into a chain by this pass.</param>
@@ -130,6 +130,35 @@ public static class AuditChainHasher
return buffer.ToArray();
}
/// <summary>
/// The <c>ChainSeq</c> of the chain-start (cutoff) anchor in <c>dbo.AuditLogChainTombstone</c>
/// (ScadaBridge#139). Real links start at 1, so slot 0 is "the link before the first row":
/// the anchor's <c>RowHash</c> is the genesis hash of the cutoff month's chain.
/// </summary>
public const long CutoffAnchorSeq = 0;
/// <summary>The fixed <c>EventId</c> the chain-start anchor row carries, so it is recognisable by eye.</summary>
public static readonly Guid CutoffAnchorEventId = new("5cadab1d-c4a1-4c0f-a000-000000000139");
private static readonly byte[] CutoffDomainTag = Encoding.ASCII.GetBytes("ScadaBridge.AuditLog.ChainCutoff.v1");
/// <summary>
/// Computes the chain-start anchor hash for a cutoff instant (ScadaBridge#139):
/// <c>SHA-256("ScadaBridge.AuditLog.ChainCutoff.v1" || YYYY-MM || ticks)</c>, big-endian like
/// the row form. It is both the anchor row's stored <c>RowHash</c> and the previous-hash the
/// first row of the cutoff month links to, so moving the cutoff breaks that month's chain.
/// </summary>
/// <param name="chainFromUtc">The cutoff: rows with an earlier <c>OccurredAtUtc</c> are never chained.</param>
/// <returns>The 32-byte anchor hash.</returns>
public static byte[] ComputeCutoffAnchorHash(DateTime chainFromUtc)
{
using var buffer = new MemoryStream(64);
buffer.Write(CutoffDomainTag);
WriteString(buffer, PartitionLabel(MonthOf(chainFromUtc)));
WriteInt64(buffer, chainFromUtc.Ticks);
return SHA256.HashData(buffer.ToArray());
}
private static void WriteInt64(Stream stream, long value)
{
Span<byte> bytes = stackalloc byte[8];
@@ -35,6 +35,15 @@ public sealed record AuditChainHeadState(
/// <b>reordered</b> rows (ChainSeq values swapped) → <see cref="AuditChainBreakKind.HashMismatch"/>
/// at the first moved row, because the sequence number is inside the hashed bytes.
/// </para>
/// <para>
/// <b>Chain start (cutoff, ScadaBridge#139).</b> When the chain was enabled with no backlog,
/// a link-0 anchor in the tombstones of the cutoff month is that month's genesis hash (instead
/// of 32 zero bytes), and no row before the cutoff is ever chained. So a chained row whose
/// <c>OccurredAtUtc</c> is before the cutoff is a break (<see cref="AuditChainBreakKind.ChainedBeforeCutoff"/>):
/// either the row's timestamp was moved back to hide it among the unprotected rows, or the
/// cutoff was moved forward. An anchor that does not hash to its own cutoff is reported as
/// <see cref="AuditChainBreakKind.CutoffAnchorInvalid"/> by the caller.
/// </para>
/// </remarks>
public sealed class AuditChainVerifier
{
@@ -43,6 +52,7 @@ public sealed class AuditChainVerifier
private readonly AuditChainHeadState? _head;
private readonly IReadOnlyDictionary<long, byte[]> _tombstones;
private readonly long _startAfter;
private readonly DateTime? _chainFromUtc;
private byte[] _previous;
private long _expected;
private long _verified;
@@ -52,11 +62,18 @@ public sealed class AuditChainVerifier
/// <summary>Initializes a verifier for one month.</summary>
/// <param name="monthStartUtc">The chain partition (first instant of the month, UTC).</param>
/// <param name="head">The month's chain head, or null when none exists.</param>
/// <param name="tombstones">Purge tombstones for the month, keyed by ChainSeq.</param>
/// <param name="tombstones">
/// Purge tombstones for the month, keyed by ChainSeq; a key of
/// <see cref="AuditChainHasher.CutoffAnchorSeq"/> is the chain-start anchor (the month's genesis).
/// </param>
/// <param name="chainFromUtc">The chain cutoff, or null when the whole table is in scope (legacy / backlog chaining).</param>
/// <param name="anchorProblem">When non-null, the chain-start anchor is invalid; verification fails with this detail.</param>
public AuditChainVerifier(
DateTime monthStartUtc,
AuditChainHeadState? head,
IReadOnlyDictionary<long, byte[]> tombstones)
IReadOnlyDictionary<long, byte[]> tombstones,
DateTime? chainFromUtc = null,
string? anchorProblem = null)
{
_monthStartUtc = monthStartUtc;
_partition = AuditChainHasher.PartitionLabel(monthStartUtc);
@@ -71,12 +88,23 @@ public sealed class AuditChainVerifier
else
{
_startAfter = 0;
_previous = AuditChainHasher.GenesisHash.ToArray();
_previous = _tombstones.TryGetValue(AuditChainHasher.CutoffAnchorSeq, out var cutoffAnchor)
? cutoffAnchor
: AuditChainHasher.GenesisHash.ToArray();
}
_expected = _startAfter + 1;
_chainFromUtc = chainFromUtc;
if (anchorProblem is not null)
{
Fail(AuditChainHasher.CutoffAnchorSeq, AuditChainHasher.CutoffAnchorEventId,
AuditChainBreakKind.CutoffAnchorInvalid, anchorProblem);
}
}
/// <summary>The chain cutoff this verification applies, or null.</summary>
public DateTime? ChainFromUtc => _chainFromUtc;
/// <summary>The chain partition label (<c>YYYY-MM</c>).</summary>
public string Partition => _partition;
@@ -106,6 +134,13 @@ public sealed class AuditChainVerifier
return true;
}
if (_chainFromUtc is { } chainFrom && row.OccurredAtUtc < chainFrom)
{
return Fail(seq, row.EventId, AuditChainBreakKind.ChainedBeforeCutoff,
$"Row at ChainSeq {seq} is dated {row.OccurredAtUtc:O}, before the chain cutoff {chainFrom:O}: "
+ "no row before the cutoff is ever chained, so its timestamp was moved back or the cutoff was moved forward.");
}
if (seq < _expected)
{
return Fail(seq, row.EventId, AuditChainBreakKind.DuplicateSequence,
@@ -147,9 +182,10 @@ public sealed class AuditChainVerifier
}
/// <summary>Finishes verification against the chain head.</summary>
/// <param name="unchainedRows">Rows in the month not yet chained (reported, never a break).</param>
/// <param name="unchainedRows">Rows in the month, at or after the cutoff, not yet chained (reported, never a break).</param>
/// <param name="preCutoffRows">Rows in the month before the chain cutoff: out of scope, never chained (reported, never a break).</param>
/// <returns>The verification result.</returns>
public AuditChainVerificationResult Complete(long unchainedRows)
public AuditChainVerificationResult Complete(long unchainedRows, long preCutoffRows = 0)
{
if (_break is null)
{
@@ -171,7 +207,7 @@ public sealed class AuditChainVerifier
}
else
{
return Result(AuditChainStatus.NotChained, unchainedRows);
return Result(AuditChainStatus.NotChained, unchainedRows, preCutoffRows);
}
}
else if (lastSeq < _head.LastSeq)
@@ -192,10 +228,10 @@ public sealed class AuditChainVerifier
}
}
return Result(_break is null ? AuditChainStatus.Intact : AuditChainStatus.Broken, unchainedRows);
return Result(_break is null ? AuditChainStatus.Intact : AuditChainStatus.Broken, unchainedRows, preCutoffRows);
}
private AuditChainVerificationResult Result(AuditChainStatus status, long unchainedRows)
private AuditChainVerificationResult Result(AuditChainStatus status, long unchainedRows, long preCutoffRows)
=> new(
_partition,
status,
@@ -204,7 +240,9 @@ public sealed class AuditChainVerifier
_head?.PurgedThroughSeq,
_head?.LastSeq,
unchainedRows,
_break);
_break,
preCutoffRows,
_chainFromUtc);
private bool Fail(long seq, Guid? eventId, AuditChainBreakKind kind, string detail)
{
@@ -39,6 +39,19 @@ public static class AuditChainLock
/// <c>OccurredAtUtc</c> order, which is why late-arriving site rows need no special case.
/// </para>
/// <para>
/// <b>Chain start (ScadaBridge#139).</b> Unless the caller asks for backlog chaining, the first
/// pass on a pristine database (no chain head, no chained row) records a <b>chain-start
/// anchor</b> and only rows dated at or after it are ever chained; the historical backlog stays
/// unchained and out of scope. The anchor is one row in <c>dbo.AuditLogChainTombstone</c> at
/// <c>ChainSeq = 0</c> of the cutoff month: <c>PurgedAtUtc</c> holds the cutoff (the database's
/// <c>SYSUTCDATETIME()</c> at first enablement, so both central nodes see the same instant
/// whatever their own clocks say), <c>RowHash</c> holds
/// <see cref="AuditChainHasher.ComputeCutoffAnchorHash"/> of it, and that hash is the genesis
/// the cutoff month's first row links to. It is written once, under the exclusive chain lock,
/// and never updated; the partition-switch purge keeps it. A database that already has a
/// chain (heads or chained rows) and no anchor keeps chaining every row, as before.
/// </para>
/// <para>
/// <b>Append-only.</b> The only AuditLog columns this class writes are <c>ChainSeq</c> and
/// <c>RowHash</c>, and only while they are NULL. In production that UPDATE runs under the
/// <c>scadabridge_audit_chainer</c> role, which holds a COLUMN-level UPDATE grant on just
@@ -72,11 +85,17 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore
/// </summary>
public int VerifyCommandTimeoutSeconds { get; init; } = 300;
/// <summary>
/// Test seam: the cutoff a new chain-start anchor records. Null (production) uses the
/// database's <c>SYSUTCDATETIME()</c> so both central nodes agree regardless of node clocks.
/// </summary>
internal Func<DateTime>? AnchorClock { get; init; }
/// <summary>How long verification waits for an in-flight chaining/purge pass to release the lock.</summary>
public TimeSpan VerifyLockTimeout { get; init; } = TimeSpan.FromSeconds(60);
/// <inheritdoc />
public async Task<AuditChainBatchResult> ChainPendingAsync(int batchSize, CancellationToken ct = default)
public async Task<AuditChainBatchResult> ChainPendingAsync(int batchSize, bool chainBacklog = false, CancellationToken ct = default)
{
if (batchSize <= 0)
{
@@ -95,12 +114,20 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore
return new AuditChainBatchResult(0, LockBusy: true);
}
// Oldest month that still has unchained rows. One month per pass keeps the
// Chain start (ScadaBridge#139): the cutoff, recorded on the first pass.
var cutoff = await ReadChainerCutoffAsync(conn, tx, ct).ConfigureAwait(false);
if (cutoff is null && !chainBacklog && await IsPristineAsync(conn, tx, ct).ConfigureAwait(false))
{
cutoff = await CreateCutoffAnchorAsync(conn, tx, ct).ConfigureAwait(false);
}
// Oldest month that still has unchained rows in scope. One month per pass keeps the
// candidate read partition-eliminated and index-ordered (no cross-partition sort).
DateTime? oldest;
await using (var cmd = Command(conn, tx,
"SELECT TOP (1) OccurredAtUtc FROM dbo.AuditLog WHERE ChainSeq IS NULL ORDER BY OccurredAtUtc;"))
"SELECT TOP (1) OccurredAtUtc FROM dbo.AuditLog WHERE ChainSeq IS NULL AND OccurredAtUtc >= @chainFrom ORDER BY OccurredAtUtc;"))
{
AddChainFrom(cmd, cutoff?.ChainFromUtc);
var scalar = await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false);
oldest = scalar is DateTime dt ? DateTime.SpecifyKind(dt, DateTimeKind.Utc) : null;
}
@@ -116,10 +143,12 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore
await using (var cmd = Command(conn, tx,
$"SELECT TOP (@batch) {RowColumns} FROM dbo.AuditLog " +
"WHERE ChainSeq IS NULL AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd " +
"AND OccurredAtUtc >= @chainFrom " +
"ORDER BY OccurredAtUtc, EventId;"))
{
AddInt(cmd, "@batch", batchSize);
AddMonthRange(cmd, month);
AddChainFrom(cmd, cutoff?.ChainFromUtc);
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
while (await reader.ReadAsync(ct).ConfigureAwait(false))
{
@@ -142,6 +171,13 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore
}
}
// The cutoff month's chain starts from the anchor, not from 32 zero bytes, so the
// cutoff is inside every later link of that month.
if (headSeq is null && cutoff is not null && cutoff.Month == month)
{
previous = cutoff.AnchorHash.ToArray();
}
var seq = headSeq ?? 0;
var links = new List<(AuditChainRow Row, long Seq, byte[] Hash)>(candidates.Count);
foreach (var row in candidates)
@@ -222,6 +258,8 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore
}
}
var cutoff = await ReadVerifierCutoffAsync(conn, tx, ct).ConfigureAwait(false);
var tombstones = new Dictionary<long, byte[]>();
await using (var cmd = Command(conn, tx,
"SELECT ChainSeq, RowHash FROM dbo.AuditLogChainTombstone WHERE ChainMonth = @month;"))
@@ -234,16 +272,25 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore
}
}
long unchained;
// Unchained rows split at the cutoff: pending (in scope, not reached yet) versus
// pre-cutoff (out of scope by design, never chained).
long unchained = 0, preCutoff = 0;
await using (var cmd = Command(conn, tx,
"SELECT COUNT_BIG(*) FROM dbo.AuditLog WHERE ChainSeq IS NULL AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd;"))
"SELECT COUNT_BIG(CASE WHEN OccurredAtUtc >= @chainFrom THEN 1 END), COUNT_BIG(CASE WHEN OccurredAtUtc < @chainFrom THEN 1 END) " +
"FROM dbo.AuditLog WHERE ChainSeq IS NULL AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd;"))
{
cmd.CommandTimeout = VerifyCommandTimeoutSeconds;
AddMonthRange(cmd, month);
unchained = Convert.ToInt64(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false));
AddChainFrom(cmd, cutoff.ChainFromUtc);
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
if (await reader.ReadAsync(ct).ConfigureAwait(false))
{
unchained = reader.GetInt64(0);
preCutoff = reader.GetInt64(1);
}
}
var verifier = new AuditChainVerifier(month, head, tombstones);
var verifier = new AuditChainVerifier(month, head, tombstones, cutoff.ChainFromUtc, cutoff.Problem);
await using (var cmd = Command(conn, tx,
$"SELECT {RowColumns}, ChainSeq, RowHash FROM dbo.AuditLog " +
"WHERE ChainSeq IS NOT NULL AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd " +
@@ -262,7 +309,7 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore
}
await tx.CommitAsync(ct).ConfigureAwait(false);
return verifier.Complete(unchained);
return verifier.Complete(unchained, preCutoff);
}
finally
{
@@ -371,7 +418,7 @@ SELECT h.ChainMonth, h.LastSeq, h.LastHash, h.PurgedThroughSeq, h.PurgedThroughH
WHERE a.ChainSeq IS NOT NULL
AND a.OccurredAtUtc >= CAST(h.ChainMonth AS datetime2(7))
AND a.OccurredAtUtc < DATEADD(MONTH, 1, CAST(h.ChainMonth AS datetime2(7)))) AS ChainedRows,
(SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone t WHERE t.ChainMonth = h.ChainMonth) AS Tombstones
(SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone t WHERE t.ChainMonth = h.ChainMonth AND t.ChainSeq > 0) AS Tombstones
FROM dbo.AuditLogChainHead h
ORDER BY h.ChainMonth;");
cmd.CommandTimeout = VerifyCommandTimeoutSeconds;
@@ -418,6 +465,135 @@ ORDER BY h.ChainMonth;");
return new AuditChainHeadComparer.StoredLink(rowHash, tombstoneHash);
}
/// <summary>The chain-start anchor as the chainer uses it (ScadaBridge#139).</summary>
/// <param name="ChainFromUtc">The cutoff.</param>
/// <param name="Month">The cutoff month (the anchor's <c>ChainMonth</c>).</param>
/// <param name="AnchorHash">The anchor's stored hash: the cutoff month's genesis.</param>
private sealed record ChainCutoff(DateTime ChainFromUtc, DateTime Month, byte[] AnchorHash);
/// <summary>The cutoff as verification sees it (ScadaBridge#139).</summary>
/// <param name="ChainFromUtc">The cutoff, or null when the chain covers the whole table.</param>
/// <param name="Problem">Why the anchor cannot be trusted, or null.</param>
private sealed record VerifierCutoff(DateTime? ChainFromUtc, string? Problem);
/// <summary>
/// Reads the chain-start anchor for the chaining pass. It can only sit in a month at or
/// before the earliest chain head (it is written before anything is chained, and nothing
/// before it ever is), so the read is bounded to those months rather than scanning every
/// per-channel purge tombstone each tick. Verification reads them all.
/// </summary>
private static async Task<ChainCutoff?> ReadChainerCutoffAsync(DbConnection conn, DbTransaction tx, CancellationToken ct)
{
await using var cmd = Command(conn, tx,
"SELECT TOP (1) ChainMonth, RowHash, PurgedAtUtc FROM dbo.AuditLogChainTombstone " +
"WHERE ChainSeq = 0 AND ChainMonth <= ISNULL((SELECT MIN(ChainMonth) FROM dbo.AuditLogChainHead), '9999-12-01') " +
"ORDER BY PurgedAtUtc, ChainMonth;");
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
if (!await reader.ReadAsync(ct).ConfigureAwait(false))
{
return null;
}
return new ChainCutoff(
DateTime.SpecifyKind(reader.GetDateTime(2), DateTimeKind.Utc),
DateTime.SpecifyKind(reader.GetDateTime(0), DateTimeKind.Utc),
(byte[])reader.GetValue(1));
}
/// <summary>
/// Reads every chain-start anchor and checks it: exactly one, in its cutoff's month, hashing
/// to its own cutoff. The earliest anchor's cutoff is used either way, matching the chainer.
/// </summary>
private async Task<VerifierCutoff> ReadVerifierCutoffAsync(DbConnection conn, DbTransaction tx, CancellationToken ct)
{
var anchors = new List<(DateTime Month, byte[] Hash, DateTime ChainFrom)>();
await using (var cmd = Command(conn, tx,
"SELECT ChainMonth, RowHash, PurgedAtUtc FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0 ORDER BY PurgedAtUtc, ChainMonth;"))
{
cmd.CommandTimeout = VerifyCommandTimeoutSeconds;
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
while (await reader.ReadAsync(ct).ConfigureAwait(false))
{
anchors.Add((
DateTime.SpecifyKind(reader.GetDateTime(0), DateTimeKind.Utc),
(byte[])reader.GetValue(1),
DateTime.SpecifyKind(reader.GetDateTime(2), DateTimeKind.Utc)));
}
}
if (anchors.Count == 0)
{
return new VerifierCutoff(null, null);
}
var (anchorMonth, hash, chainFrom) = anchors[0];
string? problem = null;
if (anchors.Count > 1)
{
problem = $"{anchors.Count} chain-start anchors exist (cutoffs {string.Join(", ", anchors.Select(a => a.ChainFrom.ToString("O", System.Globalization.CultureInfo.InvariantCulture)))}); "
+ "exactly one is ever written.";
}
else if (anchorMonth != AuditChainHasher.MonthOf(chainFrom))
{
problem = $"The chain-start anchor is filed under {AuditChainHasher.PartitionLabel(anchorMonth)} but its cutoff {chainFrom:O} is in another month.";
}
else if (!AuditChainHasher.ComputeCutoffAnchorHash(chainFrom).AsSpan().SequenceEqual(hash))
{
problem = $"The chain-start anchor's hash does not match its cutoff {chainFrom:O}: the cutoff or the anchor was altered.";
}
return new VerifierCutoff(chainFrom, problem);
}
/// <summary>True when nothing has ever been chained: no chain head and no chained row.</summary>
private static async Task<bool> IsPristineAsync(DbConnection conn, DbTransaction tx, CancellationToken ct)
{
await using var cmd = Command(conn, tx,
"SELECT CASE WHEN EXISTS (SELECT 1 FROM dbo.AuditLogChainHead) " +
"OR EXISTS (SELECT 1 FROM dbo.AuditLog WHERE ChainSeq IS NOT NULL) THEN 0 ELSE 1 END;");
return Convert.ToInt32(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false)) == 1;
}
/// <summary>
/// Writes the chain-start anchor at the database's current UTC time (ScadaBridge#139). Runs
/// inside the chaining transaction, under the exclusive chain lock, so exactly one node
/// writes it and the peer's next pass reads it.
/// </summary>
private async Task<ChainCutoff> CreateCutoffAnchorAsync(DbConnection conn, DbTransaction tx, CancellationToken ct)
{
DateTime chainFrom;
if (AnchorClock is { } clock)
{
chainFrom = DateTime.SpecifyKind(clock(), DateTimeKind.Utc);
}
else
{
await using var cmd = Command(conn, tx, "SELECT SYSUTCDATETIME();");
chainFrom = DateTime.SpecifyKind((DateTime)(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false))!, DateTimeKind.Utc);
}
var month = AuditChainHasher.MonthOf(chainFrom);
var hash = AuditChainHasher.ComputeCutoffAnchorHash(chainFrom);
await using (var cmd = Command(conn, tx,
"INSERT INTO dbo.AuditLogChainTombstone (ChainMonth, ChainSeq, RowHash, EventId, PurgedAtUtc) " +
"VALUES (@month, 0, @hash, @eventId, @chainFrom);"))
{
AddDate(cmd, "@month", month);
cmd.Parameters.Add(new SqlParameter("@hash", SqlDbType.Binary, AuditChainHasher.HashLength) { Value = hash });
cmd.Parameters.Add(new SqlParameter("@eventId", SqlDbType.UniqueIdentifier) { Value = AuditChainHasher.CutoffAnchorEventId });
cmd.Parameters.Add(new SqlParameter("@chainFrom", SqlDbType.DateTime2) { Scale = 7, Value = chainFrom });
await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
}
_logger.LogInformation(
"AuditLog hash chain started at {ChainFrom:O}: only rows dated from then on are chained; earlier rows stay unchained and out of scope (ScadaBridge#139).",
chainFrom);
return new ChainCutoff(chainFrom, month, hash);
}
private static void AddChainFrom(DbCommand cmd, DateTime? chainFromUtc)
=> cmd.Parameters.Add(new SqlParameter("@chainFrom", SqlDbType.DateTime2) { Scale = 7, Value = chainFromUtc ?? DateTime.MinValue });
/// <summary>
/// Takes the chain application lock inside <paramref name="tx"/> (released at commit /
/// rollback). Returns false when the lock could not be granted within <paramref name="timeout"/>.
@@ -378,6 +378,7 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive
// Tombstones per month, then one streaming verifier per chain month.
var tombstones = new Dictionary<DateTime, Dictionary<long, byte[]>>();
long tombstonesRead = 0;
DateTime? chainFrom = null;
await using (var input = File.OpenRead(tombstonePath))
{
await foreach (var t in AuditArchiveParquet.ReadTombstonesAsync(input, ct).ConfigureAwait(false))
@@ -389,12 +390,21 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive
byMonth[t.ChainSeq] = t.RowHash;
tombstonesRead++;
if (t.ChainSeq == AuditChainHasher.CutoffAnchorSeq)
{
// The chain-start anchor (ScadaBridge#139) travels with the archive of its month.
chainFrom = chainFrom is { } seen && seen <= t.PurgedAtUtc ? seen : t.PurgedAtUtc;
}
}
}
// Otherwise the cutoff comes from the database's anchor, which outlives every purge.
chainFrom ??= await ReadChainFromAsync(ct).ConfigureAwait(false);
var states = manifest.Chains.ToDictionary(c => AuditArchiveParquet.ParseMonth(c.Partition));
var verifiers = new SortedDictionary<DateTime, AuditChainVerifier>();
var unchained = new Dictionary<DateTime, long>();
var preCutoff = new Dictionary<DateTime, long>();
AuditChainVerifier VerifierFor(DateTime month)
{
if (!verifiers.TryGetValue(month, out var verifier))
@@ -406,7 +416,7 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive
s.PriorPurgedThroughSeq,
s.PriorPurgedThroughHash is null ? null : Convert.FromHexString(s.PriorPurgedThroughHash))
: null;
verifier = new AuditChainVerifier(month, head, tombstones.GetValueOrDefault(month) ?? []);
verifier = new AuditChainVerifier(month, head, tombstones.GetValueOrDefault(month) ?? [], chainFrom);
verifiers[month] = verifier;
}
@@ -427,7 +437,8 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive
var month = AuditChainHasher.MonthOf(row.OccurredAtUtc);
if (row.ChainSeq is null)
{
unchained[month] = unchained.GetValueOrDefault(month) + 1;
var bucket = chainFrom is { } cutoff && row.OccurredAtUtc < cutoff ? preCutoff : unchained;
bucket[month] = bucket.GetValueOrDefault(month) + 1;
VerifierFor(month);
continue;
}
@@ -450,7 +461,7 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive
var results = new List<AuditArchiveChainVerification>();
foreach (var (month, verifier) in verifiers)
{
var chain = verifier.Complete(unchained.GetValueOrDefault(month));
var chain = verifier.Complete(unchained.GetValueOrDefault(month), preCutoff.GetValueOrDefault(month));
var (anchor, detail) = CompareAnchor(states.GetValueOrDefault(month), anchors.GetValueOrDefault(month));
results.Add(new AuditArchiveChainVerification(chain, anchor, detail));
}
@@ -509,6 +520,28 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive
$"The purge anchor (ChainSeq {database.PurgedSeq}) is before this archive's chain end ({archived.LastSeq}).");
}
/// <summary>The database's chain cutoff (ScadaBridge#139), or null when the chain has none.</summary>
private async Task<DateTime?> ReadChainFromAsync(CancellationToken ct)
{
var conn = _context.Database.GetDbConnection();
var openedHere = await AuditLogHashChain.OpenAsync(conn, ct).ConfigureAwait(false);
try
{
await using var cmd = conn.CreateCommand();
cmd.CommandText = "SELECT MIN(PurgedAtUtc) FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;";
return await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false) is DateTime dt
? DateTime.SpecifyKind(dt, DateTimeKind.Utc)
: null;
}
finally
{
if (openedHere)
{
await conn.CloseAsync().ConfigureAwait(false);
}
}
}
private async Task<Dictionary<DateTime, (bool HeadExists, long? PurgedSeq, byte[]? PurgedHash)>> ReadAnchorsAsync(
IEnumerable<DateTime> months, CancellationToken ct)
{
@@ -634,12 +634,15 @@ VALUES
-- 3b. Hash chain (ScadaBridge#41): the months this partition held are gone as a
-- whole, so record each month's current chain end as its checkpoint anchor
-- (verification resumes after it; a late-arriving row chains on from it)
-- and drop the per-channel tombstones the anchor now subsumes.
-- and drop the per-channel tombstones the anchor now subsumes. The chain-start
-- anchor (ChainSeq 0, ScadaBridge#139) is NOT a purge tombstone: it records the
-- cutoff for the life of the database, so it stays.
UPDATE dbo.AuditLogChainHead
SET PurgedThroughSeq = LastSeq, PurgedThroughHash = LastHash, UpdatedAtUtc = SYSUTCDATETIME()
WHERE $partition.pf_AuditLog_Month(CAST(ChainMonth AS datetime2(7))) = @partitionNumber;
DELETE FROM dbo.AuditLogChainTombstone
WHERE $partition.pf_AuditLog_Month(CAST(ChainMonth AS datetime2(7))) = @partitionNumber;
WHERE $partition.pf_AuditLog_Month(CAST(ChainMonth AS datetime2(7))) = @partitionNumber
AND ChainSeq > 0;
-- 4. Drop staging — the rows are discarded here. This is the purge.
DROP TABLE dbo.[{stagingTableName}];