feat(audit): hash-chain cutoff - chain only rows written from first enablement, no backlog (#139)
Owner decision on #107 (2026-11-13): enabling AuditLog:HashChain must not chain the historical backlog. main chained every unchained row oldest-first. - The first chaining pass on a pristine database (no head, no chained row) writes a chain-start anchor: AuditLogChainTombstone (cutoff month, ChainSeq 0), PurgedAtUtc = DB SYSUTCDATETIME(), RowHash = SHA-256("ScadaBridge.AuditLog.ChainCutoff.v1" || YYYY-MM || ticks). Written once under the exclusive chain lock, so both centrals agree whatever their clocks/config. No migration: existing table + grants. - Only rows with OccurredAtUtc >= cutoff are chained. The anchor hash is the cutoff month's genesis, binding the cutoff into the chain. - AuditLog:HashChain:ChainBacklog (default false) restores chain-everything, read only on the very first enablement. Legacy chains (heads, no anchor) keep chaining everything. - verify-chain splits unchained rows into UnchainedRows vs PreCutoffRows (+ ChainFromUtc); new breaks ChainedBeforeCutoff (row re-dated before the cutoff, or cutoff moved forward) and CutoffAnchorInvalid (anchor hash / month / multiplicity). Archive verify uses the anchor from its own tombstone file or the DB. - Partition switch-out keeps the anchor (deletes ChainSeq > 0 only); the head store / export / verify-heads tombstone count excludes it. - CLI verify-chain table prints Chain from + Pre-cutoff rows. Tests: AuditLogHashChainCutoffTests (7, MSSQL), 6 verifier unit tests, service/CLI/management wire tests; 9 mutations all killed. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
@@ -15,10 +15,14 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
/// them chains at a time, so a dual-active pair still produces one gap-free chain per month.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Enabling it on an existing database chains the historical backlog oldest month first,
|
||||
/// <see cref="BatchSize"/> × <see cref="MaxBatchesPerTick"/> rows per tick, so the first
|
||||
/// ticks run long. Rows chained late are only protected from the moment they are chained.
|
||||
/// Run <c>audit backfill-source-node</c> BEFORE enabling: a chained row's SourceNode is frozen.
|
||||
/// <b>Chain start (ScadaBridge#139).</b> Enabling it on an existing database does NOT chain the
|
||||
/// historical backlog: the first chaining pass records a chain-start anchor in the database at
|
||||
/// the database's current UTC time, and only rows dated from then on are chained. Older rows
|
||||
/// stay unchained and are reported by <c>audit verify-chain</c> as pre-cutoff rows, never as a
|
||||
/// break. The anchor is written once and is the same for both central nodes. Set
|
||||
/// <see cref="ChainBacklog"/> to chain the backlog instead (only effective on the very first
|
||||
/// enablement). Run <c>audit backfill-source-node</c> BEFORE enabling either way: a chained
|
||||
/// row's SourceNode is frozen.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class AuditLogHashChainOptions
|
||||
@@ -26,6 +30,15 @@ public sealed class AuditLogHashChainOptions
|
||||
/// <summary>Whether the chaining job runs (default false).</summary>
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Chain the historical backlog on first enablement instead of starting the chain at the
|
||||
/// enablement instant (default false — owner decision on ScadaBridge#107, 2026-11-13: no
|
||||
/// backlog chaining). Only read while nothing has been chained yet: once the database holds
|
||||
/// a chain-start anchor or any chained row, the decision is fixed and this value is ignored.
|
||||
/// Set it the same on both central nodes; if they differ, whichever chains first decides.
|
||||
/// </summary>
|
||||
public bool ChainBacklog { get; set; }
|
||||
|
||||
/// <summary>Seconds between chaining ticks (default 60).</summary>
|
||||
public int IntervalSeconds { get; set; } = 60;
|
||||
|
||||
|
||||
@@ -92,7 +92,7 @@ public sealed class AuditLogHashChainService : IHostedService, IDisposable
|
||||
var chain = scope.ServiceProvider.GetRequiredService<IAuditLogHashChain>();
|
||||
for (var i = 0; i < options.MaxBatchesPerTick && !ct.IsCancellationRequested; i++)
|
||||
{
|
||||
var result = await chain.ChainPendingAsync(options.BatchSize, ct).ConfigureAwait(false);
|
||||
var result = await chain.ChainPendingAsync(options.BatchSize, options.ChainBacklog, ct).ConfigureAwait(false);
|
||||
total += result.RowsChained;
|
||||
if (result.LockBusy || result.RowsChained == 0)
|
||||
{
|
||||
|
||||
@@ -64,6 +64,8 @@ public static class AuditVerifyChainHelpers
|
||||
output.WriteLine($"Purged through : {Num(root, "purgedThroughSeq", "-")}");
|
||||
output.WriteLine($"Head sequence : {Num(root, "headSeq", "-")}");
|
||||
output.WriteLine($"Unchained rows : {Num(root, "unchainedRows")}");
|
||||
output.WriteLine($"Chain from : {Str(root, "chainFromUtc", "- (whole table in scope)")}");
|
||||
output.WriteLine($"Pre-cutoff rows : {Num(root, "preCutoffRows")}");
|
||||
if (root.TryGetProperty("firstBreak", out var brk) && brk.ValueKind == JsonValueKind.Object)
|
||||
{
|
||||
output.WriteLine("First break:");
|
||||
|
||||
@@ -31,10 +31,22 @@ public interface IAuditLogHashChain
|
||||
/// Returns immediately with <see cref="AuditChainBatchResult.LockBusy"/> set when another
|
||||
/// chainer holds the chain lock.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <b>Chain start (ScadaBridge#139).</b> On a pristine database (nothing ever chained) and
|
||||
/// with <paramref name="chainBacklog"/> false, the first pass records a chain-start anchor
|
||||
/// at the database's current UTC time; from then on only rows dated at or after it are
|
||||
/// chained, whatever either node passes later. With <paramref name="chainBacklog"/> true on
|
||||
/// that first pass, no anchor is written and every row is chained, oldest month first. A
|
||||
/// database that already has a chain and no anchor keeps chaining every row.
|
||||
/// </remarks>
|
||||
/// <param name="batchSize">Maximum rows to chain in this pass.</param>
|
||||
/// <param name="chainBacklog">
|
||||
/// Only consulted on a pristine database: true chains the historical backlog instead of
|
||||
/// starting the chain now.
|
||||
/// </param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>How many rows were chained, and whether the lock was busy.</returns>
|
||||
Task<AuditChainBatchResult> ChainPendingAsync(int batchSize, CancellationToken ct = default);
|
||||
Task<AuditChainBatchResult> ChainPendingAsync(int batchSize, bool chainBacklog = false, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Verifies one month's chain end to end and reports the first break.
|
||||
|
||||
@@ -46,6 +46,20 @@ public enum AuditChainBreakKind
|
||||
/// deleted from the tail, or the head itself was altered.
|
||||
/// </summary>
|
||||
HeadMismatch,
|
||||
|
||||
/// <summary>
|
||||
/// A chained row is dated before the chain cutoff (ScadaBridge#139). Rows before the cutoff
|
||||
/// are never chained, so the row's <c>OccurredAtUtc</c> was moved back (to hide it among the
|
||||
/// unprotected pre-cutoff rows) or the cutoff itself was moved forward.
|
||||
/// </summary>
|
||||
ChainedBeforeCutoff,
|
||||
|
||||
/// <summary>
|
||||
/// The chain-start (cutoff) anchor is inconsistent (ScadaBridge#139): its stored hash does not
|
||||
/// match its cutoff, it sits in the wrong month, or more than one anchor exists. The cutoff
|
||||
/// that decides which rows are in scope cannot be trusted.
|
||||
/// </summary>
|
||||
CutoffAnchorInvalid,
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -80,10 +94,20 @@ public sealed record AuditChainBreak(
|
||||
/// </param>
|
||||
/// <param name="HeadSeq">The last sequence number the chain head records, or null when the month has no head.</param>
|
||||
/// <param name="UnchainedRows">
|
||||
/// Rows in the month the chaining job has not reached yet (written since the last tick, or
|
||||
/// written while chaining was disabled). They are not covered by the chain.
|
||||
/// Rows in the month, at or after the chain cutoff, that the chaining job has not reached yet
|
||||
/// (written since the last tick, or written while chaining was disabled). They are not covered
|
||||
/// by the chain yet.
|
||||
/// </param>
|
||||
/// <param name="FirstBreak">The first break, or null when <see cref="Status"/> is not <see cref="AuditChainStatus.Broken"/>.</param>
|
||||
/// <param name="PreCutoffRows">
|
||||
/// Rows in the month dated before the chain cutoff (ScadaBridge#139). They are out of scope by
|
||||
/// design — the chain covers only rows written from its first enablement on — so they are
|
||||
/// never chained and never a break. Zero when the chain has no cutoff.
|
||||
/// </param>
|
||||
/// <param name="ChainFromUtc">
|
||||
/// The chain cutoff: the instant the chain was first enabled without backlog chaining. Null
|
||||
/// when the chain covers the whole table (legacy / <c>ChainBacklog</c> enablement).
|
||||
/// </param>
|
||||
public sealed record AuditChainVerificationResult(
|
||||
string Partition,
|
||||
AuditChainStatus Status,
|
||||
@@ -92,7 +116,9 @@ public sealed record AuditChainVerificationResult(
|
||||
long? PurgedThroughSeq,
|
||||
long? HeadSeq,
|
||||
long UnchainedRows,
|
||||
AuditChainBreak? FirstBreak);
|
||||
AuditChainBreak? FirstBreak,
|
||||
long PreCutoffRows = 0,
|
||||
DateTime? ChainFromUtc = null);
|
||||
|
||||
/// <summary>Outcome of one chaining pass over unchained AuditLog rows.</summary>
|
||||
/// <param name="RowsChained">Rows linked into a chain by this pass.</param>
|
||||
|
||||
@@ -130,6 +130,35 @@ public static class AuditChainHasher
|
||||
return buffer.ToArray();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The <c>ChainSeq</c> of the chain-start (cutoff) anchor in <c>dbo.AuditLogChainTombstone</c>
|
||||
/// (ScadaBridge#139). Real links start at 1, so slot 0 is "the link before the first row":
|
||||
/// the anchor's <c>RowHash</c> is the genesis hash of the cutoff month's chain.
|
||||
/// </summary>
|
||||
public const long CutoffAnchorSeq = 0;
|
||||
|
||||
/// <summary>The fixed <c>EventId</c> the chain-start anchor row carries, so it is recognisable by eye.</summary>
|
||||
public static readonly Guid CutoffAnchorEventId = new("5cadab1d-c4a1-4c0f-a000-000000000139");
|
||||
|
||||
private static readonly byte[] CutoffDomainTag = Encoding.ASCII.GetBytes("ScadaBridge.AuditLog.ChainCutoff.v1");
|
||||
|
||||
/// <summary>
|
||||
/// Computes the chain-start anchor hash for a cutoff instant (ScadaBridge#139):
|
||||
/// <c>SHA-256("ScadaBridge.AuditLog.ChainCutoff.v1" || YYYY-MM || ticks)</c>, big-endian like
|
||||
/// the row form. It is both the anchor row's stored <c>RowHash</c> and the previous-hash the
|
||||
/// first row of the cutoff month links to, so moving the cutoff breaks that month's chain.
|
||||
/// </summary>
|
||||
/// <param name="chainFromUtc">The cutoff: rows with an earlier <c>OccurredAtUtc</c> are never chained.</param>
|
||||
/// <returns>The 32-byte anchor hash.</returns>
|
||||
public static byte[] ComputeCutoffAnchorHash(DateTime chainFromUtc)
|
||||
{
|
||||
using var buffer = new MemoryStream(64);
|
||||
buffer.Write(CutoffDomainTag);
|
||||
WriteString(buffer, PartitionLabel(MonthOf(chainFromUtc)));
|
||||
WriteInt64(buffer, chainFromUtc.Ticks);
|
||||
return SHA256.HashData(buffer.ToArray());
|
||||
}
|
||||
|
||||
private static void WriteInt64(Stream stream, long value)
|
||||
{
|
||||
Span<byte> bytes = stackalloc byte[8];
|
||||
|
||||
@@ -35,6 +35,15 @@ public sealed record AuditChainHeadState(
|
||||
/// <b>reordered</b> rows (ChainSeq values swapped) → <see cref="AuditChainBreakKind.HashMismatch"/>
|
||||
/// at the first moved row, because the sequence number is inside the hashed bytes.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Chain start (cutoff, ScadaBridge#139).</b> When the chain was enabled with no backlog,
|
||||
/// a link-0 anchor in the tombstones of the cutoff month is that month's genesis hash (instead
|
||||
/// of 32 zero bytes), and no row before the cutoff is ever chained. So a chained row whose
|
||||
/// <c>OccurredAtUtc</c> is before the cutoff is a break (<see cref="AuditChainBreakKind.ChainedBeforeCutoff"/>):
|
||||
/// either the row's timestamp was moved back to hide it among the unprotected rows, or the
|
||||
/// cutoff was moved forward. An anchor that does not hash to its own cutoff is reported as
|
||||
/// <see cref="AuditChainBreakKind.CutoffAnchorInvalid"/> by the caller.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class AuditChainVerifier
|
||||
{
|
||||
@@ -43,6 +52,7 @@ public sealed class AuditChainVerifier
|
||||
private readonly AuditChainHeadState? _head;
|
||||
private readonly IReadOnlyDictionary<long, byte[]> _tombstones;
|
||||
private readonly long _startAfter;
|
||||
private readonly DateTime? _chainFromUtc;
|
||||
private byte[] _previous;
|
||||
private long _expected;
|
||||
private long _verified;
|
||||
@@ -52,11 +62,18 @@ public sealed class AuditChainVerifier
|
||||
/// <summary>Initializes a verifier for one month.</summary>
|
||||
/// <param name="monthStartUtc">The chain partition (first instant of the month, UTC).</param>
|
||||
/// <param name="head">The month's chain head, or null when none exists.</param>
|
||||
/// <param name="tombstones">Purge tombstones for the month, keyed by ChainSeq.</param>
|
||||
/// <param name="tombstones">
|
||||
/// Purge tombstones for the month, keyed by ChainSeq; a key of
|
||||
/// <see cref="AuditChainHasher.CutoffAnchorSeq"/> is the chain-start anchor (the month's genesis).
|
||||
/// </param>
|
||||
/// <param name="chainFromUtc">The chain cutoff, or null when the whole table is in scope (legacy / backlog chaining).</param>
|
||||
/// <param name="anchorProblem">When non-null, the chain-start anchor is invalid; verification fails with this detail.</param>
|
||||
public AuditChainVerifier(
|
||||
DateTime monthStartUtc,
|
||||
AuditChainHeadState? head,
|
||||
IReadOnlyDictionary<long, byte[]> tombstones)
|
||||
IReadOnlyDictionary<long, byte[]> tombstones,
|
||||
DateTime? chainFromUtc = null,
|
||||
string? anchorProblem = null)
|
||||
{
|
||||
_monthStartUtc = monthStartUtc;
|
||||
_partition = AuditChainHasher.PartitionLabel(monthStartUtc);
|
||||
@@ -71,12 +88,23 @@ public sealed class AuditChainVerifier
|
||||
else
|
||||
{
|
||||
_startAfter = 0;
|
||||
_previous = AuditChainHasher.GenesisHash.ToArray();
|
||||
_previous = _tombstones.TryGetValue(AuditChainHasher.CutoffAnchorSeq, out var cutoffAnchor)
|
||||
? cutoffAnchor
|
||||
: AuditChainHasher.GenesisHash.ToArray();
|
||||
}
|
||||
|
||||
_expected = _startAfter + 1;
|
||||
_chainFromUtc = chainFromUtc;
|
||||
if (anchorProblem is not null)
|
||||
{
|
||||
Fail(AuditChainHasher.CutoffAnchorSeq, AuditChainHasher.CutoffAnchorEventId,
|
||||
AuditChainBreakKind.CutoffAnchorInvalid, anchorProblem);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The chain cutoff this verification applies, or null.</summary>
|
||||
public DateTime? ChainFromUtc => _chainFromUtc;
|
||||
|
||||
/// <summary>The chain partition label (<c>YYYY-MM</c>).</summary>
|
||||
public string Partition => _partition;
|
||||
|
||||
@@ -106,6 +134,13 @@ public sealed class AuditChainVerifier
|
||||
return true;
|
||||
}
|
||||
|
||||
if (_chainFromUtc is { } chainFrom && row.OccurredAtUtc < chainFrom)
|
||||
{
|
||||
return Fail(seq, row.EventId, AuditChainBreakKind.ChainedBeforeCutoff,
|
||||
$"Row at ChainSeq {seq} is dated {row.OccurredAtUtc:O}, before the chain cutoff {chainFrom:O}: "
|
||||
+ "no row before the cutoff is ever chained, so its timestamp was moved back or the cutoff was moved forward.");
|
||||
}
|
||||
|
||||
if (seq < _expected)
|
||||
{
|
||||
return Fail(seq, row.EventId, AuditChainBreakKind.DuplicateSequence,
|
||||
@@ -147,9 +182,10 @@ public sealed class AuditChainVerifier
|
||||
}
|
||||
|
||||
/// <summary>Finishes verification against the chain head.</summary>
|
||||
/// <param name="unchainedRows">Rows in the month not yet chained (reported, never a break).</param>
|
||||
/// <param name="unchainedRows">Rows in the month, at or after the cutoff, not yet chained (reported, never a break).</param>
|
||||
/// <param name="preCutoffRows">Rows in the month before the chain cutoff: out of scope, never chained (reported, never a break).</param>
|
||||
/// <returns>The verification result.</returns>
|
||||
public AuditChainVerificationResult Complete(long unchainedRows)
|
||||
public AuditChainVerificationResult Complete(long unchainedRows, long preCutoffRows = 0)
|
||||
{
|
||||
if (_break is null)
|
||||
{
|
||||
@@ -171,7 +207,7 @@ public sealed class AuditChainVerifier
|
||||
}
|
||||
else
|
||||
{
|
||||
return Result(AuditChainStatus.NotChained, unchainedRows);
|
||||
return Result(AuditChainStatus.NotChained, unchainedRows, preCutoffRows);
|
||||
}
|
||||
}
|
||||
else if (lastSeq < _head.LastSeq)
|
||||
@@ -192,10 +228,10 @@ public sealed class AuditChainVerifier
|
||||
}
|
||||
}
|
||||
|
||||
return Result(_break is null ? AuditChainStatus.Intact : AuditChainStatus.Broken, unchainedRows);
|
||||
return Result(_break is null ? AuditChainStatus.Intact : AuditChainStatus.Broken, unchainedRows, preCutoffRows);
|
||||
}
|
||||
|
||||
private AuditChainVerificationResult Result(AuditChainStatus status, long unchainedRows)
|
||||
private AuditChainVerificationResult Result(AuditChainStatus status, long unchainedRows, long preCutoffRows)
|
||||
=> new(
|
||||
_partition,
|
||||
status,
|
||||
@@ -204,7 +240,9 @@ public sealed class AuditChainVerifier
|
||||
_head?.PurgedThroughSeq,
|
||||
_head?.LastSeq,
|
||||
unchainedRows,
|
||||
_break);
|
||||
_break,
|
||||
preCutoffRows,
|
||||
_chainFromUtc);
|
||||
|
||||
private bool Fail(long seq, Guid? eventId, AuditChainBreakKind kind, string detail)
|
||||
{
|
||||
|
||||
@@ -39,6 +39,19 @@ public static class AuditChainLock
|
||||
/// <c>OccurredAtUtc</c> order, which is why late-arriving site rows need no special case.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Chain start (ScadaBridge#139).</b> Unless the caller asks for backlog chaining, the first
|
||||
/// pass on a pristine database (no chain head, no chained row) records a <b>chain-start
|
||||
/// anchor</b> and only rows dated at or after it are ever chained; the historical backlog stays
|
||||
/// unchained and out of scope. The anchor is one row in <c>dbo.AuditLogChainTombstone</c> at
|
||||
/// <c>ChainSeq = 0</c> of the cutoff month: <c>PurgedAtUtc</c> holds the cutoff (the database's
|
||||
/// <c>SYSUTCDATETIME()</c> at first enablement, so both central nodes see the same instant
|
||||
/// whatever their own clocks say), <c>RowHash</c> holds
|
||||
/// <see cref="AuditChainHasher.ComputeCutoffAnchorHash"/> of it, and that hash is the genesis
|
||||
/// the cutoff month's first row links to. It is written once, under the exclusive chain lock,
|
||||
/// and never updated; the partition-switch purge keeps it. A database that already has a
|
||||
/// chain (heads or chained rows) and no anchor keeps chaining every row, as before.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Append-only.</b> The only AuditLog columns this class writes are <c>ChainSeq</c> and
|
||||
/// <c>RowHash</c>, and only while they are NULL. In production that UPDATE runs under the
|
||||
/// <c>scadabridge_audit_chainer</c> role, which holds a COLUMN-level UPDATE grant on just
|
||||
@@ -72,11 +85,17 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore
|
||||
/// </summary>
|
||||
public int VerifyCommandTimeoutSeconds { get; init; } = 300;
|
||||
|
||||
/// <summary>
|
||||
/// Test seam: the cutoff a new chain-start anchor records. Null (production) uses the
|
||||
/// database's <c>SYSUTCDATETIME()</c> so both central nodes agree regardless of node clocks.
|
||||
/// </summary>
|
||||
internal Func<DateTime>? AnchorClock { get; init; }
|
||||
|
||||
/// <summary>How long verification waits for an in-flight chaining/purge pass to release the lock.</summary>
|
||||
public TimeSpan VerifyLockTimeout { get; init; } = TimeSpan.FromSeconds(60);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<AuditChainBatchResult> ChainPendingAsync(int batchSize, CancellationToken ct = default)
|
||||
public async Task<AuditChainBatchResult> ChainPendingAsync(int batchSize, bool chainBacklog = false, CancellationToken ct = default)
|
||||
{
|
||||
if (batchSize <= 0)
|
||||
{
|
||||
@@ -95,12 +114,20 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore
|
||||
return new AuditChainBatchResult(0, LockBusy: true);
|
||||
}
|
||||
|
||||
// Oldest month that still has unchained rows. One month per pass keeps the
|
||||
// Chain start (ScadaBridge#139): the cutoff, recorded on the first pass.
|
||||
var cutoff = await ReadChainerCutoffAsync(conn, tx, ct).ConfigureAwait(false);
|
||||
if (cutoff is null && !chainBacklog && await IsPristineAsync(conn, tx, ct).ConfigureAwait(false))
|
||||
{
|
||||
cutoff = await CreateCutoffAnchorAsync(conn, tx, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Oldest month that still has unchained rows in scope. One month per pass keeps the
|
||||
// candidate read partition-eliminated and index-ordered (no cross-partition sort).
|
||||
DateTime? oldest;
|
||||
await using (var cmd = Command(conn, tx,
|
||||
"SELECT TOP (1) OccurredAtUtc FROM dbo.AuditLog WHERE ChainSeq IS NULL ORDER BY OccurredAtUtc;"))
|
||||
"SELECT TOP (1) OccurredAtUtc FROM dbo.AuditLog WHERE ChainSeq IS NULL AND OccurredAtUtc >= @chainFrom ORDER BY OccurredAtUtc;"))
|
||||
{
|
||||
AddChainFrom(cmd, cutoff?.ChainFromUtc);
|
||||
var scalar = await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false);
|
||||
oldest = scalar is DateTime dt ? DateTime.SpecifyKind(dt, DateTimeKind.Utc) : null;
|
||||
}
|
||||
@@ -116,10 +143,12 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore
|
||||
await using (var cmd = Command(conn, tx,
|
||||
$"SELECT TOP (@batch) {RowColumns} FROM dbo.AuditLog " +
|
||||
"WHERE ChainSeq IS NULL AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd " +
|
||||
"AND OccurredAtUtc >= @chainFrom " +
|
||||
"ORDER BY OccurredAtUtc, EventId;"))
|
||||
{
|
||||
AddInt(cmd, "@batch", batchSize);
|
||||
AddMonthRange(cmd, month);
|
||||
AddChainFrom(cmd, cutoff?.ChainFromUtc);
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
@@ -142,6 +171,13 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore
|
||||
}
|
||||
}
|
||||
|
||||
// The cutoff month's chain starts from the anchor, not from 32 zero bytes, so the
|
||||
// cutoff is inside every later link of that month.
|
||||
if (headSeq is null && cutoff is not null && cutoff.Month == month)
|
||||
{
|
||||
previous = cutoff.AnchorHash.ToArray();
|
||||
}
|
||||
|
||||
var seq = headSeq ?? 0;
|
||||
var links = new List<(AuditChainRow Row, long Seq, byte[] Hash)>(candidates.Count);
|
||||
foreach (var row in candidates)
|
||||
@@ -222,6 +258,8 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore
|
||||
}
|
||||
}
|
||||
|
||||
var cutoff = await ReadVerifierCutoffAsync(conn, tx, ct).ConfigureAwait(false);
|
||||
|
||||
var tombstones = new Dictionary<long, byte[]>();
|
||||
await using (var cmd = Command(conn, tx,
|
||||
"SELECT ChainSeq, RowHash FROM dbo.AuditLogChainTombstone WHERE ChainMonth = @month;"))
|
||||
@@ -234,16 +272,25 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore
|
||||
}
|
||||
}
|
||||
|
||||
long unchained;
|
||||
// Unchained rows split at the cutoff: pending (in scope, not reached yet) versus
|
||||
// pre-cutoff (out of scope by design, never chained).
|
||||
long unchained = 0, preCutoff = 0;
|
||||
await using (var cmd = Command(conn, tx,
|
||||
"SELECT COUNT_BIG(*) FROM dbo.AuditLog WHERE ChainSeq IS NULL AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd;"))
|
||||
"SELECT COUNT_BIG(CASE WHEN OccurredAtUtc >= @chainFrom THEN 1 END), COUNT_BIG(CASE WHEN OccurredAtUtc < @chainFrom THEN 1 END) " +
|
||||
"FROM dbo.AuditLog WHERE ChainSeq IS NULL AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd;"))
|
||||
{
|
||||
cmd.CommandTimeout = VerifyCommandTimeoutSeconds;
|
||||
AddMonthRange(cmd, month);
|
||||
unchained = Convert.ToInt64(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false));
|
||||
AddChainFrom(cmd, cutoff.ChainFromUtc);
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
if (await reader.ReadAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
unchained = reader.GetInt64(0);
|
||||
preCutoff = reader.GetInt64(1);
|
||||
}
|
||||
}
|
||||
|
||||
var verifier = new AuditChainVerifier(month, head, tombstones);
|
||||
var verifier = new AuditChainVerifier(month, head, tombstones, cutoff.ChainFromUtc, cutoff.Problem);
|
||||
await using (var cmd = Command(conn, tx,
|
||||
$"SELECT {RowColumns}, ChainSeq, RowHash FROM dbo.AuditLog " +
|
||||
"WHERE ChainSeq IS NOT NULL AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd " +
|
||||
@@ -262,7 +309,7 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore
|
||||
}
|
||||
|
||||
await tx.CommitAsync(ct).ConfigureAwait(false);
|
||||
return verifier.Complete(unchained);
|
||||
return verifier.Complete(unchained, preCutoff);
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -371,7 +418,7 @@ SELECT h.ChainMonth, h.LastSeq, h.LastHash, h.PurgedThroughSeq, h.PurgedThroughH
|
||||
WHERE a.ChainSeq IS NOT NULL
|
||||
AND a.OccurredAtUtc >= CAST(h.ChainMonth AS datetime2(7))
|
||||
AND a.OccurredAtUtc < DATEADD(MONTH, 1, CAST(h.ChainMonth AS datetime2(7)))) AS ChainedRows,
|
||||
(SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone t WHERE t.ChainMonth = h.ChainMonth) AS Tombstones
|
||||
(SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone t WHERE t.ChainMonth = h.ChainMonth AND t.ChainSeq > 0) AS Tombstones
|
||||
FROM dbo.AuditLogChainHead h
|
||||
ORDER BY h.ChainMonth;");
|
||||
cmd.CommandTimeout = VerifyCommandTimeoutSeconds;
|
||||
@@ -418,6 +465,135 @@ ORDER BY h.ChainMonth;");
|
||||
return new AuditChainHeadComparer.StoredLink(rowHash, tombstoneHash);
|
||||
}
|
||||
|
||||
/// <summary>The chain-start anchor as the chainer uses it (ScadaBridge#139).</summary>
|
||||
/// <param name="ChainFromUtc">The cutoff.</param>
|
||||
/// <param name="Month">The cutoff month (the anchor's <c>ChainMonth</c>).</param>
|
||||
/// <param name="AnchorHash">The anchor's stored hash: the cutoff month's genesis.</param>
|
||||
private sealed record ChainCutoff(DateTime ChainFromUtc, DateTime Month, byte[] AnchorHash);
|
||||
|
||||
/// <summary>The cutoff as verification sees it (ScadaBridge#139).</summary>
|
||||
/// <param name="ChainFromUtc">The cutoff, or null when the chain covers the whole table.</param>
|
||||
/// <param name="Problem">Why the anchor cannot be trusted, or null.</param>
|
||||
private sealed record VerifierCutoff(DateTime? ChainFromUtc, string? Problem);
|
||||
|
||||
/// <summary>
|
||||
/// Reads the chain-start anchor for the chaining pass. It can only sit in a month at or
|
||||
/// before the earliest chain head (it is written before anything is chained, and nothing
|
||||
/// before it ever is), so the read is bounded to those months rather than scanning every
|
||||
/// per-channel purge tombstone each tick. Verification reads them all.
|
||||
/// </summary>
|
||||
private static async Task<ChainCutoff?> ReadChainerCutoffAsync(DbConnection conn, DbTransaction tx, CancellationToken ct)
|
||||
{
|
||||
await using var cmd = Command(conn, tx,
|
||||
"SELECT TOP (1) ChainMonth, RowHash, PurgedAtUtc FROM dbo.AuditLogChainTombstone " +
|
||||
"WHERE ChainSeq = 0 AND ChainMonth <= ISNULL((SELECT MIN(ChainMonth) FROM dbo.AuditLogChainHead), '9999-12-01') " +
|
||||
"ORDER BY PurgedAtUtc, ChainMonth;");
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
if (!await reader.ReadAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new ChainCutoff(
|
||||
DateTime.SpecifyKind(reader.GetDateTime(2), DateTimeKind.Utc),
|
||||
DateTime.SpecifyKind(reader.GetDateTime(0), DateTimeKind.Utc),
|
||||
(byte[])reader.GetValue(1));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads every chain-start anchor and checks it: exactly one, in its cutoff's month, hashing
|
||||
/// to its own cutoff. The earliest anchor's cutoff is used either way, matching the chainer.
|
||||
/// </summary>
|
||||
private async Task<VerifierCutoff> ReadVerifierCutoffAsync(DbConnection conn, DbTransaction tx, CancellationToken ct)
|
||||
{
|
||||
var anchors = new List<(DateTime Month, byte[] Hash, DateTime ChainFrom)>();
|
||||
await using (var cmd = Command(conn, tx,
|
||||
"SELECT ChainMonth, RowHash, PurgedAtUtc FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0 ORDER BY PurgedAtUtc, ChainMonth;"))
|
||||
{
|
||||
cmd.CommandTimeout = VerifyCommandTimeoutSeconds;
|
||||
await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
anchors.Add((
|
||||
DateTime.SpecifyKind(reader.GetDateTime(0), DateTimeKind.Utc),
|
||||
(byte[])reader.GetValue(1),
|
||||
DateTime.SpecifyKind(reader.GetDateTime(2), DateTimeKind.Utc)));
|
||||
}
|
||||
}
|
||||
|
||||
if (anchors.Count == 0)
|
||||
{
|
||||
return new VerifierCutoff(null, null);
|
||||
}
|
||||
|
||||
var (anchorMonth, hash, chainFrom) = anchors[0];
|
||||
string? problem = null;
|
||||
if (anchors.Count > 1)
|
||||
{
|
||||
problem = $"{anchors.Count} chain-start anchors exist (cutoffs {string.Join(", ", anchors.Select(a => a.ChainFrom.ToString("O", System.Globalization.CultureInfo.InvariantCulture)))}); "
|
||||
+ "exactly one is ever written.";
|
||||
}
|
||||
else if (anchorMonth != AuditChainHasher.MonthOf(chainFrom))
|
||||
{
|
||||
problem = $"The chain-start anchor is filed under {AuditChainHasher.PartitionLabel(anchorMonth)} but its cutoff {chainFrom:O} is in another month.";
|
||||
}
|
||||
else if (!AuditChainHasher.ComputeCutoffAnchorHash(chainFrom).AsSpan().SequenceEqual(hash))
|
||||
{
|
||||
problem = $"The chain-start anchor's hash does not match its cutoff {chainFrom:O}: the cutoff or the anchor was altered.";
|
||||
}
|
||||
|
||||
return new VerifierCutoff(chainFrom, problem);
|
||||
}
|
||||
|
||||
/// <summary>True when nothing has ever been chained: no chain head and no chained row.</summary>
|
||||
private static async Task<bool> IsPristineAsync(DbConnection conn, DbTransaction tx, CancellationToken ct)
|
||||
{
|
||||
await using var cmd = Command(conn, tx,
|
||||
"SELECT CASE WHEN EXISTS (SELECT 1 FROM dbo.AuditLogChainHead) " +
|
||||
"OR EXISTS (SELECT 1 FROM dbo.AuditLog WHERE ChainSeq IS NOT NULL) THEN 0 ELSE 1 END;");
|
||||
return Convert.ToInt32(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false)) == 1;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes the chain-start anchor at the database's current UTC time (ScadaBridge#139). Runs
|
||||
/// inside the chaining transaction, under the exclusive chain lock, so exactly one node
|
||||
/// writes it and the peer's next pass reads it.
|
||||
/// </summary>
|
||||
private async Task<ChainCutoff> CreateCutoffAnchorAsync(DbConnection conn, DbTransaction tx, CancellationToken ct)
|
||||
{
|
||||
DateTime chainFrom;
|
||||
if (AnchorClock is { } clock)
|
||||
{
|
||||
chainFrom = DateTime.SpecifyKind(clock(), DateTimeKind.Utc);
|
||||
}
|
||||
else
|
||||
{
|
||||
await using var cmd = Command(conn, tx, "SELECT SYSUTCDATETIME();");
|
||||
chainFrom = DateTime.SpecifyKind((DateTime)(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false))!, DateTimeKind.Utc);
|
||||
}
|
||||
|
||||
var month = AuditChainHasher.MonthOf(chainFrom);
|
||||
var hash = AuditChainHasher.ComputeCutoffAnchorHash(chainFrom);
|
||||
await using (var cmd = Command(conn, tx,
|
||||
"INSERT INTO dbo.AuditLogChainTombstone (ChainMonth, ChainSeq, RowHash, EventId, PurgedAtUtc) " +
|
||||
"VALUES (@month, 0, @hash, @eventId, @chainFrom);"))
|
||||
{
|
||||
AddDate(cmd, "@month", month);
|
||||
cmd.Parameters.Add(new SqlParameter("@hash", SqlDbType.Binary, AuditChainHasher.HashLength) { Value = hash });
|
||||
cmd.Parameters.Add(new SqlParameter("@eventId", SqlDbType.UniqueIdentifier) { Value = AuditChainHasher.CutoffAnchorEventId });
|
||||
cmd.Parameters.Add(new SqlParameter("@chainFrom", SqlDbType.DateTime2) { Scale = 7, Value = chainFrom });
|
||||
await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"AuditLog hash chain started at {ChainFrom:O}: only rows dated from then on are chained; earlier rows stay unchained and out of scope (ScadaBridge#139).",
|
||||
chainFrom);
|
||||
return new ChainCutoff(chainFrom, month, hash);
|
||||
}
|
||||
|
||||
private static void AddChainFrom(DbCommand cmd, DateTime? chainFromUtc)
|
||||
=> cmd.Parameters.Add(new SqlParameter("@chainFrom", SqlDbType.DateTime2) { Scale = 7, Value = chainFromUtc ?? DateTime.MinValue });
|
||||
|
||||
/// <summary>
|
||||
/// Takes the chain application lock inside <paramref name="tx"/> (released at commit /
|
||||
/// rollback). Returns false when the lock could not be granted within <paramref name="timeout"/>.
|
||||
|
||||
+36
-3
@@ -378,6 +378,7 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive
|
||||
// Tombstones per month, then one streaming verifier per chain month.
|
||||
var tombstones = new Dictionary<DateTime, Dictionary<long, byte[]>>();
|
||||
long tombstonesRead = 0;
|
||||
DateTime? chainFrom = null;
|
||||
await using (var input = File.OpenRead(tombstonePath))
|
||||
{
|
||||
await foreach (var t in AuditArchiveParquet.ReadTombstonesAsync(input, ct).ConfigureAwait(false))
|
||||
@@ -389,12 +390,21 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive
|
||||
|
||||
byMonth[t.ChainSeq] = t.RowHash;
|
||||
tombstonesRead++;
|
||||
if (t.ChainSeq == AuditChainHasher.CutoffAnchorSeq)
|
||||
{
|
||||
// The chain-start anchor (ScadaBridge#139) travels with the archive of its month.
|
||||
chainFrom = chainFrom is { } seen && seen <= t.PurgedAtUtc ? seen : t.PurgedAtUtc;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Otherwise the cutoff comes from the database's anchor, which outlives every purge.
|
||||
chainFrom ??= await ReadChainFromAsync(ct).ConfigureAwait(false);
|
||||
|
||||
var states = manifest.Chains.ToDictionary(c => AuditArchiveParquet.ParseMonth(c.Partition));
|
||||
var verifiers = new SortedDictionary<DateTime, AuditChainVerifier>();
|
||||
var unchained = new Dictionary<DateTime, long>();
|
||||
var preCutoff = new Dictionary<DateTime, long>();
|
||||
AuditChainVerifier VerifierFor(DateTime month)
|
||||
{
|
||||
if (!verifiers.TryGetValue(month, out var verifier))
|
||||
@@ -406,7 +416,7 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive
|
||||
s.PriorPurgedThroughSeq,
|
||||
s.PriorPurgedThroughHash is null ? null : Convert.FromHexString(s.PriorPurgedThroughHash))
|
||||
: null;
|
||||
verifier = new AuditChainVerifier(month, head, tombstones.GetValueOrDefault(month) ?? []);
|
||||
verifier = new AuditChainVerifier(month, head, tombstones.GetValueOrDefault(month) ?? [], chainFrom);
|
||||
verifiers[month] = verifier;
|
||||
}
|
||||
|
||||
@@ -427,7 +437,8 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive
|
||||
var month = AuditChainHasher.MonthOf(row.OccurredAtUtc);
|
||||
if (row.ChainSeq is null)
|
||||
{
|
||||
unchained[month] = unchained.GetValueOrDefault(month) + 1;
|
||||
var bucket = chainFrom is { } cutoff && row.OccurredAtUtc < cutoff ? preCutoff : unchained;
|
||||
bucket[month] = bucket.GetValueOrDefault(month) + 1;
|
||||
VerifierFor(month);
|
||||
continue;
|
||||
}
|
||||
@@ -450,7 +461,7 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive
|
||||
var results = new List<AuditArchiveChainVerification>();
|
||||
foreach (var (month, verifier) in verifiers)
|
||||
{
|
||||
var chain = verifier.Complete(unchained.GetValueOrDefault(month));
|
||||
var chain = verifier.Complete(unchained.GetValueOrDefault(month), preCutoff.GetValueOrDefault(month));
|
||||
var (anchor, detail) = CompareAnchor(states.GetValueOrDefault(month), anchors.GetValueOrDefault(month));
|
||||
results.Add(new AuditArchiveChainVerification(chain, anchor, detail));
|
||||
}
|
||||
@@ -509,6 +520,28 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive
|
||||
$"The purge anchor (ChainSeq {database.PurgedSeq}) is before this archive's chain end ({archived.LastSeq}).");
|
||||
}
|
||||
|
||||
/// <summary>The database's chain cutoff (ScadaBridge#139), or null when the chain has none.</summary>
|
||||
private async Task<DateTime?> ReadChainFromAsync(CancellationToken ct)
|
||||
{
|
||||
var conn = _context.Database.GetDbConnection();
|
||||
var openedHere = await AuditLogHashChain.OpenAsync(conn, ct).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
await using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = "SELECT MIN(PurgedAtUtc) FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;";
|
||||
return await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false) is DateTime dt
|
||||
? DateTime.SpecifyKind(dt, DateTimeKind.Utc)
|
||||
: null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (openedHere)
|
||||
{
|
||||
await conn.CloseAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<Dictionary<DateTime, (bool HeadExists, long? PurgedSeq, byte[]? PurgedHash)>> ReadAnchorsAsync(
|
||||
IEnumerable<DateTime> months, CancellationToken ct)
|
||||
{
|
||||
|
||||
@@ -634,12 +634,15 @@ VALUES
|
||||
-- 3b. Hash chain (ScadaBridge#41): the months this partition held are gone as a
|
||||
-- whole, so record each month's current chain end as its checkpoint anchor
|
||||
-- (verification resumes after it; a late-arriving row chains on from it)
|
||||
-- and drop the per-channel tombstones the anchor now subsumes.
|
||||
-- and drop the per-channel tombstones the anchor now subsumes. The chain-start
|
||||
-- anchor (ChainSeq 0, ScadaBridge#139) is NOT a purge tombstone: it records the
|
||||
-- cutoff for the life of the database, so it stays.
|
||||
UPDATE dbo.AuditLogChainHead
|
||||
SET PurgedThroughSeq = LastSeq, PurgedThroughHash = LastHash, UpdatedAtUtc = SYSUTCDATETIME()
|
||||
WHERE $partition.pf_AuditLog_Month(CAST(ChainMonth AS datetime2(7))) = @partitionNumber;
|
||||
DELETE FROM dbo.AuditLogChainTombstone
|
||||
WHERE $partition.pf_AuditLog_Month(CAST(ChainMonth AS datetime2(7))) = @partitionNumber;
|
||||
WHERE $partition.pf_AuditLog_Month(CAST(ChainMonth AS datetime2(7))) = @partitionNumber
|
||||
AND ChainSeq > 0;
|
||||
|
||||
-- 4. Drop staging — the rows are discarded here. This is the purge.
|
||||
DROP TABLE dbo.[{stagingTableName}];
|
||||
|
||||
Reference in New Issue
Block a user