From 52da02a76ae392a4c0abe2300ec520d9802eecb3 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Fri, 13 Nov 2026 03:14:03 -0500 Subject: [PATCH] feat(audit): hash-chain cutoff - chain only rows written from first enablement, no backlog (#139) Owner decision on #107 (2026-11-13): enabling AuditLog:HashChain must not chain the historical backlog. main chained every unchained row oldest-first. - The first chaining pass on a pristine database (no head, no chained row) writes a chain-start anchor: AuditLogChainTombstone (cutoff month, ChainSeq 0), PurgedAtUtc = DB SYSUTCDATETIME(), RowHash = SHA-256("ScadaBridge.AuditLog.ChainCutoff.v1" || YYYY-MM || ticks). Written once under the exclusive chain lock, so both centrals agree whatever their clocks/config. No migration: existing table + grants. - Only rows with OccurredAtUtc >= cutoff are chained. The anchor hash is the cutoff month's genesis, binding the cutoff into the chain. - AuditLog:HashChain:ChainBacklog (default false) restores chain-everything, read only on the very first enablement. Legacy chains (heads, no anchor) keep chaining everything. - verify-chain splits unchained rows into UnchainedRows vs PreCutoffRows (+ ChainFromUtc); new breaks ChainedBeforeCutoff (row re-dated before the cutoff, or cutoff moved forward) and CutoffAnchorInvalid (anchor hash / month / multiplicity). Archive verify uses the anchor from its own tombstone file or the DB. - Partition switch-out keeps the anchor (deletes ChainSeq > 0 only); the head store / export / verify-heads tombstone count excludes it. - CLI verify-chain table prints Chain from + Pre-cutoff rows. Tests: AuditLogHashChainCutoffTests (7, MSSQL), 6 verifier unit tests, service/CLI/management wire tests; 9 mutations all killed. Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi --- docs/requirements/Component-AuditLog.md | 74 ++- .../Central/AuditLogHashChainOptions.cs | 21 +- .../Central/AuditLogHashChainService.cs | 2 +- .../Commands/AuditVerifyChainHelpers.cs | 2 + .../Repositories/IAuditLogHashChain.cs | 14 +- .../Types/Audit/AuditChainVerification.cs | 32 +- .../Maintenance/AuditChainHasher.cs | 29 + .../Maintenance/AuditChainVerifier.cs | 56 +- .../Maintenance/AuditLogHashChain.cs | 194 +++++- .../Maintenance/AuditLogPartitionArchive.cs | 39 +- .../Repositories/AuditLogRepository.cs | 7 +- .../Central/AuditLogHashChainServiceTests.cs | 32 +- .../Commands/AuditVerifyChainCommandTests.cs | 31 + .../Maintenance/AuditChainHeadStoreTests.cs | 2 +- .../Maintenance/AuditChainVerifierTests.cs | 128 +++- .../AuditLogArchiveCleanupTests.cs | 2 +- .../AuditLogHashChainCutoffTests.cs | 561 ++++++++++++++++++ .../Maintenance/AuditLogHashChainTests.cs | 4 +- .../AuditLogPartitionArchiveTests.cs | 2 +- .../Maintenance/AuditMaintenanceLoginTests.cs | 4 +- .../VerifyAuditChainCommandTests.cs | 21 + 21 files changed, 1208 insertions(+), 49 deletions(-) create mode 100644 tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditLogHashChainCutoffTests.cs diff --git a/docs/requirements/Component-AuditLog.md b/docs/requirements/Component-AuditLog.md index e5235afa..fe7c8841 100644 --- a/docs/requirements/Component-AuditLog.md +++ b/docs/requirements/Component-AuditLog.md @@ -936,7 +936,8 @@ MS SQL for direct-write events). Unredacted secrets never persist. - **Model.** One chain per **partition = UTC calendar month of `OccurredAtUtc`** (the `ps_AuditLog_Month` month). Each chained row carries `ChainSeq` (1, 2, 3 …, gap-free within its month) and `RowHash = SHA-256(prev.RowHash || canonical(row))`, - `prev` being the row at `ChainSeq - 1` (32 zero bytes for the first). The canonical + `prev` being the row at `ChainSeq - 1` (32 zero bytes for the first — or, in the month + the chain started in, the chain-start anchor below). The canonical v1 form (`AuditChainHasher`) is a domain tag, the month and sequence number, then the ten writable columns in table order, each string length-prefixed with a NULL marker; the computed columns derive from `DetailsJson` and are covered by it. A @@ -960,10 +961,44 @@ MS SQL for direct-write events). Unredacted secrets never persist. `OUTPUT … INTO`; verification bridges the gap with it. Both purge paths and the SourceNode backfill hold the same lock exclusively. The backfill skips chained rows (a chained row's `SourceNode` is frozen) — run it before enabling the chain. - - **Pre-existing rows** start unchained (`ChainSeq`/`RowHash` NULL). Enabling the job - chains the historical backlog oldest month first, so those rows are protected from - the moment they are chained, not from when they were written. Verification reports - still-unchained rows as `UnchainedRows`, never as a break. + - **Chain start: no backlog chaining (ScadaBridge#139; owner decision on #107, + 2026-11-13).** Enabling the job on a database with history does **not** chain the + backlog. The first chaining pass on a pristine database (no chain head, no chained row) + writes a **chain-start anchor** and from then on only rows with + `OccurredAtUtc >= cutoff` are chained; older rows keep `ChainSeq`/`RowHash` NULL for + ever and are out of scope. + - *Where the cutoff lives.* One row in `dbo.AuditLogChainTombstone` at `ChainSeq = 0` + of the cutoff month (real links start at 1, so slot 0 is "the link before the + first"): `PurgedAtUtc` = the cutoff, `RowHash` = + `SHA-256("ScadaBridge.AuditLog.ChainCutoff.v1" || YYYY-MM || ticks)`, `EventId` = + `5cadab1d-c4a1-4c0f-a000-000000000139`. The cutoff is the database's + `SYSUTCDATETIME()` at that first pass, written once inside the chaining transaction + under the exclusive chain lock — so both central nodes read the same instant whatever + their own clocks or config say, and it survives restarts. It is never updated; the + partition switch-out keeps it (it deletes only `ChainSeq > 0` tombstones); the head + store's tombstone count, the head export and `verify-heads` ignore it. **No + migration**: the table and the grants exist (the maintenance login writes it through + `scadabridge_audit_purger`'s INSERT on the tombstone table). + - *Bound into the chain.* The anchor hash is the genesis the cutoff month's first row + links to (instead of 32 zero bytes). Moving the cutoff therefore breaks that month's + chain, and the month's exported head anchors the cutoff off-box like any other link. + - *`AuditLog:HashChain:ChainBacklog`* (default `false`) restores the old behaviour — + chain every row, oldest month first, no anchor — but only on the very first + enablement: once an anchor or any chained row exists the decision is fixed and the + option is ignored, so two centrals with different values still end up with one + chain (whichever chains first decides). A database whose chain was enabled before + this change (heads, no anchor) keeps chaining every row, as before. + - *Late rows.* The cutoff is on `OccurredAtUtc` (the partition key, indexed), not on + ingest time (`IngestedAtUtc` is a non-persisted JSON column that must never be + filtered on). A site row dated before the cutoff that reaches central after + enablement — the site's store-and-forward backlog at that moment — stays out of + scope; so does a row from a node whose clock runs behind the database's by the + skew. Both are bounded and reported, never silently dropped. + - *Index.* Pre-cutoff rows stay in the filtered `IX_AuditLog_Unchained` until retention + purges their partition; the chaining read seeks `OccurredAtUtc >= cutoff`, so they + cost storage only, not chaining time. + - Verification reports in-scope rows not chained yet as `UnchainedRows` and the + backlog as `PreCutoffRows` (plus `ChainFromUtc`); neither is ever a break. - **Verification.** `scadabridge audit verify-chain --month YYYY-MM` sends the `VerifyAuditChain` management command (OperationalAudit roles; refused for a site-scoped caller because a month's chain spans every site; long-running Ask). It @@ -971,8 +1006,28 @@ MS SQL for direct-write events). Unredacted secrets never persist. `NotChained` or `Broken` with the **first break**: partition, `ChainSeq`, `EventId` and kind — `HashMismatch` (an edited row, or rows reordered — the sequence number is inside the hashed bytes), `MissingRow` (a deleted row with no tombstone), - `DuplicateSequence`, `IncompleteLink`, or `HeadMismatch` (rows deleted from the end). - CLI exit code 3 on `Broken`. + `DuplicateSequence`, `IncompleteLink`, `HeadMismatch` (rows deleted from the end), + `ChainedBeforeCutoff` or `CutoffAnchorInvalid` (below). CLI exit code 3 on `Broken`; + the table output also prints `Chain from` and `Pre-cutoff rows`. + - **Cutoff-eligibility attacks (ScadaBridge#139).** Because pre-cutoff rows are + unprotected by design, an attacker would like to (a) move a chained row's + `OccurredAtUtc` back before the cutoff to hide it among them, or (b) move the cutoff + forward. (a) with the link left in place is `ChainedBeforeCutoff` (no row before the + cutoff is ever chained — this also catches an attacker who re-hashes the row and every + later link); with the link cleared too, its sequence number is a `MissingRow` (or + `HeadMismatch` at the tail); moved into an earlier month, the source month reports + `MissingRow` and the target month `ChainedBeforeCutoff`. (b) editing the anchor's + cutoff without its hash, filing it under another month, or adding a second anchor is + `CutoffAnchorInvalid` (in every month's verification); re-hashing the anchor to the new + cutoff leaves the already-chained rows before it (`ChainedBeforeCutoff`) and changes the + cutoff month's genesis (`HashMismatch` at `ChainSeq` 1); deleting the anchor is a + `HashMismatch` at `ChainSeq` 1. **Known limits:** pre-cutoff rows themselves carry no + tamper evidence (that is the decision), and a backdated row forged *before* the cutoff + is indistinguishable from a genuine backlog row — the chain never proved a row genuine, + only that a chained row was not changed after chaining. A principal that rewrites the + anchor, the cutoff month's whole chain and its head is caught only by the off-box head + export, as for any full-chain rewrite. `AuditLogHashChainCutoffTests` and + `AuditChainVerifierTests` pin every case above. - **External anchor: off-box head export (ScadaBridge#107 item 1) — default OFF.** Inside the database alone, a principal that can rewrite rows AND recompute every later link AND the head produces a chain `verify-chain` accepts. Owner decision: **no signing**; instead @@ -1006,7 +1061,9 @@ MS SQL for direct-write events). Unredacted secrets never persist. head), grants the purger the head/tombstone rights its purge paths need, and DENYs INSERT/UPDATE/DELETE on both chain tables to `scadabridge_audit_writer`. - **Parquet archival (T2) — shipped (ScadaBridge#42), default OFF.** See Retention & - Purge. The archive carries the hash-chain columns and the chain head, so tamper + Purge. The archive carries the hash-chain columns and the chain head (and, for the + cutoff month, the chain-start anchor in its tombstone file, so the archive verifies from + its own files and splits its unchained rows into pending vs pre-cutoff), so tamper evidence survives the purge: `audit archive verify` ties an archive to the checkpoint anchor still in `AuditLogChainHead`. Like the chain itself, archives are unsigned by owner decision (ScadaBridge#107/#109); a principal able to rewrite both the archive files and the @@ -1088,6 +1145,7 @@ Security & Tamper-Evidence): "IntervalSeconds": 60, "BatchSize": 1000, "MaxBatchesPerTick": 100, + "ChainBacklog": false, // default: chain only rows from first enablement on (#139); true = chain the backlog, first enablement only "HeadExport": { "Enabled": false, // default off "IntervalMinutes": 60, diff --git a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogHashChainOptions.cs b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogHashChainOptions.cs index 4deb3734..66e801a5 100644 --- a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogHashChainOptions.cs +++ b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogHashChainOptions.cs @@ -15,10 +15,14 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central; /// them chains at a time, so a dual-active pair still produces one gap-free chain per month. /// /// -/// Enabling it on an existing database chains the historical backlog oldest month first, -/// × rows per tick, so the first -/// ticks run long. Rows chained late are only protected from the moment they are chained. -/// Run audit backfill-source-node BEFORE enabling: a chained row's SourceNode is frozen. +/// Chain start (ScadaBridge#139). Enabling it on an existing database does NOT chain the +/// historical backlog: the first chaining pass records a chain-start anchor in the database at +/// the database's current UTC time, and only rows dated from then on are chained. Older rows +/// stay unchained and are reported by audit verify-chain as pre-cutoff rows, never as a +/// break. The anchor is written once and is the same for both central nodes. Set +/// to chain the backlog instead (only effective on the very first +/// enablement). Run audit backfill-source-node BEFORE enabling either way: a chained +/// row's SourceNode is frozen. /// /// public sealed class AuditLogHashChainOptions @@ -26,6 +30,15 @@ public sealed class AuditLogHashChainOptions /// Whether the chaining job runs (default false). public bool Enabled { get; set; } + /// + /// Chain the historical backlog on first enablement instead of starting the chain at the + /// enablement instant (default false — owner decision on ScadaBridge#107, 2026-11-13: no + /// backlog chaining). Only read while nothing has been chained yet: once the database holds + /// a chain-start anchor or any chained row, the decision is fixed and this value is ignored. + /// Set it the same on both central nodes; if they differ, whichever chains first decides. + /// + public bool ChainBacklog { get; set; } + /// Seconds between chaining ticks (default 60). public int IntervalSeconds { get; set; } = 60; diff --git a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogHashChainService.cs b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogHashChainService.cs index 4a003743..df2f148c 100644 --- a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogHashChainService.cs +++ b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogHashChainService.cs @@ -92,7 +92,7 @@ public sealed class AuditLogHashChainService : IHostedService, IDisposable var chain = scope.ServiceProvider.GetRequiredService(); for (var i = 0; i < options.MaxBatchesPerTick && !ct.IsCancellationRequested; i++) { - var result = await chain.ChainPendingAsync(options.BatchSize, ct).ConfigureAwait(false); + var result = await chain.ChainPendingAsync(options.BatchSize, options.ChainBacklog, ct).ConfigureAwait(false); total += result.RowsChained; if (result.LockBusy || result.RowsChained == 0) { diff --git a/src/ZB.MOM.WW.ScadaBridge.CLI/Commands/AuditVerifyChainHelpers.cs b/src/ZB.MOM.WW.ScadaBridge.CLI/Commands/AuditVerifyChainHelpers.cs index 3e3e142f..e3dde5a5 100644 --- a/src/ZB.MOM.WW.ScadaBridge.CLI/Commands/AuditVerifyChainHelpers.cs +++ b/src/ZB.MOM.WW.ScadaBridge.CLI/Commands/AuditVerifyChainHelpers.cs @@ -64,6 +64,8 @@ public static class AuditVerifyChainHelpers output.WriteLine($"Purged through : {Num(root, "purgedThroughSeq", "-")}"); output.WriteLine($"Head sequence : {Num(root, "headSeq", "-")}"); output.WriteLine($"Unchained rows : {Num(root, "unchainedRows")}"); + output.WriteLine($"Chain from : {Str(root, "chainFromUtc", "- (whole table in scope)")}"); + output.WriteLine($"Pre-cutoff rows : {Num(root, "preCutoffRows")}"); if (root.TryGetProperty("firstBreak", out var brk) && brk.ValueKind == JsonValueKind.Object) { output.WriteLine("First break:"); diff --git a/src/ZB.MOM.WW.ScadaBridge.Commons/Interfaces/Repositories/IAuditLogHashChain.cs b/src/ZB.MOM.WW.ScadaBridge.Commons/Interfaces/Repositories/IAuditLogHashChain.cs index 077c738f..78e852d9 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Commons/Interfaces/Repositories/IAuditLogHashChain.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Commons/Interfaces/Repositories/IAuditLogHashChain.cs @@ -31,10 +31,22 @@ public interface IAuditLogHashChain /// Returns immediately with set when another /// chainer holds the chain lock. /// + /// + /// Chain start (ScadaBridge#139). On a pristine database (nothing ever chained) and + /// with false, the first pass records a chain-start anchor + /// at the database's current UTC time; from then on only rows dated at or after it are + /// chained, whatever either node passes later. With true on + /// that first pass, no anchor is written and every row is chained, oldest month first. A + /// database that already has a chain and no anchor keeps chaining every row. + /// /// Maximum rows to chain in this pass. + /// + /// Only consulted on a pristine database: true chains the historical backlog instead of + /// starting the chain now. + /// /// Cancellation token. /// How many rows were chained, and whether the lock was busy. - Task ChainPendingAsync(int batchSize, CancellationToken ct = default); + Task ChainPendingAsync(int batchSize, bool chainBacklog = false, CancellationToken ct = default); /// /// Verifies one month's chain end to end and reports the first break. diff --git a/src/ZB.MOM.WW.ScadaBridge.Commons/Types/Audit/AuditChainVerification.cs b/src/ZB.MOM.WW.ScadaBridge.Commons/Types/Audit/AuditChainVerification.cs index 1f309a28..b987ccd0 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Commons/Types/Audit/AuditChainVerification.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Commons/Types/Audit/AuditChainVerification.cs @@ -46,6 +46,20 @@ public enum AuditChainBreakKind /// deleted from the tail, or the head itself was altered. /// HeadMismatch, + + /// + /// A chained row is dated before the chain cutoff (ScadaBridge#139). Rows before the cutoff + /// are never chained, so the row's OccurredAtUtc was moved back (to hide it among the + /// unprotected pre-cutoff rows) or the cutoff itself was moved forward. + /// + ChainedBeforeCutoff, + + /// + /// The chain-start (cutoff) anchor is inconsistent (ScadaBridge#139): its stored hash does not + /// match its cutoff, it sits in the wrong month, or more than one anchor exists. The cutoff + /// that decides which rows are in scope cannot be trusted. + /// + CutoffAnchorInvalid, } /// @@ -80,10 +94,20 @@ public sealed record AuditChainBreak( /// /// The last sequence number the chain head records, or null when the month has no head. /// -/// Rows in the month the chaining job has not reached yet (written since the last tick, or -/// written while chaining was disabled). They are not covered by the chain. +/// Rows in the month, at or after the chain cutoff, that the chaining job has not reached yet +/// (written since the last tick, or written while chaining was disabled). They are not covered +/// by the chain yet. /// /// The first break, or null when is not . +/// +/// Rows in the month dated before the chain cutoff (ScadaBridge#139). They are out of scope by +/// design — the chain covers only rows written from its first enablement on — so they are +/// never chained and never a break. Zero when the chain has no cutoff. +/// +/// +/// The chain cutoff: the instant the chain was first enabled without backlog chaining. Null +/// when the chain covers the whole table (legacy / ChainBacklog enablement). +/// public sealed record AuditChainVerificationResult( string Partition, AuditChainStatus Status, @@ -92,7 +116,9 @@ public sealed record AuditChainVerificationResult( long? PurgedThroughSeq, long? HeadSeq, long UnchainedRows, - AuditChainBreak? FirstBreak); + AuditChainBreak? FirstBreak, + long PreCutoffRows = 0, + DateTime? ChainFromUtc = null); /// Outcome of one chaining pass over unchained AuditLog rows. /// Rows linked into a chain by this pass. diff --git a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditChainHasher.cs b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditChainHasher.cs index 65b18efc..eb8f489a 100644 --- a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditChainHasher.cs +++ b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditChainHasher.cs @@ -130,6 +130,35 @@ public static class AuditChainHasher return buffer.ToArray(); } + /// + /// The ChainSeq of the chain-start (cutoff) anchor in dbo.AuditLogChainTombstone + /// (ScadaBridge#139). Real links start at 1, so slot 0 is "the link before the first row": + /// the anchor's RowHash is the genesis hash of the cutoff month's chain. + /// + public const long CutoffAnchorSeq = 0; + + /// The fixed EventId the chain-start anchor row carries, so it is recognisable by eye. + public static readonly Guid CutoffAnchorEventId = new("5cadab1d-c4a1-4c0f-a000-000000000139"); + + private static readonly byte[] CutoffDomainTag = Encoding.ASCII.GetBytes("ScadaBridge.AuditLog.ChainCutoff.v1"); + + /// + /// Computes the chain-start anchor hash for a cutoff instant (ScadaBridge#139): + /// SHA-256("ScadaBridge.AuditLog.ChainCutoff.v1" || YYYY-MM || ticks), big-endian like + /// the row form. It is both the anchor row's stored RowHash and the previous-hash the + /// first row of the cutoff month links to, so moving the cutoff breaks that month's chain. + /// + /// The cutoff: rows with an earlier OccurredAtUtc are never chained. + /// The 32-byte anchor hash. + public static byte[] ComputeCutoffAnchorHash(DateTime chainFromUtc) + { + using var buffer = new MemoryStream(64); + buffer.Write(CutoffDomainTag); + WriteString(buffer, PartitionLabel(MonthOf(chainFromUtc))); + WriteInt64(buffer, chainFromUtc.Ticks); + return SHA256.HashData(buffer.ToArray()); + } + private static void WriteInt64(Stream stream, long value) { Span bytes = stackalloc byte[8]; diff --git a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditChainVerifier.cs b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditChainVerifier.cs index 2551cecb..b5226c6e 100644 --- a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditChainVerifier.cs +++ b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditChainVerifier.cs @@ -35,6 +35,15 @@ public sealed record AuditChainHeadState( /// reordered rows (ChainSeq values swapped) → /// at the first moved row, because the sequence number is inside the hashed bytes. /// +/// +/// Chain start (cutoff, ScadaBridge#139). When the chain was enabled with no backlog, +/// a link-0 anchor in the tombstones of the cutoff month is that month's genesis hash (instead +/// of 32 zero bytes), and no row before the cutoff is ever chained. So a chained row whose +/// OccurredAtUtc is before the cutoff is a break (): +/// either the row's timestamp was moved back to hide it among the unprotected rows, or the +/// cutoff was moved forward. An anchor that does not hash to its own cutoff is reported as +/// by the caller. +/// /// public sealed class AuditChainVerifier { @@ -43,6 +52,7 @@ public sealed class AuditChainVerifier private readonly AuditChainHeadState? _head; private readonly IReadOnlyDictionary _tombstones; private readonly long _startAfter; + private readonly DateTime? _chainFromUtc; private byte[] _previous; private long _expected; private long _verified; @@ -52,11 +62,18 @@ public sealed class AuditChainVerifier /// Initializes a verifier for one month. /// The chain partition (first instant of the month, UTC). /// The month's chain head, or null when none exists. - /// Purge tombstones for the month, keyed by ChainSeq. + /// + /// Purge tombstones for the month, keyed by ChainSeq; a key of + /// is the chain-start anchor (the month's genesis). + /// + /// The chain cutoff, or null when the whole table is in scope (legacy / backlog chaining). + /// When non-null, the chain-start anchor is invalid; verification fails with this detail. public AuditChainVerifier( DateTime monthStartUtc, AuditChainHeadState? head, - IReadOnlyDictionary tombstones) + IReadOnlyDictionary tombstones, + DateTime? chainFromUtc = null, + string? anchorProblem = null) { _monthStartUtc = monthStartUtc; _partition = AuditChainHasher.PartitionLabel(monthStartUtc); @@ -71,12 +88,23 @@ public sealed class AuditChainVerifier else { _startAfter = 0; - _previous = AuditChainHasher.GenesisHash.ToArray(); + _previous = _tombstones.TryGetValue(AuditChainHasher.CutoffAnchorSeq, out var cutoffAnchor) + ? cutoffAnchor + : AuditChainHasher.GenesisHash.ToArray(); } _expected = _startAfter + 1; + _chainFromUtc = chainFromUtc; + if (anchorProblem is not null) + { + Fail(AuditChainHasher.CutoffAnchorSeq, AuditChainHasher.CutoffAnchorEventId, + AuditChainBreakKind.CutoffAnchorInvalid, anchorProblem); + } } + /// The chain cutoff this verification applies, or null. + public DateTime? ChainFromUtc => _chainFromUtc; + /// The chain partition label (YYYY-MM). public string Partition => _partition; @@ -106,6 +134,13 @@ public sealed class AuditChainVerifier return true; } + if (_chainFromUtc is { } chainFrom && row.OccurredAtUtc < chainFrom) + { + return Fail(seq, row.EventId, AuditChainBreakKind.ChainedBeforeCutoff, + $"Row at ChainSeq {seq} is dated {row.OccurredAtUtc:O}, before the chain cutoff {chainFrom:O}: " + + "no row before the cutoff is ever chained, so its timestamp was moved back or the cutoff was moved forward."); + } + if (seq < _expected) { return Fail(seq, row.EventId, AuditChainBreakKind.DuplicateSequence, @@ -147,9 +182,10 @@ public sealed class AuditChainVerifier } /// Finishes verification against the chain head. - /// Rows in the month not yet chained (reported, never a break). + /// Rows in the month, at or after the cutoff, not yet chained (reported, never a break). + /// Rows in the month before the chain cutoff: out of scope, never chained (reported, never a break). /// The verification result. - public AuditChainVerificationResult Complete(long unchainedRows) + public AuditChainVerificationResult Complete(long unchainedRows, long preCutoffRows = 0) { if (_break is null) { @@ -171,7 +207,7 @@ public sealed class AuditChainVerifier } else { - return Result(AuditChainStatus.NotChained, unchainedRows); + return Result(AuditChainStatus.NotChained, unchainedRows, preCutoffRows); } } else if (lastSeq < _head.LastSeq) @@ -192,10 +228,10 @@ public sealed class AuditChainVerifier } } - return Result(_break is null ? AuditChainStatus.Intact : AuditChainStatus.Broken, unchainedRows); + return Result(_break is null ? AuditChainStatus.Intact : AuditChainStatus.Broken, unchainedRows, preCutoffRows); } - private AuditChainVerificationResult Result(AuditChainStatus status, long unchainedRows) + private AuditChainVerificationResult Result(AuditChainStatus status, long unchainedRows, long preCutoffRows) => new( _partition, status, @@ -204,7 +240,9 @@ public sealed class AuditChainVerifier _head?.PurgedThroughSeq, _head?.LastSeq, unchainedRows, - _break); + _break, + preCutoffRows, + _chainFromUtc); private bool Fail(long seq, Guid? eventId, AuditChainBreakKind kind, string detail) { diff --git a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditLogHashChain.cs b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditLogHashChain.cs index 1bd3fe5a..d154b8a2 100644 --- a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditLogHashChain.cs +++ b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditLogHashChain.cs @@ -39,6 +39,19 @@ public static class AuditChainLock /// OccurredAtUtc order, which is why late-arriving site rows need no special case. /// /// +/// Chain start (ScadaBridge#139). Unless the caller asks for backlog chaining, the first +/// pass on a pristine database (no chain head, no chained row) records a chain-start +/// anchor and only rows dated at or after it are ever chained; the historical backlog stays +/// unchained and out of scope. The anchor is one row in dbo.AuditLogChainTombstone at +/// ChainSeq = 0 of the cutoff month: PurgedAtUtc holds the cutoff (the database's +/// SYSUTCDATETIME() at first enablement, so both central nodes see the same instant +/// whatever their own clocks say), RowHash holds +/// of it, and that hash is the genesis +/// the cutoff month's first row links to. It is written once, under the exclusive chain lock, +/// and never updated; the partition-switch purge keeps it. A database that already has a +/// chain (heads or chained rows) and no anchor keeps chaining every row, as before. +/// +/// /// Append-only. The only AuditLog columns this class writes are ChainSeq and /// RowHash, and only while they are NULL. In production that UPDATE runs under the /// scadabridge_audit_chainer role, which holds a COLUMN-level UPDATE grant on just @@ -72,11 +85,17 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore /// public int VerifyCommandTimeoutSeconds { get; init; } = 300; + /// + /// Test seam: the cutoff a new chain-start anchor records. Null (production) uses the + /// database's SYSUTCDATETIME() so both central nodes agree regardless of node clocks. + /// + internal Func? AnchorClock { get; init; } + /// How long verification waits for an in-flight chaining/purge pass to release the lock. public TimeSpan VerifyLockTimeout { get; init; } = TimeSpan.FromSeconds(60); /// - public async Task ChainPendingAsync(int batchSize, CancellationToken ct = default) + public async Task ChainPendingAsync(int batchSize, bool chainBacklog = false, CancellationToken ct = default) { if (batchSize <= 0) { @@ -95,12 +114,20 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore return new AuditChainBatchResult(0, LockBusy: true); } - // Oldest month that still has unchained rows. One month per pass keeps the + // Chain start (ScadaBridge#139): the cutoff, recorded on the first pass. + var cutoff = await ReadChainerCutoffAsync(conn, tx, ct).ConfigureAwait(false); + if (cutoff is null && !chainBacklog && await IsPristineAsync(conn, tx, ct).ConfigureAwait(false)) + { + cutoff = await CreateCutoffAnchorAsync(conn, tx, ct).ConfigureAwait(false); + } + + // Oldest month that still has unchained rows in scope. One month per pass keeps the // candidate read partition-eliminated and index-ordered (no cross-partition sort). DateTime? oldest; await using (var cmd = Command(conn, tx, - "SELECT TOP (1) OccurredAtUtc FROM dbo.AuditLog WHERE ChainSeq IS NULL ORDER BY OccurredAtUtc;")) + "SELECT TOP (1) OccurredAtUtc FROM dbo.AuditLog WHERE ChainSeq IS NULL AND OccurredAtUtc >= @chainFrom ORDER BY OccurredAtUtc;")) { + AddChainFrom(cmd, cutoff?.ChainFromUtc); var scalar = await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false); oldest = scalar is DateTime dt ? DateTime.SpecifyKind(dt, DateTimeKind.Utc) : null; } @@ -116,10 +143,12 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore await using (var cmd = Command(conn, tx, $"SELECT TOP (@batch) {RowColumns} FROM dbo.AuditLog " + "WHERE ChainSeq IS NULL AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd " + + "AND OccurredAtUtc >= @chainFrom " + "ORDER BY OccurredAtUtc, EventId;")) { AddInt(cmd, "@batch", batchSize); AddMonthRange(cmd, month); + AddChainFrom(cmd, cutoff?.ChainFromUtc); await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); while (await reader.ReadAsync(ct).ConfigureAwait(false)) { @@ -142,6 +171,13 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore } } + // The cutoff month's chain starts from the anchor, not from 32 zero bytes, so the + // cutoff is inside every later link of that month. + if (headSeq is null && cutoff is not null && cutoff.Month == month) + { + previous = cutoff.AnchorHash.ToArray(); + } + var seq = headSeq ?? 0; var links = new List<(AuditChainRow Row, long Seq, byte[] Hash)>(candidates.Count); foreach (var row in candidates) @@ -222,6 +258,8 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore } } + var cutoff = await ReadVerifierCutoffAsync(conn, tx, ct).ConfigureAwait(false); + var tombstones = new Dictionary(); await using (var cmd = Command(conn, tx, "SELECT ChainSeq, RowHash FROM dbo.AuditLogChainTombstone WHERE ChainMonth = @month;")) @@ -234,16 +272,25 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore } } - long unchained; + // Unchained rows split at the cutoff: pending (in scope, not reached yet) versus + // pre-cutoff (out of scope by design, never chained). + long unchained = 0, preCutoff = 0; await using (var cmd = Command(conn, tx, - "SELECT COUNT_BIG(*) FROM dbo.AuditLog WHERE ChainSeq IS NULL AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd;")) + "SELECT COUNT_BIG(CASE WHEN OccurredAtUtc >= @chainFrom THEN 1 END), COUNT_BIG(CASE WHEN OccurredAtUtc < @chainFrom THEN 1 END) " + + "FROM dbo.AuditLog WHERE ChainSeq IS NULL AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd;")) { cmd.CommandTimeout = VerifyCommandTimeoutSeconds; AddMonthRange(cmd, month); - unchained = Convert.ToInt64(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false)); + AddChainFrom(cmd, cutoff.ChainFromUtc); + await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); + if (await reader.ReadAsync(ct).ConfigureAwait(false)) + { + unchained = reader.GetInt64(0); + preCutoff = reader.GetInt64(1); + } } - var verifier = new AuditChainVerifier(month, head, tombstones); + var verifier = new AuditChainVerifier(month, head, tombstones, cutoff.ChainFromUtc, cutoff.Problem); await using (var cmd = Command(conn, tx, $"SELECT {RowColumns}, ChainSeq, RowHash FROM dbo.AuditLog " + "WHERE ChainSeq IS NOT NULL AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd " + @@ -262,7 +309,7 @@ public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore } await tx.CommitAsync(ct).ConfigureAwait(false); - return verifier.Complete(unchained); + return verifier.Complete(unchained, preCutoff); } finally { @@ -371,7 +418,7 @@ SELECT h.ChainMonth, h.LastSeq, h.LastHash, h.PurgedThroughSeq, h.PurgedThroughH WHERE a.ChainSeq IS NOT NULL AND a.OccurredAtUtc >= CAST(h.ChainMonth AS datetime2(7)) AND a.OccurredAtUtc < DATEADD(MONTH, 1, CAST(h.ChainMonth AS datetime2(7)))) AS ChainedRows, - (SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone t WHERE t.ChainMonth = h.ChainMonth) AS Tombstones + (SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone t WHERE t.ChainMonth = h.ChainMonth AND t.ChainSeq > 0) AS Tombstones FROM dbo.AuditLogChainHead h ORDER BY h.ChainMonth;"); cmd.CommandTimeout = VerifyCommandTimeoutSeconds; @@ -418,6 +465,135 @@ ORDER BY h.ChainMonth;"); return new AuditChainHeadComparer.StoredLink(rowHash, tombstoneHash); } + /// The chain-start anchor as the chainer uses it (ScadaBridge#139). + /// The cutoff. + /// The cutoff month (the anchor's ChainMonth). + /// The anchor's stored hash: the cutoff month's genesis. + private sealed record ChainCutoff(DateTime ChainFromUtc, DateTime Month, byte[] AnchorHash); + + /// The cutoff as verification sees it (ScadaBridge#139). + /// The cutoff, or null when the chain covers the whole table. + /// Why the anchor cannot be trusted, or null. + private sealed record VerifierCutoff(DateTime? ChainFromUtc, string? Problem); + + /// + /// Reads the chain-start anchor for the chaining pass. It can only sit in a month at or + /// before the earliest chain head (it is written before anything is chained, and nothing + /// before it ever is), so the read is bounded to those months rather than scanning every + /// per-channel purge tombstone each tick. Verification reads them all. + /// + private static async Task ReadChainerCutoffAsync(DbConnection conn, DbTransaction tx, CancellationToken ct) + { + await using var cmd = Command(conn, tx, + "SELECT TOP (1) ChainMonth, RowHash, PurgedAtUtc FROM dbo.AuditLogChainTombstone " + + "WHERE ChainSeq = 0 AND ChainMonth <= ISNULL((SELECT MIN(ChainMonth) FROM dbo.AuditLogChainHead), '9999-12-01') " + + "ORDER BY PurgedAtUtc, ChainMonth;"); + await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); + if (!await reader.ReadAsync(ct).ConfigureAwait(false)) + { + return null; + } + + return new ChainCutoff( + DateTime.SpecifyKind(reader.GetDateTime(2), DateTimeKind.Utc), + DateTime.SpecifyKind(reader.GetDateTime(0), DateTimeKind.Utc), + (byte[])reader.GetValue(1)); + } + + /// + /// Reads every chain-start anchor and checks it: exactly one, in its cutoff's month, hashing + /// to its own cutoff. The earliest anchor's cutoff is used either way, matching the chainer. + /// + private async Task ReadVerifierCutoffAsync(DbConnection conn, DbTransaction tx, CancellationToken ct) + { + var anchors = new List<(DateTime Month, byte[] Hash, DateTime ChainFrom)>(); + await using (var cmd = Command(conn, tx, + "SELECT ChainMonth, RowHash, PurgedAtUtc FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0 ORDER BY PurgedAtUtc, ChainMonth;")) + { + cmd.CommandTimeout = VerifyCommandTimeoutSeconds; + await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); + while (await reader.ReadAsync(ct).ConfigureAwait(false)) + { + anchors.Add(( + DateTime.SpecifyKind(reader.GetDateTime(0), DateTimeKind.Utc), + (byte[])reader.GetValue(1), + DateTime.SpecifyKind(reader.GetDateTime(2), DateTimeKind.Utc))); + } + } + + if (anchors.Count == 0) + { + return new VerifierCutoff(null, null); + } + + var (anchorMonth, hash, chainFrom) = anchors[0]; + string? problem = null; + if (anchors.Count > 1) + { + problem = $"{anchors.Count} chain-start anchors exist (cutoffs {string.Join(", ", anchors.Select(a => a.ChainFrom.ToString("O", System.Globalization.CultureInfo.InvariantCulture)))}); " + + "exactly one is ever written."; + } + else if (anchorMonth != AuditChainHasher.MonthOf(chainFrom)) + { + problem = $"The chain-start anchor is filed under {AuditChainHasher.PartitionLabel(anchorMonth)} but its cutoff {chainFrom:O} is in another month."; + } + else if (!AuditChainHasher.ComputeCutoffAnchorHash(chainFrom).AsSpan().SequenceEqual(hash)) + { + problem = $"The chain-start anchor's hash does not match its cutoff {chainFrom:O}: the cutoff or the anchor was altered."; + } + + return new VerifierCutoff(chainFrom, problem); + } + + /// True when nothing has ever been chained: no chain head and no chained row. + private static async Task IsPristineAsync(DbConnection conn, DbTransaction tx, CancellationToken ct) + { + await using var cmd = Command(conn, tx, + "SELECT CASE WHEN EXISTS (SELECT 1 FROM dbo.AuditLogChainHead) " + + "OR EXISTS (SELECT 1 FROM dbo.AuditLog WHERE ChainSeq IS NOT NULL) THEN 0 ELSE 1 END;"); + return Convert.ToInt32(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false)) == 1; + } + + /// + /// Writes the chain-start anchor at the database's current UTC time (ScadaBridge#139). Runs + /// inside the chaining transaction, under the exclusive chain lock, so exactly one node + /// writes it and the peer's next pass reads it. + /// + private async Task CreateCutoffAnchorAsync(DbConnection conn, DbTransaction tx, CancellationToken ct) + { + DateTime chainFrom; + if (AnchorClock is { } clock) + { + chainFrom = DateTime.SpecifyKind(clock(), DateTimeKind.Utc); + } + else + { + await using var cmd = Command(conn, tx, "SELECT SYSUTCDATETIME();"); + chainFrom = DateTime.SpecifyKind((DateTime)(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false))!, DateTimeKind.Utc); + } + + var month = AuditChainHasher.MonthOf(chainFrom); + var hash = AuditChainHasher.ComputeCutoffAnchorHash(chainFrom); + await using (var cmd = Command(conn, tx, + "INSERT INTO dbo.AuditLogChainTombstone (ChainMonth, ChainSeq, RowHash, EventId, PurgedAtUtc) " + + "VALUES (@month, 0, @hash, @eventId, @chainFrom);")) + { + AddDate(cmd, "@month", month); + cmd.Parameters.Add(new SqlParameter("@hash", SqlDbType.Binary, AuditChainHasher.HashLength) { Value = hash }); + cmd.Parameters.Add(new SqlParameter("@eventId", SqlDbType.UniqueIdentifier) { Value = AuditChainHasher.CutoffAnchorEventId }); + cmd.Parameters.Add(new SqlParameter("@chainFrom", SqlDbType.DateTime2) { Scale = 7, Value = chainFrom }); + await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false); + } + + _logger.LogInformation( + "AuditLog hash chain started at {ChainFrom:O}: only rows dated from then on are chained; earlier rows stay unchained and out of scope (ScadaBridge#139).", + chainFrom); + return new ChainCutoff(chainFrom, month, hash); + } + + private static void AddChainFrom(DbCommand cmd, DateTime? chainFromUtc) + => cmd.Parameters.Add(new SqlParameter("@chainFrom", SqlDbType.DateTime2) { Scale = 7, Value = chainFromUtc ?? DateTime.MinValue }); + /// /// Takes the chain application lock inside (released at commit / /// rollback). Returns false when the lock could not be granted within . diff --git a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditLogPartitionArchive.cs b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditLogPartitionArchive.cs index 54af5355..d6890f3b 100644 --- a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditLogPartitionArchive.cs +++ b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditLogPartitionArchive.cs @@ -378,6 +378,7 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive // Tombstones per month, then one streaming verifier per chain month. var tombstones = new Dictionary>(); long tombstonesRead = 0; + DateTime? chainFrom = null; await using (var input = File.OpenRead(tombstonePath)) { await foreach (var t in AuditArchiveParquet.ReadTombstonesAsync(input, ct).ConfigureAwait(false)) @@ -389,12 +390,21 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive byMonth[t.ChainSeq] = t.RowHash; tombstonesRead++; + if (t.ChainSeq == AuditChainHasher.CutoffAnchorSeq) + { + // The chain-start anchor (ScadaBridge#139) travels with the archive of its month. + chainFrom = chainFrom is { } seen && seen <= t.PurgedAtUtc ? seen : t.PurgedAtUtc; + } } } + // Otherwise the cutoff comes from the database's anchor, which outlives every purge. + chainFrom ??= await ReadChainFromAsync(ct).ConfigureAwait(false); + var states = manifest.Chains.ToDictionary(c => AuditArchiveParquet.ParseMonth(c.Partition)); var verifiers = new SortedDictionary(); var unchained = new Dictionary(); + var preCutoff = new Dictionary(); AuditChainVerifier VerifierFor(DateTime month) { if (!verifiers.TryGetValue(month, out var verifier)) @@ -406,7 +416,7 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive s.PriorPurgedThroughSeq, s.PriorPurgedThroughHash is null ? null : Convert.FromHexString(s.PriorPurgedThroughHash)) : null; - verifier = new AuditChainVerifier(month, head, tombstones.GetValueOrDefault(month) ?? []); + verifier = new AuditChainVerifier(month, head, tombstones.GetValueOrDefault(month) ?? [], chainFrom); verifiers[month] = verifier; } @@ -427,7 +437,8 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive var month = AuditChainHasher.MonthOf(row.OccurredAtUtc); if (row.ChainSeq is null) { - unchained[month] = unchained.GetValueOrDefault(month) + 1; + var bucket = chainFrom is { } cutoff && row.OccurredAtUtc < cutoff ? preCutoff : unchained; + bucket[month] = bucket.GetValueOrDefault(month) + 1; VerifierFor(month); continue; } @@ -450,7 +461,7 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive var results = new List(); foreach (var (month, verifier) in verifiers) { - var chain = verifier.Complete(unchained.GetValueOrDefault(month)); + var chain = verifier.Complete(unchained.GetValueOrDefault(month), preCutoff.GetValueOrDefault(month)); var (anchor, detail) = CompareAnchor(states.GetValueOrDefault(month), anchors.GetValueOrDefault(month)); results.Add(new AuditArchiveChainVerification(chain, anchor, detail)); } @@ -509,6 +520,28 @@ public sealed partial class AuditLogPartitionArchive : IAuditLogArchive $"The purge anchor (ChainSeq {database.PurgedSeq}) is before this archive's chain end ({archived.LastSeq})."); } + /// The database's chain cutoff (ScadaBridge#139), or null when the chain has none. + private async Task ReadChainFromAsync(CancellationToken ct) + { + var conn = _context.Database.GetDbConnection(); + var openedHere = await AuditLogHashChain.OpenAsync(conn, ct).ConfigureAwait(false); + try + { + await using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT MIN(PurgedAtUtc) FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;"; + return await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false) is DateTime dt + ? DateTime.SpecifyKind(dt, DateTimeKind.Utc) + : null; + } + finally + { + if (openedHere) + { + await conn.CloseAsync().ConfigureAwait(false); + } + } + } + private async Task> ReadAnchorsAsync( IEnumerable months, CancellationToken ct) { diff --git a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Repositories/AuditLogRepository.cs b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Repositories/AuditLogRepository.cs index 96e6660b..4497871b 100644 --- a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Repositories/AuditLogRepository.cs +++ b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Repositories/AuditLogRepository.cs @@ -634,12 +634,15 @@ VALUES -- 3b. Hash chain (ScadaBridge#41): the months this partition held are gone as a -- whole, so record each month's current chain end as its checkpoint anchor -- (verification resumes after it; a late-arriving row chains on from it) - -- and drop the per-channel tombstones the anchor now subsumes. + -- and drop the per-channel tombstones the anchor now subsumes. The chain-start + -- anchor (ChainSeq 0, ScadaBridge#139) is NOT a purge tombstone: it records the + -- cutoff for the life of the database, so it stays. UPDATE dbo.AuditLogChainHead SET PurgedThroughSeq = LastSeq, PurgedThroughHash = LastHash, UpdatedAtUtc = SYSUTCDATETIME() WHERE $partition.pf_AuditLog_Month(CAST(ChainMonth AS datetime2(7))) = @partitionNumber; DELETE FROM dbo.AuditLogChainTombstone - WHERE $partition.pf_AuditLog_Month(CAST(ChainMonth AS datetime2(7))) = @partitionNumber; + WHERE $partition.pf_AuditLog_Month(CAST(ChainMonth AS datetime2(7))) = @partitionNumber + AND ChainSeq > 0; -- 4. Drop staging — the rows are discarded here. This is the purge. DROP TABLE dbo.[{stagingTableName}]; diff --git a/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/AuditLogHashChainServiceTests.cs b/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/AuditLogHashChainServiceTests.cs index 6595fb95..6b56c57e 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/AuditLogHashChainServiceTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/AuditLogHashChainServiceTests.cs @@ -21,8 +21,11 @@ public class AuditLogHashChainServiceTests public int Calls => _next; public Exception? Throw { get; init; } - public Task ChainPendingAsync(int batchSize, CancellationToken ct = default) + public List ChainBacklogSeen { get; } = []; + + public Task ChainPendingAsync(int batchSize, bool chainBacklog = false, CancellationToken ct = default) { + ChainBacklogSeen.Add(chainBacklog); if (Throw is not null) throw Throw; var i = _next++; return Task.FromResult(i < script.Length ? script[i] : new AuditChainBatchResult(0, false)); @@ -71,6 +74,33 @@ public class AuditLogHashChainServiceTests Assert.Equal(4, chain.Calls); // the 4th returns 0 and ends the tick } + /// + /// ScadaBridge#139: backlog chaining is off by default (owner decision, #107) and the + /// option reaches every chaining pass when set. + /// + /// The configured value, or null to leave the default. + /// A task that represents the asynchronous operation. + [Theory] + [InlineData(null)] + [InlineData(false)] + [InlineData(true)] + public async Task Tick_PassesChainBacklog_DefaultOff(bool? chainBacklog) + { + var options = new AuditLogHashChainOptions { Enabled = true }; + if (chainBacklog is { } value) + { + options.ChainBacklog = value; + } + + var chain = new ScriptedChain(new AuditChainBatchResult(4, false)); + using var service = Create(chain, options); + + await service.TickAsync(CancellationToken.None); + + Assert.Equal(2, chain.ChainBacklogSeen.Count); + Assert.All(chain.ChainBacklogSeen, seen => Assert.Equal(chainBacklog ?? false, seen)); + } + /// The peer node holding the lock ends the tick at once. /// A task that represents the asynchronous operation. [Fact] diff --git a/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/Commands/AuditVerifyChainCommandTests.cs b/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/Commands/AuditVerifyChainCommandTests.cs index ccfde2db..0bb934c2 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/Commands/AuditVerifyChainCommandTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/Commands/AuditVerifyChainCommandTests.cs @@ -111,6 +111,37 @@ public class AuditVerifyChainCommandTests Assert.Contains("--month", err); } + /// + /// ScadaBridge#139: the table report shows the chain cutoff and the pre-cutoff rows, and a + /// month whose backlog is out of scope still exits 0 (pre-cutoff rows are never a break). + /// + [Fact] + public void Render_Table_ShowsCutoffAndPreCutoffRows() + { + const string body = "{\"partition\":\"2026-11\",\"status\":\"Intact\",\"rowsVerified\":12,\"tombstonesApplied\":0," + + "\"purgedThroughSeq\":null,\"headSeq\":12,\"unchainedRows\":0,\"firstBreak\":null," + + "\"preCutoffRows\":3441,\"chainFromUtc\":\"2026-11-13T09:30:00.1234567Z\"}"; + using var output = new StringWriter(); + + var exit = AuditVerifyChainHelpers.Render(body, "table", output); + + Assert.Equal(0, exit); + Assert.Contains("Chain from : 2026-11-13T09:30:00.1234567Z", output.ToString()); + Assert.Contains("Pre-cutoff rows : 3441", output.ToString()); + } + + /// A legacy (whole-table) chain renders without a cutoff. + [Fact] + public void Render_Table_NoCutoff_SaysWholeTable() + { + const string body = "{\"partition\":\"2026-05\",\"status\":\"Intact\",\"rowsVerified\":1,\"unchainedRows\":0}"; + using var output = new StringWriter(); + + Assert.Equal(0, AuditVerifyChainHelpers.Render(body, "table", output)); + Assert.Contains("Chain from : - (whole table in scope)", output.ToString()); + Assert.Contains("Pre-cutoff rows : 0", output.ToString()); + } + /// Verifies that month validation accepts only values in yyyy-MM format. /// The month string to validate. /// Whether the month is expected to be valid. diff --git a/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditChainHeadStoreTests.cs b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditChainHeadStoreTests.cs index af79f3de..0ccbd588 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditChainHeadStoreTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditChainHeadStoreTests.cs @@ -283,7 +283,7 @@ public sealed class AuditChainHeadStoreTests : IClassFixtureA cutoff-started month links its first row to the anchor and verifies intact. + [Fact] + public void CutoffStartedChain_Verifies_FromTheAnchor() + { + var anchor = AuditChainHasher.ComputeCutoffAnchorHash(CutoffAt); + var (rows, head) = BuildChain(4, anchor); + + var result = VerifyCutoff(rows, head, anchor); + + Assert.Equal(AuditChainStatus.Intact, result.Status); + Assert.Equal(4, result.RowsVerified); + Assert.Equal(0, result.TombstonesApplied); // the anchor is not a purge tombstone + Assert.Equal(CutoffAt, result.ChainFromUtc); + Assert.Equal(7, result.PreCutoffRows); + } + + /// Without its anchor, a cutoff-started month fails at its first link. + [Fact] + public void CutoffStartedChain_WithoutItsAnchor_FailsAtTheFirstLink() + { + var anchor = AuditChainHasher.ComputeCutoffAnchorHash(CutoffAt); + var (rows, head) = BuildChain(4, anchor); + + var result = Verify(rows, head); + + Assert.Equal(AuditChainBreakKind.HashMismatch, result.FirstBreak!.Kind); + Assert.Equal(1, result.FirstBreak.ChainSeq); + } + + /// A different cutoff is a different anchor: the anchor hash binds the instant. + [Fact] + public void CutoffAnchorHash_BindsTheInstant() + { + Assert.False(AuditChainHasher.ComputeCutoffAnchorHash(CutoffAt).AsSpan() + .SequenceEqual(AuditChainHasher.ComputeCutoffAnchorHash(CutoffAt.AddTicks(1)))); + Assert.Equal(AuditChainHasher.HashLength, AuditChainHasher.ComputeCutoffAnchorHash(CutoffAt).Length); + } + + /// + /// The eligibility attack: a chained row re-dated before the cutoff is a + /// ChainedBeforeCutoff break even when its hash were recomputed to match. + /// + [Fact] + public void ChainedRowDatedBeforeTheCutoff_IsChainedBeforeCutoff() + { + var anchor = AuditChainHasher.ComputeCutoffAnchorHash(CutoffAt); + var (rows, _) = BuildChain(4, anchor); + + // Re-date row 3 before the cutoff and re-hash the chain from it (an attacker who + // rewrites the links): only the cutoff rule can still see it. + rows[2] = rows[2] with { OccurredAtUtc = CutoffAt.AddMinutes(-1) }; + var previous = rows[1].RowHash!; + for (var i = 2; i < rows.Count; i++) + { + var hash = AuditChainHasher.ComputeRowHash(previous, Month, i + 1, rows[i] with { ChainSeq = null, RowHash = null }); + rows[i] = rows[i] with { RowHash = hash }; + previous = hash; + } + + var result = VerifyCutoff(rows, new AuditChainHeadState(4, previous, null, null), anchor); + + Assert.Equal(AuditChainStatus.Broken, result.Status); + Assert.Equal(AuditChainBreakKind.ChainedBeforeCutoff, result.FirstBreak!.Kind); + Assert.Equal(3, result.FirstBreak.ChainSeq); + Assert.Equal(rows[2].EventId, result.FirstBreak.EventId); + } + + /// An anchor the caller found invalid fails the month even when every link verifies. + [Fact] + public void InvalidAnchor_IsCutoffAnchorInvalid() + { + var anchor = AuditChainHasher.ComputeCutoffAnchorHash(CutoffAt); + var (rows, head) = BuildChain(4, anchor); + var tombstones = new Dictionary { [AuditChainHasher.CutoffAnchorSeq] = anchor }; + var verifier = new AuditChainVerifier(Month, head, tombstones, CutoffAt, "anchor altered"); + foreach (var row in rows) + { + verifier.Accept(row); + } + + var result = verifier.Complete(0); + + Assert.Equal(AuditChainStatus.Broken, result.Status); + Assert.Equal(AuditChainBreakKind.CutoffAnchorInvalid, result.FirstBreak!.Kind); + Assert.Equal(AuditChainHasher.CutoffAnchorSeq, result.FirstBreak.ChainSeq); + } + + /// + /// After a partition purge the checkpoint anchor, not the cutoff anchor, is where + /// verification resumes; the cutoff anchor in the same tombstone set is ignored. + /// + [Fact] + public void PurgeCheckpoint_TakesPrecedenceOverTheCutoffAnchor() + { + var anchor = AuditChainHasher.ComputeCutoffAnchorHash(CutoffAt); + var (rows, _) = BuildChain(5, anchor); + var head = new AuditChainHeadState(5, rows[4].RowHash!, 3, rows[2].RowHash!); + + var result = VerifyCutoff(rows.Skip(3), head, anchor); + + Assert.Equal(AuditChainStatus.Intact, result.Status); + Assert.Equal(2, result.RowsVerified); + } + + private static AuditChainVerificationResult VerifyCutoff(IEnumerable rows, AuditChainHeadState head, byte[] anchor) + { + var tombstones = new Dictionary { [AuditChainHasher.CutoffAnchorSeq] = anchor }; + var verifier = new AuditChainVerifier(Month, head, tombstones, CutoffAt); + foreach (var row in rows) + { + if (!verifier.Accept(row)) + { + break; + } + } + + return verifier.Complete(0, preCutoffRows: 7); + } + private static (List Rows, AuditChainHeadState Head) BuildChain(int count) + => BuildChain(count, AuditChainHasher.GenesisHash.ToArray()); + + private static (List Rows, AuditChainHeadState Head) BuildChain(int count, byte[] genesis) { var rows = new List(); - var previous = AuditChainHasher.GenesisHash.ToArray(); + var previous = genesis; for (var seq = 1; seq <= count; seq++) { var row = NewRow(seq); diff --git a/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditLogArchiveCleanupTests.cs b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditLogArchiveCleanupTests.cs index 3f357d07..66bb58dd 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditLogArchiveCleanupTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditLogArchiveCleanupTests.cs @@ -329,7 +329,7 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture +/// SQL Server integration tests for the hash-chain cutoff (ScadaBridge#139): enabling the +/// chain on a database with history chains only rows from the enablement instant on (owner +/// decision on #107, 2026-11-13), verification treats the backlog as out of scope, the +/// tamper cases are still caught, moving a row (or the cutoff) across the cutoff is caught, +/// both purge paths keep the chain verifiable across the cutoff, and both central nodes +/// settle on one cutoff. +/// +/// +/// The cutoff is one row per database, so every test starts from an empty AuditLog and empty +/// chain tables (). xunit runs a class's tests sequentially. +/// +public sealed class AuditLogHashChainCutoffTests : IClassFixture, IDisposable +{ + private static readonly DateTime Backlog = new(2026, 2, 1, 0, 0, 0, DateTimeKind.Utc); + private static readonly DateTime CutoffMonth = new(2026, 3, 1, 0, 0, 0, DateTimeKind.Utc); + private static readonly DateTime After = new(2026, 4, 1, 0, 0, 0, DateTimeKind.Utc); + private static readonly DateTime Cutoff = new(2026, 3, 15, 12, 0, 0, DateTimeKind.Utc); + + private readonly MsSqlMigrationFixture _fixture; + private readonly DirectoryInfo _archiveRoot = Directory.CreateTempSubdirectory("auditlog-cutoff-archive-"); + + /// Creates the test class with its own migrated SQL Server database. + /// The per-class migration fixture. + public AuditLogHashChainCutoffTests(MsSqlMigrationFixture fixture) => _fixture = fixture; + + /// + public void Dispose() => _archiveRoot.Delete(recursive: true); + + /// + /// The owner's decision end to end: a backlog exists, the chain is enabled, only rows dated + /// from the cutoff on are chained, the backlog is untouched, and every month verifies — + /// the backlog as pre-cutoff rows, never as unchained rows or a break. + /// + /// A task that represents the asynchronous operation. + [SkippableFact] + public async Task FirstEnablement_ChainsOnlyFromTheCutoff_BacklogUntouched_VerifiesIntact() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + await ResetAsync(); + var backlog = Rows(Backlog.AddDays(3), 4).Concat(Rows(CutoffMonth.AddDays(2), 3)).ToList(); + await InsertAsync(backlog); + + await DrainAsync(); // first enablement: records the anchor at Cutoff, chains nothing + + Assert.Equal(Cutoff, await AnchorCutoffAsync()); + Assert.Equal(0L, await ScalarAsync("SELECT COUNT_BIG(*) FROM dbo.AuditLog WHERE ChainSeq IS NOT NULL;")); + Assert.Equal(0L, await ScalarAsync("SELECT COUNT_BIG(*) FROM dbo.AuditLogChainHead;")); + + var post = Rows(Cutoff.AddMinutes(1), 5).Concat(Rows(After.AddDays(1), 2)).ToList(); + await InsertAsync(post); + await DrainAsync(); + + // Every backlog row stays exactly as it was: unchained. + foreach (var e in backlog) + { + Assert.Equal(DBNull.Value, await ScalarAsync($"SELECT ChainSeq FROM dbo.AuditLog WHERE EventId = '{e.EventId}';")); + } + + var march = await VerifyAsync(CutoffMonth); + Assert.Equal(AuditChainStatus.Intact, march.Status); + Assert.Equal(5, march.RowsVerified); + Assert.Equal(0, march.UnchainedRows); + Assert.Equal(3, march.PreCutoffRows); + Assert.Equal(Cutoff, march.ChainFromUtc); + + var february = await VerifyAsync(Backlog); + Assert.Equal(AuditChainStatus.NotChained, february.Status); + Assert.Equal(0, february.UnchainedRows); + Assert.Equal(4, february.PreCutoffRows); + + var april = await VerifyAsync(After); + Assert.Equal(AuditChainStatus.Intact, april.Status); + Assert.Equal(2, april.RowsVerified); + Assert.Equal(0, april.PreCutoffRows); + + // A late site row dated before the cutoff that reaches central after enablement is + // keyed on OccurredAtUtc: it stays out of scope (documented limit), not pending. + await InsertAsync(Rows(Cutoff.AddHours(-2), 1)); + await DrainAsync(); + march = await VerifyAsync(CutoffMonth); + Assert.Equal(AuditChainStatus.Intact, march.Status); + Assert.Equal(4, march.PreCutoffRows); + Assert.Equal(0, march.UnchainedRows); + } + + /// + /// The cutoff month's first row links to the anchor, not to 32 zero bytes, and the + /// ordinary tamper cases are still caught after a cutoff start: edit, delete, tail delete. + /// + /// A task that represents the asynchronous operation. + [SkippableFact] + public async Task TamperCases_AreStillDetected_AfterACutoffStart() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + var events = await SeedCutoffChainAsync(postRows: 6); + Assert.Equal(AuditChainStatus.Intact, (await VerifyAsync(CutoffMonth)).Status); + + // Edit the FIRST link (the one hashed against the anchor). + await ExecAsync($"UPDATE dbo.AuditLog SET Target = N'forged' WHERE EventId = '{events[0]}';"); + var edited = await VerifyAsync(CutoffMonth); + Assert.Equal(AuditChainStatus.Broken, edited.Status); + Assert.Equal(AuditChainBreakKind.HashMismatch, edited.FirstBreak!.Kind); + Assert.Equal(1, edited.FirstBreak.ChainSeq); + Assert.Equal(events[0], edited.FirstBreak.EventId); + await ExecAsync($"UPDATE dbo.AuditLog SET Target = N'erp' WHERE EventId = '{events[0]}';"); + Assert.Equal(AuditChainStatus.Intact, (await VerifyAsync(CutoffMonth)).Status); + + var middleSeq = await SeqOfAsync(events[2]); + await ExecAsync($"DELETE FROM dbo.AuditLog WHERE EventId = '{events[2]}';"); + var deleted = await VerifyAsync(CutoffMonth); + Assert.Equal(AuditChainStatus.Broken, deleted.Status); + Assert.Equal(AuditChainBreakKind.MissingRow, deleted.FirstBreak!.Kind); + Assert.Equal(middleSeq, deleted.FirstBreak.ChainSeq); + + await ResetAsync(); + events = await SeedCutoffChainAsync(postRows: 4); + await ExecAsync($"DELETE FROM dbo.AuditLog WHERE EventId = '{events[^1]}';"); + var tail = await VerifyAsync(CutoffMonth); + Assert.Equal(AuditChainStatus.Broken, tail.Status); + Assert.Equal(AuditChainBreakKind.HeadMismatch, tail.FirstBreak!.Kind); + } + + /// + /// The cutoff-eligibility attack: move a chained row's timestamp back before the cutoff to + /// hide it among the unprotected backlog rows. Caught with its ChainSeq left in place + /// (), and with its link cleared too + /// (its sequence number goes missing). Moving it into an earlier month is caught in both months. + /// + /// A task that represents the asynchronous operation. + [SkippableFact] + public async Task ChainedRowMovedBeforeTheCutoff_IsDetected() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + var events = await SeedCutoffChainAsync(postRows: 5); + var victim = events[2]; + var victimSeq = await SeqOfAsync(victim); + + // 1. Timestamp moved back across the cutoff, link left in place. + await ExecAsync($"UPDATE dbo.AuditLog SET OccurredAtUtc = '{Cutoff.AddHours(-1):yyyy-MM-dd HH:mm:ss}' WHERE EventId = '{victim}';"); + var moved = await VerifyAsync(CutoffMonth); + Assert.Equal(AuditChainStatus.Broken, moved.Status); + Assert.Equal(AuditChainBreakKind.ChainedBeforeCutoff, moved.FirstBreak!.Kind); + Assert.Equal(victim, moved.FirstBreak.EventId); + Assert.Equal(victimSeq, moved.FirstBreak.ChainSeq); + + // 2. ...and its link cleared, so it looks like any other pre-cutoff row. + await ExecAsync($"UPDATE dbo.AuditLog SET ChainSeq = NULL, RowHash = NULL WHERE EventId = '{victim}';"); + var hidden = await VerifyAsync(CutoffMonth); + Assert.Equal(AuditChainStatus.Broken, hidden.Status); + Assert.Equal(AuditChainBreakKind.MissingRow, hidden.FirstBreak!.Kind); + Assert.Equal(victimSeq, hidden.FirstBreak.ChainSeq); + Assert.Equal(4, hidden.PreCutoffRows); // the 3 backlog rows + the hidden one + + // 3. Moved into the previous (backlog) month with its link: caught in both months. + await ResetAsync(); + events = await SeedCutoffChainAsync(postRows: 5); + victim = events[1]; + await ExecAsync($"UPDATE dbo.AuditLog SET OccurredAtUtc = '{Backlog.AddDays(9):yyyy-MM-dd HH:mm:ss}' WHERE EventId = '{victim}';"); + Assert.Equal(AuditChainBreakKind.MissingRow, (await VerifyAsync(CutoffMonth)).FirstBreak!.Kind); + var february = await VerifyAsync(Backlog); + Assert.Equal(AuditChainStatus.Broken, february.Status); + Assert.Equal(AuditChainBreakKind.ChainedBeforeCutoff, february.FirstBreak!.Kind); + Assert.Equal(victim, february.FirstBreak.EventId); + } + + /// + /// The other half of the eligibility attack: moving the cutoff itself. Editing the anchor's + /// cutoff, re-hashing the anchor to match, deleting it, or adding a second anchor are each + /// reported, so the backlog/scope boundary cannot be shifted silently. + /// + /// A task that represents the asynchronous operation. + [SkippableFact] + public async Task CutoffAnchorTampering_IsDetected() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + await SeedCutoffChainAsync(postRows: 4); + var later = Cutoff.AddDays(5); + + // Cutoff moved forward, anchor hash left alone. + await ExecAsync($"UPDATE dbo.AuditLogChainTombstone SET PurgedAtUtc = '{later:yyyy-MM-dd HH:mm:ss}' WHERE ChainSeq = 0;"); + var moved = await VerifyAsync(CutoffMonth); + Assert.Equal(AuditChainStatus.Broken, moved.Status); + Assert.Equal(AuditChainBreakKind.CutoffAnchorInvalid, moved.FirstBreak!.Kind); + Assert.Equal(AuditChainStatus.Broken, (await VerifyAsync(After)).Status); // every month sees it + + // Cutoff moved forward AND the anchor re-hashed to match: the chained rows now precede it. + await SetAnchorAsync(later, AuditChainHasher.ComputeCutoffAnchorHash(later)); + var rehashed = await VerifyAsync(CutoffMonth); + Assert.Equal(AuditChainStatus.Broken, rehashed.Status); + Assert.Equal(AuditChainBreakKind.ChainedBeforeCutoff, rehashed.FirstBreak!.Kind); + + // Anchor deleted: the cutoff month's first link no longer starts from it. + await ExecAsync("DELETE FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;"); + var deleted = await VerifyAsync(CutoffMonth); + Assert.Equal(AuditChainStatus.Broken, deleted.Status); + Assert.Equal(AuditChainBreakKind.HashMismatch, deleted.FirstBreak!.Kind); + Assert.Equal(1, deleted.FirstBreak.ChainSeq); + + // Restored → intact (positive control); then a second anchor is reported. + await SetAnchorAsync(Cutoff, AuditChainHasher.ComputeCutoffAnchorHash(Cutoff)); + Assert.Equal(AuditChainStatus.Intact, (await VerifyAsync(CutoffMonth)).Status); + var earlier = Backlog.AddDays(1); + await ExecAsync( + "INSERT INTO dbo.AuditLogChainTombstone (ChainMonth, ChainSeq, RowHash, EventId, PurgedAtUtc) " + + $"VALUES ('{Backlog:yyyy-MM-dd}', 0, 0x{Convert.ToHexString(AuditChainHasher.ComputeCutoffAnchorHash(earlier))}, NEWID(), '{earlier:yyyy-MM-dd HH:mm:ss}');"); + var twoAnchors = await VerifyAsync(CutoffMonth); + Assert.Equal(AuditChainStatus.Broken, twoAnchors.Status); + Assert.Equal(AuditChainBreakKind.CutoffAnchorInvalid, twoAnchors.FirstBreak!.Kind); + } + + /// + /// Both purge paths across the cutoff: a per-channel purge that deletes backlog and chained + /// rows alike leaves an intact chain; the head store and an export of it agree (the anchor is + /// not counted as a purge tombstone); the partition switch-out of the cutoff month keeps the + /// anchor and anchors the month; the archive of that month verifies from its own files; and + /// the chainer still honours the cutoff afterwards. + /// + /// A task that represents the asynchronous operation. + [SkippableFact] + public async Task PurgeAcrossTheCutoff_KeepsEveryChainVerifiable() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + await ResetAsync(); + var preDb = Rows(CutoffMonth.AddDays(1), 2, AuditChannel.DbOutbound); + var preApi = Rows(CutoffMonth.AddDays(2), 2); + await InsertAsync(preDb.Concat(preApi)); + await DrainAsync(); + var postDb = Rows(Cutoff.AddHours(1), 3, AuditChannel.DbOutbound); + var postApi = Rows(Cutoff.AddHours(2), 3); + await InsertAsync(postDb.Concat(postApi)); + await DrainAsync(); + + // Per-channel purge with a threshold past the cutoff: 2 backlog + 3 chained DbOutbound rows. + await using (var ctx = CreateContext()) + { + var purged = await new AuditLogRepository(ctx).PurgeChannelOlderThanAsync( + nameof(AuditChannel.DbOutbound), Cutoff.AddDays(1), batchSize: 2); + Assert.Equal(5, purged); + } + + var afterChannelPurge = await VerifyAsync(CutoffMonth); + Assert.Equal(AuditChainStatus.Intact, afterChannelPurge.Status); + Assert.Equal(3, afterChannelPurge.TombstonesApplied); + Assert.Equal(3, afterChannelPurge.RowsVerified); + Assert.Equal(2, afterChannelPurge.PreCutoffRows); + + // Head store + export: the anchor is not a purge tombstone; an export verifies as Matches. + await using (var ctx = CreateContext()) + { + var head = Assert.Single(await new AuditLogHashChain(ctx).ReadHeadsAsync()); + Assert.Equal(3, head.Tombstones); + Assert.Equal(3, head.ChainedRows); + var exported = await new AuditLogHashChain(ctx).VerifyExportedHeadsAsync([Export(head)]); + Assert.True(exported.Verified); + Assert.Equal(AuditChainHeadStatus.Matches, Assert.Single(exported.Checks).Status); + } + + // Archive the cutoff month, then switch it out through the archive guard. + AuditArchiveManifest manifest; + await using (var ctx = CreateContext()) + { + manifest = await Archive(ctx).ArchivePartitionAsync(CutoffMonth); + await Archive(ctx).PurgeArchivedPartitionAsync(manifest); + } + + Assert.Equal(Cutoff, await AnchorCutoffAsync()); // the anchor survives the switch-out + var switched = await VerifyAsync(CutoffMonth); + Assert.Equal(AuditChainStatus.Intact, switched.Status); + Assert.Equal(6, switched.PurgedThroughSeq); + Assert.Equal(0, switched.PreCutoffRows); + + await using (var ctx = CreateContext()) + { + var archived = await Archive(ctx).VerifyAsync(manifest.Name); + Assert.True(archived.Verified, archived.FileProblem); + var chain = Assert.Single(archived.Chains).Chain; + Assert.Equal(AuditChainStatus.Intact, chain.Status); + Assert.Equal(3, chain.RowsVerified); + Assert.Equal(3, chain.TombstonesApplied); + Assert.Equal(2, chain.PreCutoffRows); // the archive classifies its backlog rows too + Assert.Equal(0, chain.UnchainedRows); + Assert.Equal(Cutoff, chain.ChainFromUtc); + } + + // After the purge the cutoff still holds: a late backlog row stays out, a late post-cutoff row chains on. + var lateBacklog = Rows(Backlog.AddDays(4), 1); + var latePost = Rows(Cutoff.AddHours(5), 1); + await InsertAsync(lateBacklog.Concat(latePost)); + await DrainAsync(); + Assert.Equal(DBNull.Value, await ScalarAsync($"SELECT ChainSeq FROM dbo.AuditLog WHERE EventId = '{lateBacklog[0].EventId}';")); + var late = await VerifyAsync(CutoffMonth); + Assert.Equal(AuditChainStatus.Intact, late.Status); + Assert.Equal(7, late.HeadSeq); + Assert.Equal(1, (await VerifyAsync(Backlog)).PreCutoffRows); + } + + /// + /// Both central nodes settle on ONE cutoff: two chainers race the first enablement on a + /// pristine database with the real database clock, exactly one anchor is written (inside the + /// database clock window of the race), and a node configured to chain the backlog that runs + /// after the decision cannot override it. + /// + /// A task that represents the asynchronous operation. + [SkippableFact] + public async Task BothNodes_AgreeOnOneCutoff() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + await ResetAsync(); + var backlog = Rows(CutoffMonth.AddDays(1), 5); + await InsertAsync(backlog); + + var before = await ScalarAsync("SELECT SYSUTCDATETIME();"); + var start = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + async Task Node() + { + await start.Task; + for (var i = 0; i < 20; i++) + { + await using var ctx = CreateContext(); + await new AuditLogHashChain(ctx).ChainPendingAsync(100); // production clock + } + } + + var nodes = new[] { Task.Run(Node), Task.Run(Node) }; + start.SetResult(); + await Task.WhenAll(nodes); + var after = await ScalarAsync("SELECT SYSUTCDATETIME();"); + + Assert.Equal(1L, await ScalarAsync("SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;")); + var cutoff = await AnchorCutoffAsync(); + Assert.InRange(cutoff, before, after); + + // The peer node configured with ChainBacklog=true runs after the decision: still no backlog chained. + await using (var ctx = CreateContext()) + { + Assert.Equal(0, (await new AuditLogHashChain(ctx).ChainPendingAsync(100, chainBacklog: true)).RowsChained); + } + + Assert.Equal(0L, await ScalarAsync("SELECT COUNT_BIG(*) FROM dbo.AuditLog WHERE ChainSeq IS NOT NULL;")); + Assert.Equal(1L, await ScalarAsync("SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;")); + var verified = await VerifyAsync(AuditChainHasher.MonthOf(cutoff)); + Assert.Equal(cutoff, verified.ChainFromUtc); + } + + /// + /// The old behaviour stays reachable and unchanged: ChainBacklog=true on a pristine database + /// writes no anchor and chains everything, and a database that already holds a chain (legacy + /// enablement, no anchor) keeps chaining every row even with ChainBacklog=false. + /// + /// A task that represents the asynchronous operation. + [SkippableFact] + public async Task ChainBacklog_AndLegacyChains_KeepChainingEverything() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + await ResetAsync(); + await InsertAsync(Rows(Backlog.AddDays(1), 3)); + await DrainAsync(chainBacklog: true); + Assert.Equal(0L, await ScalarAsync("SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;")); + Assert.Equal(3L, await ScalarAsync("SELECT COUNT_BIG(*) FROM dbo.AuditLog WHERE ChainSeq IS NOT NULL;")); + + // Now a node on the default (no backlog) sees an existing chain: no anchor, old rows still chain. + await InsertAsync(Rows(Backlog.AddDays(10), 2)); + await DrainAsync(); + Assert.Equal(0L, await ScalarAsync("SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;")); + var february = await VerifyAsync(Backlog); + Assert.Equal(AuditChainStatus.Intact, february.Status); + Assert.Equal(5, february.RowsVerified); + Assert.Null(february.ChainFromUtc); + Assert.Equal(0, february.PreCutoffRows); + } + + /// + /// The hardened maintenance login (scadabridge_audit_chainer + scadabridge_audit_purger, + /// ScadaBridge#107 item 3) can write the chain-start anchor: no extra grant, no migration. + /// + /// A task that represents the asynchronous operation. + [SkippableFact] + public async Task MaintenanceLogin_CanWriteTheAnchor() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + await ResetAsync(); + var login = "sbc_" + Guid.NewGuid().ToString("N")[..10]; + const string password = "Sb_W22_cutoff#2026"; + await ExecAsync($@" +CREATE LOGIN [{login}] WITH PASSWORD = N'{password}', CHECK_POLICY = OFF; +CREATE USER [{login}] FOR LOGIN [{login}]; +ALTER ROLE scadabridge_audit_chainer ADD MEMBER [{login}]; +ALTER ROLE scadabridge_audit_purger ADD MEMBER [{login}];"); + try + { + await InsertAsync(Rows(Backlog.AddDays(2), 2)); + var maintenance = new SqlConnectionStringBuilder(_fixture.ConnectionString) + { + IntegratedSecurity = false, + UserID = login, + Password = password, + Pooling = false, + }.ConnectionString; + await using (var ctx = new ScadaBridgeDbContext( + new DbContextOptionsBuilder().UseSqlServer(maintenance).Options)) + { + var chain = new AuditLogHashChain(ctx) { AnchorClock = () => Cutoff }; + Assert.Equal(0, (await chain.ChainPendingAsync(100)).RowsChained); + } + + Assert.Equal(Cutoff, await AnchorCutoffAsync()); + } + finally + { + SqlConnection.ClearAllPools(); + await ExecAsync($@" +DECLARE @kill nvarchar(max) = N''; +SELECT @kill = @kill + N'KILL ' + CAST(session_id AS nvarchar(10)) + N';' FROM sys.dm_exec_sessions WHERE login_name = N'{login}'; +EXEC sp_executesql @kill; +DROP USER [{login}]; +DROP LOGIN [{login}];"); + } + } + + // ── helpers ────────────────────────────────────────────────────────────── + + /// + /// Backlog in February and early March, first enablement at , then + /// rows after it, all chained. Returns the post-cutoff EventIds in chain order. + /// + private async Task> SeedCutoffChainAsync(int postRows) + { + await ResetAsync(); + await InsertAsync(Rows(Backlog.AddDays(3), 2).Concat(Rows(CutoffMonth.AddDays(2), 3))); + await DrainAsync(); + var post = Rows(Cutoff.AddMinutes(1), postRows); + await InsertAsync(post); + await DrainAsync(); + return post.Select(e => e.EventId).ToList(); + } + + private async Task ResetAsync() + { + if (!_fixture.Available) + { + return; + } + + await ExecAsync("DELETE FROM dbo.AuditLog; DELETE FROM dbo.AuditLogChainHead; DELETE FROM dbo.AuditLogChainTombstone;"); + } + + private async Task DrainAsync(bool chainBacklog = false) + { + await using var ctx = CreateContext(); + var chain = new AuditLogHashChain(ctx) { AnchorClock = () => Cutoff }; + for (var i = 0; i < 1000; i++) + { + var r = await chain.ChainPendingAsync(500, chainBacklog); + if (r.RowsChained == 0 && !r.LockBusy) + { + return; + } + } + + throw new InvalidOperationException("Chain did not drain."); + } + + private async Task VerifyAsync(DateTime month) + { + await using var ctx = CreateContext(); + return await new AuditLogHashChain(ctx).VerifyMonthAsync(month); + } + + private async Task AnchorCutoffAsync() + => DateTime.SpecifyKind( + await ScalarAsync("SELECT PurgedAtUtc FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;"), + DateTimeKind.Utc); + + private async Task SetAnchorAsync(DateTime chainFrom, byte[] hash) + { + await ExecAsync("DELETE FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;"); + await ExecAsync( + "INSERT INTO dbo.AuditLogChainTombstone (ChainMonth, ChainSeq, RowHash, EventId, PurgedAtUtc) " + + $"VALUES ('{AuditChainHasher.MonthOf(chainFrom):yyyy-MM-dd}', 0, 0x{Convert.ToHexString(hash)}, '{AuditChainHasher.CutoffAnchorEventId}', '{chainFrom:yyyy-MM-dd HH:mm:ss}');"); + } + + private async Task SeqOfAsync(Guid eventId) + => await ScalarAsync($"SELECT ChainSeq FROM dbo.AuditLog WHERE EventId = '{eventId}';"); + + private async Task InsertAsync(IEnumerable events) + { + await using var ctx = CreateContext(); + var repo = new AuditLogRepository(ctx); + foreach (var e in events) + { + await repo.InsertIfNotExistsAsync(e); + } + } + + private async Task ExecAsync(string sql) + { + await using var conn = new SqlConnection(_fixture.ConnectionString); + await conn.OpenAsync(); + await using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + await cmd.ExecuteNonQueryAsync(); + } + + private async Task ScalarAsync(string sql) + { + await using var conn = new SqlConnection(_fixture.ConnectionString); + await conn.OpenAsync(); + await using var cmd = conn.CreateCommand(); + cmd.CommandText = sql; + var value = await cmd.ExecuteScalarAsync(); + return value is DateTime dt && typeof(T) == typeof(DateTime) + ? (T)(object)DateTime.SpecifyKind(dt, DateTimeKind.Utc) + : (T)value!; + } + + private AuditLogPartitionArchive Archive(ScadaBridgeDbContext ctx) => new(ctx, new AuditLogArchiveOptions + { + Enabled = true, + Directory = _archiveRoot.FullName, + RowGroupSize = 1_000, + }); + + private static AuditChainHeadRecord Export(AuditChainHeadSnapshot head) => new( + AuditChainHeadRecord.CurrentVersion, + AuditChainHasher.PartitionLabel(head.Month), + head.LastSeq, + AuditChainHeadComparer.ToHex(head.LastHash), + head.PurgedThroughSeq, + head.PurgedThroughHash is null ? null : AuditChainHeadComparer.ToHex(head.PurgedThroughHash), + head.ChainedRows, + head.Tombstones, + head.UpdatedAtUtc, + DateTime.UtcNow, + "central-a"); + + private ScadaBridgeDbContext CreateContext() + => new(new DbContextOptionsBuilder().UseSqlServer(_fixture.ConnectionString).Options); + + private static List Rows(DateTime first, int count, AuditChannel channel = AuditChannel.ApiOutbound) + => Enumerable.Range(0, count).Select(i => ScadaBridgeAuditEventFactory.Create( + channel: channel, + kind: channel == AuditChannel.DbOutbound ? AuditKind.DbWrite : AuditKind.ApiCall, + status: AuditStatus.Delivered, + occurredAtUtc: first.AddMinutes(i), + target: "erp", + sourceNode: "central-a", + sourceSiteId: "cutoff-test")).ToList(); +} diff --git a/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditLogHashChainTests.cs b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditLogHashChainTests.cs index 4ff728e0..57e0ff76 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditLogHashChainTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditLogHashChainTests.cs @@ -47,7 +47,7 @@ public class AuditLogHashChainTests : IClassFixture while (!stop.IsCancellationRequested) { await using var ctx = CreateContext(); - var r = await new AuditLogHashChain(ctx).ChainPendingAsync(17); + var r = await new AuditLogHashChain(ctx).ChainPendingAsync(17, chainBacklog: true); if (r.LockBusy) Interlocked.Increment(ref lockBusySeen); if (r.RowsChained == 0) await Task.Delay(5); } @@ -272,7 +272,7 @@ public class AuditLogHashChainTests : IClassFixture var chain = new AuditLogHashChain(ctx); for (var i = 0; i < 1000; i++) { - var r = await chain.ChainPendingAsync(500); + var r = await chain.ChainPendingAsync(500, chainBacklog: true); if (r.RowsChained == 0 && !r.LockBusy) { return; diff --git a/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditLogPartitionArchiveTests.cs b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditLogPartitionArchiveTests.cs index 3a61776a..77fb1c61 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditLogPartitionArchiveTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditLogPartitionArchiveTests.cs @@ -393,7 +393,7 @@ public sealed class AuditLogPartitionArchiveTests : IClassFixture(() => new AuditLogHashChain(writerCtx).ChainPendingAsync(100)); + await Assert.ThrowsAsync(() => new AuditLogHashChain(writerCtx).ChainPendingAsync(100, chainBacklog: true)); await Assert.ThrowsAsync(() => new AuditLogRepository(writerCtx).PurgeChannelOlderThanAsync("DbOutbound", month.AddDays(1), 100)); } @@ -133,7 +133,7 @@ IF SUSER_ID(N'{MaintenanceLogin}') IS NOT NULL DROP LOGIN [{MaintenanceLogin}];" Assert.Equal(WriterLogin, await LoginNameAsync(scope.ServiceProvider.GetRequiredService())); var chain = scope.ServiceProvider.GetRequiredService(); - Assert.Equal(5, (await chain.ChainPendingAsync(100)).RowsChained); + Assert.Equal(5, (await chain.ChainPendingAsync(100, chainBacklog: true)).RowsChained); Assert.Equal(AuditChainStatus.Intact, (await chain.VerifyMonthAsync(month)).Status); var head = Assert.Single(await scope.ServiceProvider.GetRequiredService().ReadHeadsAsync()); Assert.Equal(5, head.LastSeq); diff --git a/tests/ZB.MOM.WW.ScadaBridge.ManagementService.Tests/VerifyAuditChainCommandTests.cs b/tests/ZB.MOM.WW.ScadaBridge.ManagementService.Tests/VerifyAuditChainCommandTests.cs index f559c2cb..482c9e08 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.ManagementService.Tests/VerifyAuditChainCommandTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.ManagementService.Tests/VerifyAuditChainCommandTests.cs @@ -50,6 +50,27 @@ public class VerifyAuditChainCommandTests : TestKit new DateTime(2026, 5, 1, 0, 0, 0, DateTimeKind.Utc), Arg.Any()); } + /// + /// ScadaBridge#139: the cutoff, the pre-cutoff row count and the new break kinds reach the + /// wire (the CLI renders them; the kinds as names like every other kind). + /// + [Fact] + public void CutoffFields_AndCutoffBreakKind_ReachTheWire() + { + var chainFrom = new DateTime(2026, 11, 13, 9, 30, 0, DateTimeKind.Utc); + var brk = new AuditChainBreak("2026-11", 4, Guid.NewGuid(), AuditChainBreakKind.ChainedBeforeCutoff, "moved back"); + _chain.VerifyMonthAsync(Arg.Any(), Arg.Any()) + .Returns(new AuditChainVerificationResult("2026-11", AuditChainStatus.Broken, 3, 0, null, 9, 0, brk, 3441, chainFrom)); + + CreateActor().Tell(Envelope(new VerifyAuditChainCommand("2026-11"), [], "Viewer")); + + var response = ExpectMsg(TimeSpan.FromSeconds(5)); + using var json = JsonDocument.Parse(response.JsonData); + Assert.Equal(3441, json.RootElement.GetProperty("preCutoffRows").GetInt64()); + Assert.Equal(chainFrom, json.RootElement.GetProperty("chainFromUtc").GetDateTime().ToUniversalTime()); + Assert.Equal("ChainedBeforeCutoff", json.RootElement.GetProperty("firstBreak").GetProperty("kind").GetString()); + } + /// A Designer/Deployer without an OperationalAudit role is refused. [Fact] public void NonAuditRole_IsUnauthorized()