feat(audit): hash-chain cutoff - chain only rows written from first enablement, no backlog (#139)

Owner decision on #107 (2026-11-13): enabling AuditLog:HashChain must not
chain the historical backlog. main chained every unchained row oldest-first.

- The first chaining pass on a pristine database (no head, no chained row)
  writes a chain-start anchor: AuditLogChainTombstone (cutoff month,
  ChainSeq 0), PurgedAtUtc = DB SYSUTCDATETIME(), RowHash =
  SHA-256("ScadaBridge.AuditLog.ChainCutoff.v1" || YYYY-MM || ticks).
  Written once under the exclusive chain lock, so both centrals agree
  whatever their clocks/config. No migration: existing table + grants.
- Only rows with OccurredAtUtc >= cutoff are chained. The anchor hash is
  the cutoff month's genesis, binding the cutoff into the chain.
- AuditLog:HashChain:ChainBacklog (default false) restores chain-everything,
  read only on the very first enablement. Legacy chains (heads, no anchor)
  keep chaining everything.
- verify-chain splits unchained rows into UnchainedRows vs PreCutoffRows
  (+ ChainFromUtc); new breaks ChainedBeforeCutoff (row re-dated before the
  cutoff, or cutoff moved forward) and CutoffAnchorInvalid (anchor hash /
  month / multiplicity). Archive verify uses the anchor from its own
  tombstone file or the DB.
- Partition switch-out keeps the anchor (deletes ChainSeq > 0 only); the
  head store / export / verify-heads tombstone count excludes it.
- CLI verify-chain table prints Chain from + Pre-cutoff rows.

Tests: AuditLogHashChainCutoffTests (7, MSSQL), 6 verifier unit tests,
service/CLI/management wire tests; 9 mutations all killed.

Claude-Session: https://claude.ai/code/session_01GT67ACmpMR7b9LGRPLijMi
This commit is contained in:
Joseph Doherty
2026-11-13 03:14:03 -05:00
parent 712c5117ee
commit 52da02a76a
21 changed files with 1208 additions and 49 deletions
@@ -21,8 +21,11 @@ public class AuditLogHashChainServiceTests
public int Calls => _next;
public Exception? Throw { get; init; }
public Task<AuditChainBatchResult> ChainPendingAsync(int batchSize, CancellationToken ct = default)
public List<bool> ChainBacklogSeen { get; } = [];
public Task<AuditChainBatchResult> ChainPendingAsync(int batchSize, bool chainBacklog = false, CancellationToken ct = default)
{
ChainBacklogSeen.Add(chainBacklog);
if (Throw is not null) throw Throw;
var i = _next++;
return Task.FromResult(i < script.Length ? script[i] : new AuditChainBatchResult(0, false));
@@ -71,6 +74,33 @@ public class AuditLogHashChainServiceTests
Assert.Equal(4, chain.Calls); // the 4th returns 0 and ends the tick
}
/// <summary>
/// ScadaBridge#139: backlog chaining is off by default (owner decision, #107) and the
/// option reaches every chaining pass when set.
/// </summary>
/// <param name="chainBacklog">The configured value, or null to leave the default.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
[Theory]
[InlineData(null)]
[InlineData(false)]
[InlineData(true)]
public async Task Tick_PassesChainBacklog_DefaultOff(bool? chainBacklog)
{
var options = new AuditLogHashChainOptions { Enabled = true };
if (chainBacklog is { } value)
{
options.ChainBacklog = value;
}
var chain = new ScriptedChain(new AuditChainBatchResult(4, false));
using var service = Create(chain, options);
await service.TickAsync(CancellationToken.None);
Assert.Equal(2, chain.ChainBacklogSeen.Count);
Assert.All(chain.ChainBacklogSeen, seen => Assert.Equal(chainBacklog ?? false, seen));
}
/// <summary>The peer node holding the lock ends the tick at once.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[Fact]
@@ -111,6 +111,37 @@ public class AuditVerifyChainCommandTests
Assert.Contains("--month", err);
}
/// <summary>
/// ScadaBridge#139: the table report shows the chain cutoff and the pre-cutoff rows, and a
/// month whose backlog is out of scope still exits 0 (pre-cutoff rows are never a break).
/// </summary>
[Fact]
public void Render_Table_ShowsCutoffAndPreCutoffRows()
{
const string body = "{\"partition\":\"2026-11\",\"status\":\"Intact\",\"rowsVerified\":12,\"tombstonesApplied\":0," +
"\"purgedThroughSeq\":null,\"headSeq\":12,\"unchainedRows\":0,\"firstBreak\":null," +
"\"preCutoffRows\":3441,\"chainFromUtc\":\"2026-11-13T09:30:00.1234567Z\"}";
using var output = new StringWriter();
var exit = AuditVerifyChainHelpers.Render(body, "table", output);
Assert.Equal(0, exit);
Assert.Contains("Chain from : 2026-11-13T09:30:00.1234567Z", output.ToString());
Assert.Contains("Pre-cutoff rows : 3441", output.ToString());
}
/// <summary>A legacy (whole-table) chain renders without a cutoff.</summary>
[Fact]
public void Render_Table_NoCutoff_SaysWholeTable()
{
const string body = "{\"partition\":\"2026-05\",\"status\":\"Intact\",\"rowsVerified\":1,\"unchainedRows\":0}";
using var output = new StringWriter();
Assert.Equal(0, AuditVerifyChainHelpers.Render(body, "table", output));
Assert.Contains("Chain from : - (whole table in scope)", output.ToString());
Assert.Contains("Pre-cutoff rows : 0", output.ToString());
}
/// <summary>Verifies that month validation accepts only values in <c>yyyy-MM</c> format.</summary>
/// <param name="month">The month string to validate.</param>
/// <param name="expected">Whether the month is expected to be valid.</param>
@@ -283,7 +283,7 @@ public sealed class AuditChainHeadStoreTests : IClassFixture<MsSqlMigrationFixtu
var chain = new AuditLogHashChain(ctx);
for (var i = 0; i < 1000; i++)
{
var r = await chain.ChainPendingAsync(500);
var r = await chain.ChainPendingAsync(500, chainBacklog: true);
if (r.RowsChained == 0 && !r.LockBusy)
{
return;
@@ -214,10 +214,136 @@ public class AuditChainVerifierTests
return verifier.Complete(unchained);
}
// ── chain start / cutoff (ScadaBridge#139) ───────────────────────────────
private static readonly DateTime CutoffAt = Month.AddHours(12); // rows are dated Month + i days
/// <summary>A cutoff-started month links its first row to the anchor and verifies intact.</summary>
[Fact]
public void CutoffStartedChain_Verifies_FromTheAnchor()
{
var anchor = AuditChainHasher.ComputeCutoffAnchorHash(CutoffAt);
var (rows, head) = BuildChain(4, anchor);
var result = VerifyCutoff(rows, head, anchor);
Assert.Equal(AuditChainStatus.Intact, result.Status);
Assert.Equal(4, result.RowsVerified);
Assert.Equal(0, result.TombstonesApplied); // the anchor is not a purge tombstone
Assert.Equal(CutoffAt, result.ChainFromUtc);
Assert.Equal(7, result.PreCutoffRows);
}
/// <summary>Without its anchor, a cutoff-started month fails at its first link.</summary>
[Fact]
public void CutoffStartedChain_WithoutItsAnchor_FailsAtTheFirstLink()
{
var anchor = AuditChainHasher.ComputeCutoffAnchorHash(CutoffAt);
var (rows, head) = BuildChain(4, anchor);
var result = Verify(rows, head);
Assert.Equal(AuditChainBreakKind.HashMismatch, result.FirstBreak!.Kind);
Assert.Equal(1, result.FirstBreak.ChainSeq);
}
/// <summary>A different cutoff is a different anchor: the anchor hash binds the instant.</summary>
[Fact]
public void CutoffAnchorHash_BindsTheInstant()
{
Assert.False(AuditChainHasher.ComputeCutoffAnchorHash(CutoffAt).AsSpan()
.SequenceEqual(AuditChainHasher.ComputeCutoffAnchorHash(CutoffAt.AddTicks(1))));
Assert.Equal(AuditChainHasher.HashLength, AuditChainHasher.ComputeCutoffAnchorHash(CutoffAt).Length);
}
/// <summary>
/// The eligibility attack: a chained row re-dated before the cutoff is a
/// ChainedBeforeCutoff break even when its hash were recomputed to match.
/// </summary>
[Fact]
public void ChainedRowDatedBeforeTheCutoff_IsChainedBeforeCutoff()
{
var anchor = AuditChainHasher.ComputeCutoffAnchorHash(CutoffAt);
var (rows, _) = BuildChain(4, anchor);
// Re-date row 3 before the cutoff and re-hash the chain from it (an attacker who
// rewrites the links): only the cutoff rule can still see it.
rows[2] = rows[2] with { OccurredAtUtc = CutoffAt.AddMinutes(-1) };
var previous = rows[1].RowHash!;
for (var i = 2; i < rows.Count; i++)
{
var hash = AuditChainHasher.ComputeRowHash(previous, Month, i + 1, rows[i] with { ChainSeq = null, RowHash = null });
rows[i] = rows[i] with { RowHash = hash };
previous = hash;
}
var result = VerifyCutoff(rows, new AuditChainHeadState(4, previous, null, null), anchor);
Assert.Equal(AuditChainStatus.Broken, result.Status);
Assert.Equal(AuditChainBreakKind.ChainedBeforeCutoff, result.FirstBreak!.Kind);
Assert.Equal(3, result.FirstBreak.ChainSeq);
Assert.Equal(rows[2].EventId, result.FirstBreak.EventId);
}
/// <summary>An anchor the caller found invalid fails the month even when every link verifies.</summary>
[Fact]
public void InvalidAnchor_IsCutoffAnchorInvalid()
{
var anchor = AuditChainHasher.ComputeCutoffAnchorHash(CutoffAt);
var (rows, head) = BuildChain(4, anchor);
var tombstones = new Dictionary<long, byte[]> { [AuditChainHasher.CutoffAnchorSeq] = anchor };
var verifier = new AuditChainVerifier(Month, head, tombstones, CutoffAt, "anchor altered");
foreach (var row in rows)
{
verifier.Accept(row);
}
var result = verifier.Complete(0);
Assert.Equal(AuditChainStatus.Broken, result.Status);
Assert.Equal(AuditChainBreakKind.CutoffAnchorInvalid, result.FirstBreak!.Kind);
Assert.Equal(AuditChainHasher.CutoffAnchorSeq, result.FirstBreak.ChainSeq);
}
/// <summary>
/// After a partition purge the checkpoint anchor, not the cutoff anchor, is where
/// verification resumes; the cutoff anchor in the same tombstone set is ignored.
/// </summary>
[Fact]
public void PurgeCheckpoint_TakesPrecedenceOverTheCutoffAnchor()
{
var anchor = AuditChainHasher.ComputeCutoffAnchorHash(CutoffAt);
var (rows, _) = BuildChain(5, anchor);
var head = new AuditChainHeadState(5, rows[4].RowHash!, 3, rows[2].RowHash!);
var result = VerifyCutoff(rows.Skip(3), head, anchor);
Assert.Equal(AuditChainStatus.Intact, result.Status);
Assert.Equal(2, result.RowsVerified);
}
private static AuditChainVerificationResult VerifyCutoff(IEnumerable<AuditChainRow> rows, AuditChainHeadState head, byte[] anchor)
{
var tombstones = new Dictionary<long, byte[]> { [AuditChainHasher.CutoffAnchorSeq] = anchor };
var verifier = new AuditChainVerifier(Month, head, tombstones, CutoffAt);
foreach (var row in rows)
{
if (!verifier.Accept(row))
{
break;
}
}
return verifier.Complete(0, preCutoffRows: 7);
}
private static (List<AuditChainRow> Rows, AuditChainHeadState Head) BuildChain(int count)
=> BuildChain(count, AuditChainHasher.GenesisHash.ToArray());
private static (List<AuditChainRow> Rows, AuditChainHeadState Head) BuildChain(int count, byte[] genesis)
{
var rows = new List<AuditChainRow>();
var previous = AuditChainHasher.GenesisHash.ToArray();
var previous = genesis;
for (var seq = 1; seq <= count; seq++)
{
var row = NewRow(seq);
@@ -329,7 +329,7 @@ public sealed class AuditLogArchiveCleanupTests : IClassFixture<MsSqlMigrationFi
var chain = new AuditLogHashChain(ctx);
for (var i = 0; i < 1000; i++)
{
var r = await chain.ChainPendingAsync(500);
var r = await chain.ChainPendingAsync(500, chainBacklog: true);
if (r.RowsChained == 0 && !r.LockBusy)
{
return;
@@ -0,0 +1,561 @@
using System.Data;
using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Maintenance;
/// <summary>
/// SQL Server integration tests for the hash-chain cutoff (ScadaBridge#139): enabling the
/// chain on a database with history chains only rows from the enablement instant on (owner
/// decision on #107, 2026-11-13), verification treats the backlog as out of scope, the
/// tamper cases are still caught, moving a row (or the cutoff) across the cutoff is caught,
/// both purge paths keep the chain verifiable across the cutoff, and both central nodes
/// settle on one cutoff.
/// </summary>
/// <remarks>
/// The cutoff is one row per database, so every test starts from an empty AuditLog and empty
/// chain tables (<see cref="ResetAsync"/>). xunit runs a class's tests sequentially.
/// </remarks>
public sealed class AuditLogHashChainCutoffTests : IClassFixture<MsSqlMigrationFixture>, IDisposable
{
private static readonly DateTime Backlog = new(2026, 2, 1, 0, 0, 0, DateTimeKind.Utc);
private static readonly DateTime CutoffMonth = new(2026, 3, 1, 0, 0, 0, DateTimeKind.Utc);
private static readonly DateTime After = new(2026, 4, 1, 0, 0, 0, DateTimeKind.Utc);
private static readonly DateTime Cutoff = new(2026, 3, 15, 12, 0, 0, DateTimeKind.Utc);
private readonly MsSqlMigrationFixture _fixture;
private readonly DirectoryInfo _archiveRoot = Directory.CreateTempSubdirectory("auditlog-cutoff-archive-");
/// <summary>Creates the test class with its own migrated SQL Server database.</summary>
/// <param name="fixture">The per-class migration fixture.</param>
public AuditLogHashChainCutoffTests(MsSqlMigrationFixture fixture) => _fixture = fixture;
/// <inheritdoc />
public void Dispose() => _archiveRoot.Delete(recursive: true);
/// <summary>
/// The owner's decision end to end: a backlog exists, the chain is enabled, only rows dated
/// from the cutoff on are chained, the backlog is untouched, and every month verifies —
/// the backlog as pre-cutoff rows, never as unchained rows or a break.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task FirstEnablement_ChainsOnlyFromTheCutoff_BacklogUntouched_VerifiesIntact()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
await ResetAsync();
var backlog = Rows(Backlog.AddDays(3), 4).Concat(Rows(CutoffMonth.AddDays(2), 3)).ToList();
await InsertAsync(backlog);
await DrainAsync(); // first enablement: records the anchor at Cutoff, chains nothing
Assert.Equal(Cutoff, await AnchorCutoffAsync());
Assert.Equal(0L, await ScalarAsync<long>("SELECT COUNT_BIG(*) FROM dbo.AuditLog WHERE ChainSeq IS NOT NULL;"));
Assert.Equal(0L, await ScalarAsync<long>("SELECT COUNT_BIG(*) FROM dbo.AuditLogChainHead;"));
var post = Rows(Cutoff.AddMinutes(1), 5).Concat(Rows(After.AddDays(1), 2)).ToList();
await InsertAsync(post);
await DrainAsync();
// Every backlog row stays exactly as it was: unchained.
foreach (var e in backlog)
{
Assert.Equal(DBNull.Value, await ScalarAsync<object>($"SELECT ChainSeq FROM dbo.AuditLog WHERE EventId = '{e.EventId}';"));
}
var march = await VerifyAsync(CutoffMonth);
Assert.Equal(AuditChainStatus.Intact, march.Status);
Assert.Equal(5, march.RowsVerified);
Assert.Equal(0, march.UnchainedRows);
Assert.Equal(3, march.PreCutoffRows);
Assert.Equal(Cutoff, march.ChainFromUtc);
var february = await VerifyAsync(Backlog);
Assert.Equal(AuditChainStatus.NotChained, february.Status);
Assert.Equal(0, february.UnchainedRows);
Assert.Equal(4, february.PreCutoffRows);
var april = await VerifyAsync(After);
Assert.Equal(AuditChainStatus.Intact, april.Status);
Assert.Equal(2, april.RowsVerified);
Assert.Equal(0, april.PreCutoffRows);
// A late site row dated before the cutoff that reaches central after enablement is
// keyed on OccurredAtUtc: it stays out of scope (documented limit), not pending.
await InsertAsync(Rows(Cutoff.AddHours(-2), 1));
await DrainAsync();
march = await VerifyAsync(CutoffMonth);
Assert.Equal(AuditChainStatus.Intact, march.Status);
Assert.Equal(4, march.PreCutoffRows);
Assert.Equal(0, march.UnchainedRows);
}
/// <summary>
/// The cutoff month's first row links to the anchor, not to 32 zero bytes, and the
/// ordinary tamper cases are still caught after a cutoff start: edit, delete, tail delete.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task TamperCases_AreStillDetected_AfterACutoffStart()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var events = await SeedCutoffChainAsync(postRows: 6);
Assert.Equal(AuditChainStatus.Intact, (await VerifyAsync(CutoffMonth)).Status);
// Edit the FIRST link (the one hashed against the anchor).
await ExecAsync($"UPDATE dbo.AuditLog SET Target = N'forged' WHERE EventId = '{events[0]}';");
var edited = await VerifyAsync(CutoffMonth);
Assert.Equal(AuditChainStatus.Broken, edited.Status);
Assert.Equal(AuditChainBreakKind.HashMismatch, edited.FirstBreak!.Kind);
Assert.Equal(1, edited.FirstBreak.ChainSeq);
Assert.Equal(events[0], edited.FirstBreak.EventId);
await ExecAsync($"UPDATE dbo.AuditLog SET Target = N'erp' WHERE EventId = '{events[0]}';");
Assert.Equal(AuditChainStatus.Intact, (await VerifyAsync(CutoffMonth)).Status);
var middleSeq = await SeqOfAsync(events[2]);
await ExecAsync($"DELETE FROM dbo.AuditLog WHERE EventId = '{events[2]}';");
var deleted = await VerifyAsync(CutoffMonth);
Assert.Equal(AuditChainStatus.Broken, deleted.Status);
Assert.Equal(AuditChainBreakKind.MissingRow, deleted.FirstBreak!.Kind);
Assert.Equal(middleSeq, deleted.FirstBreak.ChainSeq);
await ResetAsync();
events = await SeedCutoffChainAsync(postRows: 4);
await ExecAsync($"DELETE FROM dbo.AuditLog WHERE EventId = '{events[^1]}';");
var tail = await VerifyAsync(CutoffMonth);
Assert.Equal(AuditChainStatus.Broken, tail.Status);
Assert.Equal(AuditChainBreakKind.HeadMismatch, tail.FirstBreak!.Kind);
}
/// <summary>
/// The cutoff-eligibility attack: move a chained row's timestamp back before the cutoff to
/// hide it among the unprotected backlog rows. Caught with its ChainSeq left in place
/// (<see cref="AuditChainBreakKind.ChainedBeforeCutoff"/>), and with its link cleared too
/// (its sequence number goes missing). Moving it into an earlier month is caught in both months.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task ChainedRowMovedBeforeTheCutoff_IsDetected()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var events = await SeedCutoffChainAsync(postRows: 5);
var victim = events[2];
var victimSeq = await SeqOfAsync(victim);
// 1. Timestamp moved back across the cutoff, link left in place.
await ExecAsync($"UPDATE dbo.AuditLog SET OccurredAtUtc = '{Cutoff.AddHours(-1):yyyy-MM-dd HH:mm:ss}' WHERE EventId = '{victim}';");
var moved = await VerifyAsync(CutoffMonth);
Assert.Equal(AuditChainStatus.Broken, moved.Status);
Assert.Equal(AuditChainBreakKind.ChainedBeforeCutoff, moved.FirstBreak!.Kind);
Assert.Equal(victim, moved.FirstBreak.EventId);
Assert.Equal(victimSeq, moved.FirstBreak.ChainSeq);
// 2. ...and its link cleared, so it looks like any other pre-cutoff row.
await ExecAsync($"UPDATE dbo.AuditLog SET ChainSeq = NULL, RowHash = NULL WHERE EventId = '{victim}';");
var hidden = await VerifyAsync(CutoffMonth);
Assert.Equal(AuditChainStatus.Broken, hidden.Status);
Assert.Equal(AuditChainBreakKind.MissingRow, hidden.FirstBreak!.Kind);
Assert.Equal(victimSeq, hidden.FirstBreak.ChainSeq);
Assert.Equal(4, hidden.PreCutoffRows); // the 3 backlog rows + the hidden one
// 3. Moved into the previous (backlog) month with its link: caught in both months.
await ResetAsync();
events = await SeedCutoffChainAsync(postRows: 5);
victim = events[1];
await ExecAsync($"UPDATE dbo.AuditLog SET OccurredAtUtc = '{Backlog.AddDays(9):yyyy-MM-dd HH:mm:ss}' WHERE EventId = '{victim}';");
Assert.Equal(AuditChainBreakKind.MissingRow, (await VerifyAsync(CutoffMonth)).FirstBreak!.Kind);
var february = await VerifyAsync(Backlog);
Assert.Equal(AuditChainStatus.Broken, february.Status);
Assert.Equal(AuditChainBreakKind.ChainedBeforeCutoff, february.FirstBreak!.Kind);
Assert.Equal(victim, february.FirstBreak.EventId);
}
/// <summary>
/// The other half of the eligibility attack: moving the cutoff itself. Editing the anchor's
/// cutoff, re-hashing the anchor to match, deleting it, or adding a second anchor are each
/// reported, so the backlog/scope boundary cannot be shifted silently.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task CutoffAnchorTampering_IsDetected()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
await SeedCutoffChainAsync(postRows: 4);
var later = Cutoff.AddDays(5);
// Cutoff moved forward, anchor hash left alone.
await ExecAsync($"UPDATE dbo.AuditLogChainTombstone SET PurgedAtUtc = '{later:yyyy-MM-dd HH:mm:ss}' WHERE ChainSeq = 0;");
var moved = await VerifyAsync(CutoffMonth);
Assert.Equal(AuditChainStatus.Broken, moved.Status);
Assert.Equal(AuditChainBreakKind.CutoffAnchorInvalid, moved.FirstBreak!.Kind);
Assert.Equal(AuditChainStatus.Broken, (await VerifyAsync(After)).Status); // every month sees it
// Cutoff moved forward AND the anchor re-hashed to match: the chained rows now precede it.
await SetAnchorAsync(later, AuditChainHasher.ComputeCutoffAnchorHash(later));
var rehashed = await VerifyAsync(CutoffMonth);
Assert.Equal(AuditChainStatus.Broken, rehashed.Status);
Assert.Equal(AuditChainBreakKind.ChainedBeforeCutoff, rehashed.FirstBreak!.Kind);
// Anchor deleted: the cutoff month's first link no longer starts from it.
await ExecAsync("DELETE FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;");
var deleted = await VerifyAsync(CutoffMonth);
Assert.Equal(AuditChainStatus.Broken, deleted.Status);
Assert.Equal(AuditChainBreakKind.HashMismatch, deleted.FirstBreak!.Kind);
Assert.Equal(1, deleted.FirstBreak.ChainSeq);
// Restored → intact (positive control); then a second anchor is reported.
await SetAnchorAsync(Cutoff, AuditChainHasher.ComputeCutoffAnchorHash(Cutoff));
Assert.Equal(AuditChainStatus.Intact, (await VerifyAsync(CutoffMonth)).Status);
var earlier = Backlog.AddDays(1);
await ExecAsync(
"INSERT INTO dbo.AuditLogChainTombstone (ChainMonth, ChainSeq, RowHash, EventId, PurgedAtUtc) " +
$"VALUES ('{Backlog:yyyy-MM-dd}', 0, 0x{Convert.ToHexString(AuditChainHasher.ComputeCutoffAnchorHash(earlier))}, NEWID(), '{earlier:yyyy-MM-dd HH:mm:ss}');");
var twoAnchors = await VerifyAsync(CutoffMonth);
Assert.Equal(AuditChainStatus.Broken, twoAnchors.Status);
Assert.Equal(AuditChainBreakKind.CutoffAnchorInvalid, twoAnchors.FirstBreak!.Kind);
}
/// <summary>
/// Both purge paths across the cutoff: a per-channel purge that deletes backlog and chained
/// rows alike leaves an intact chain; the head store and an export of it agree (the anchor is
/// not counted as a purge tombstone); the partition switch-out of the cutoff month keeps the
/// anchor and anchors the month; the archive of that month verifies from its own files; and
/// the chainer still honours the cutoff afterwards.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task PurgeAcrossTheCutoff_KeepsEveryChainVerifiable()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
await ResetAsync();
var preDb = Rows(CutoffMonth.AddDays(1), 2, AuditChannel.DbOutbound);
var preApi = Rows(CutoffMonth.AddDays(2), 2);
await InsertAsync(preDb.Concat(preApi));
await DrainAsync();
var postDb = Rows(Cutoff.AddHours(1), 3, AuditChannel.DbOutbound);
var postApi = Rows(Cutoff.AddHours(2), 3);
await InsertAsync(postDb.Concat(postApi));
await DrainAsync();
// Per-channel purge with a threshold past the cutoff: 2 backlog + 3 chained DbOutbound rows.
await using (var ctx = CreateContext())
{
var purged = await new AuditLogRepository(ctx).PurgeChannelOlderThanAsync(
nameof(AuditChannel.DbOutbound), Cutoff.AddDays(1), batchSize: 2);
Assert.Equal(5, purged);
}
var afterChannelPurge = await VerifyAsync(CutoffMonth);
Assert.Equal(AuditChainStatus.Intact, afterChannelPurge.Status);
Assert.Equal(3, afterChannelPurge.TombstonesApplied);
Assert.Equal(3, afterChannelPurge.RowsVerified);
Assert.Equal(2, afterChannelPurge.PreCutoffRows);
// Head store + export: the anchor is not a purge tombstone; an export verifies as Matches.
await using (var ctx = CreateContext())
{
var head = Assert.Single(await new AuditLogHashChain(ctx).ReadHeadsAsync());
Assert.Equal(3, head.Tombstones);
Assert.Equal(3, head.ChainedRows);
var exported = await new AuditLogHashChain(ctx).VerifyExportedHeadsAsync([Export(head)]);
Assert.True(exported.Verified);
Assert.Equal(AuditChainHeadStatus.Matches, Assert.Single(exported.Checks).Status);
}
// Archive the cutoff month, then switch it out through the archive guard.
AuditArchiveManifest manifest;
await using (var ctx = CreateContext())
{
manifest = await Archive(ctx).ArchivePartitionAsync(CutoffMonth);
await Archive(ctx).PurgeArchivedPartitionAsync(manifest);
}
Assert.Equal(Cutoff, await AnchorCutoffAsync()); // the anchor survives the switch-out
var switched = await VerifyAsync(CutoffMonth);
Assert.Equal(AuditChainStatus.Intact, switched.Status);
Assert.Equal(6, switched.PurgedThroughSeq);
Assert.Equal(0, switched.PreCutoffRows);
await using (var ctx = CreateContext())
{
var archived = await Archive(ctx).VerifyAsync(manifest.Name);
Assert.True(archived.Verified, archived.FileProblem);
var chain = Assert.Single(archived.Chains).Chain;
Assert.Equal(AuditChainStatus.Intact, chain.Status);
Assert.Equal(3, chain.RowsVerified);
Assert.Equal(3, chain.TombstonesApplied);
Assert.Equal(2, chain.PreCutoffRows); // the archive classifies its backlog rows too
Assert.Equal(0, chain.UnchainedRows);
Assert.Equal(Cutoff, chain.ChainFromUtc);
}
// After the purge the cutoff still holds: a late backlog row stays out, a late post-cutoff row chains on.
var lateBacklog = Rows(Backlog.AddDays(4), 1);
var latePost = Rows(Cutoff.AddHours(5), 1);
await InsertAsync(lateBacklog.Concat(latePost));
await DrainAsync();
Assert.Equal(DBNull.Value, await ScalarAsync<object>($"SELECT ChainSeq FROM dbo.AuditLog WHERE EventId = '{lateBacklog[0].EventId}';"));
var late = await VerifyAsync(CutoffMonth);
Assert.Equal(AuditChainStatus.Intact, late.Status);
Assert.Equal(7, late.HeadSeq);
Assert.Equal(1, (await VerifyAsync(Backlog)).PreCutoffRows);
}
/// <summary>
/// Both central nodes settle on ONE cutoff: two chainers race the first enablement on a
/// pristine database with the real database clock, exactly one anchor is written (inside the
/// database clock window of the race), and a node configured to chain the backlog that runs
/// after the decision cannot override it.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task BothNodes_AgreeOnOneCutoff()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
await ResetAsync();
var backlog = Rows(CutoffMonth.AddDays(1), 5);
await InsertAsync(backlog);
var before = await ScalarAsync<DateTime>("SELECT SYSUTCDATETIME();");
var start = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
async Task Node()
{
await start.Task;
for (var i = 0; i < 20; i++)
{
await using var ctx = CreateContext();
await new AuditLogHashChain(ctx).ChainPendingAsync(100); // production clock
}
}
var nodes = new[] { Task.Run(Node), Task.Run(Node) };
start.SetResult();
await Task.WhenAll(nodes);
var after = await ScalarAsync<DateTime>("SELECT SYSUTCDATETIME();");
Assert.Equal(1L, await ScalarAsync<long>("SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;"));
var cutoff = await AnchorCutoffAsync();
Assert.InRange(cutoff, before, after);
// The peer node configured with ChainBacklog=true runs after the decision: still no backlog chained.
await using (var ctx = CreateContext())
{
Assert.Equal(0, (await new AuditLogHashChain(ctx).ChainPendingAsync(100, chainBacklog: true)).RowsChained);
}
Assert.Equal(0L, await ScalarAsync<long>("SELECT COUNT_BIG(*) FROM dbo.AuditLog WHERE ChainSeq IS NOT NULL;"));
Assert.Equal(1L, await ScalarAsync<long>("SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;"));
var verified = await VerifyAsync(AuditChainHasher.MonthOf(cutoff));
Assert.Equal(cutoff, verified.ChainFromUtc);
}
/// <summary>
/// The old behaviour stays reachable and unchanged: ChainBacklog=true on a pristine database
/// writes no anchor and chains everything, and a database that already holds a chain (legacy
/// enablement, no anchor) keeps chaining every row even with ChainBacklog=false.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task ChainBacklog_AndLegacyChains_KeepChainingEverything()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
await ResetAsync();
await InsertAsync(Rows(Backlog.AddDays(1), 3));
await DrainAsync(chainBacklog: true);
Assert.Equal(0L, await ScalarAsync<long>("SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;"));
Assert.Equal(3L, await ScalarAsync<long>("SELECT COUNT_BIG(*) FROM dbo.AuditLog WHERE ChainSeq IS NOT NULL;"));
// Now a node on the default (no backlog) sees an existing chain: no anchor, old rows still chain.
await InsertAsync(Rows(Backlog.AddDays(10), 2));
await DrainAsync();
Assert.Equal(0L, await ScalarAsync<long>("SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;"));
var february = await VerifyAsync(Backlog);
Assert.Equal(AuditChainStatus.Intact, february.Status);
Assert.Equal(5, february.RowsVerified);
Assert.Null(february.ChainFromUtc);
Assert.Equal(0, february.PreCutoffRows);
}
/// <summary>
/// The hardened maintenance login (<c>scadabridge_audit_chainer</c> + <c>scadabridge_audit_purger</c>,
/// ScadaBridge#107 item 3) can write the chain-start anchor: no extra grant, no migration.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task MaintenanceLogin_CanWriteTheAnchor()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
await ResetAsync();
var login = "sbc_" + Guid.NewGuid().ToString("N")[..10];
const string password = "Sb_W22_cutoff#2026";
await ExecAsync($@"
CREATE LOGIN [{login}] WITH PASSWORD = N'{password}', CHECK_POLICY = OFF;
CREATE USER [{login}] FOR LOGIN [{login}];
ALTER ROLE scadabridge_audit_chainer ADD MEMBER [{login}];
ALTER ROLE scadabridge_audit_purger ADD MEMBER [{login}];");
try
{
await InsertAsync(Rows(Backlog.AddDays(2), 2));
var maintenance = new SqlConnectionStringBuilder(_fixture.ConnectionString)
{
IntegratedSecurity = false,
UserID = login,
Password = password,
Pooling = false,
}.ConnectionString;
await using (var ctx = new ScadaBridgeDbContext(
new DbContextOptionsBuilder<ScadaBridgeDbContext>().UseSqlServer(maintenance).Options))
{
var chain = new AuditLogHashChain(ctx) { AnchorClock = () => Cutoff };
Assert.Equal(0, (await chain.ChainPendingAsync(100)).RowsChained);
}
Assert.Equal(Cutoff, await AnchorCutoffAsync());
}
finally
{
SqlConnection.ClearAllPools();
await ExecAsync($@"
DECLARE @kill nvarchar(max) = N'';
SELECT @kill = @kill + N'KILL ' + CAST(session_id AS nvarchar(10)) + N';' FROM sys.dm_exec_sessions WHERE login_name = N'{login}';
EXEC sp_executesql @kill;
DROP USER [{login}];
DROP LOGIN [{login}];");
}
}
// ── helpers ──────────────────────────────────────────────────────────────
/// <summary>
/// Backlog in February and early March, first enablement at <see cref="Cutoff"/>, then
/// <paramref name="postRows"/> rows after it, all chained. Returns the post-cutoff EventIds in chain order.
/// </summary>
private async Task<List<Guid>> SeedCutoffChainAsync(int postRows)
{
await ResetAsync();
await InsertAsync(Rows(Backlog.AddDays(3), 2).Concat(Rows(CutoffMonth.AddDays(2), 3)));
await DrainAsync();
var post = Rows(Cutoff.AddMinutes(1), postRows);
await InsertAsync(post);
await DrainAsync();
return post.Select(e => e.EventId).ToList();
}
private async Task ResetAsync()
{
if (!_fixture.Available)
{
return;
}
await ExecAsync("DELETE FROM dbo.AuditLog; DELETE FROM dbo.AuditLogChainHead; DELETE FROM dbo.AuditLogChainTombstone;");
}
private async Task DrainAsync(bool chainBacklog = false)
{
await using var ctx = CreateContext();
var chain = new AuditLogHashChain(ctx) { AnchorClock = () => Cutoff };
for (var i = 0; i < 1000; i++)
{
var r = await chain.ChainPendingAsync(500, chainBacklog);
if (r.RowsChained == 0 && !r.LockBusy)
{
return;
}
}
throw new InvalidOperationException("Chain did not drain.");
}
private async Task<AuditChainVerificationResult> VerifyAsync(DateTime month)
{
await using var ctx = CreateContext();
return await new AuditLogHashChain(ctx).VerifyMonthAsync(month);
}
private async Task<DateTime> AnchorCutoffAsync()
=> DateTime.SpecifyKind(
await ScalarAsync<DateTime>("SELECT PurgedAtUtc FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;"),
DateTimeKind.Utc);
private async Task SetAnchorAsync(DateTime chainFrom, byte[] hash)
{
await ExecAsync("DELETE FROM dbo.AuditLogChainTombstone WHERE ChainSeq = 0;");
await ExecAsync(
"INSERT INTO dbo.AuditLogChainTombstone (ChainMonth, ChainSeq, RowHash, EventId, PurgedAtUtc) " +
$"VALUES ('{AuditChainHasher.MonthOf(chainFrom):yyyy-MM-dd}', 0, 0x{Convert.ToHexString(hash)}, '{AuditChainHasher.CutoffAnchorEventId}', '{chainFrom:yyyy-MM-dd HH:mm:ss}');");
}
private async Task<long> SeqOfAsync(Guid eventId)
=> await ScalarAsync<long>($"SELECT ChainSeq FROM dbo.AuditLog WHERE EventId = '{eventId}';");
private async Task InsertAsync(IEnumerable<AuditEvent> events)
{
await using var ctx = CreateContext();
var repo = new AuditLogRepository(ctx);
foreach (var e in events)
{
await repo.InsertIfNotExistsAsync(e);
}
}
private async Task ExecAsync(string sql)
{
await using var conn = new SqlConnection(_fixture.ConnectionString);
await conn.OpenAsync();
await using var cmd = conn.CreateCommand();
cmd.CommandText = sql;
await cmd.ExecuteNonQueryAsync();
}
private async Task<T> ScalarAsync<T>(string sql)
{
await using var conn = new SqlConnection(_fixture.ConnectionString);
await conn.OpenAsync();
await using var cmd = conn.CreateCommand();
cmd.CommandText = sql;
var value = await cmd.ExecuteScalarAsync();
return value is DateTime dt && typeof(T) == typeof(DateTime)
? (T)(object)DateTime.SpecifyKind(dt, DateTimeKind.Utc)
: (T)value!;
}
private AuditLogPartitionArchive Archive(ScadaBridgeDbContext ctx) => new(ctx, new AuditLogArchiveOptions
{
Enabled = true,
Directory = _archiveRoot.FullName,
RowGroupSize = 1_000,
});
private static AuditChainHeadRecord Export(AuditChainHeadSnapshot head) => new(
AuditChainHeadRecord.CurrentVersion,
AuditChainHasher.PartitionLabel(head.Month),
head.LastSeq,
AuditChainHeadComparer.ToHex(head.LastHash),
head.PurgedThroughSeq,
head.PurgedThroughHash is null ? null : AuditChainHeadComparer.ToHex(head.PurgedThroughHash),
head.ChainedRows,
head.Tombstones,
head.UpdatedAtUtc,
DateTime.UtcNow,
"central-a");
private ScadaBridgeDbContext CreateContext()
=> new(new DbContextOptionsBuilder<ScadaBridgeDbContext>().UseSqlServer(_fixture.ConnectionString).Options);
private static List<AuditEvent> Rows(DateTime first, int count, AuditChannel channel = AuditChannel.ApiOutbound)
=> Enumerable.Range(0, count).Select(i => ScadaBridgeAuditEventFactory.Create(
channel: channel,
kind: channel == AuditChannel.DbOutbound ? AuditKind.DbWrite : AuditKind.ApiCall,
status: AuditStatus.Delivered,
occurredAtUtc: first.AddMinutes(i),
target: "erp",
sourceNode: "central-a",
sourceSiteId: "cutoff-test")).ToList();
}
@@ -47,7 +47,7 @@ public class AuditLogHashChainTests : IClassFixture<MsSqlMigrationFixture>
while (!stop.IsCancellationRequested)
{
await using var ctx = CreateContext();
var r = await new AuditLogHashChain(ctx).ChainPendingAsync(17);
var r = await new AuditLogHashChain(ctx).ChainPendingAsync(17, chainBacklog: true);
if (r.LockBusy) Interlocked.Increment(ref lockBusySeen);
if (r.RowsChained == 0) await Task.Delay(5);
}
@@ -272,7 +272,7 @@ public class AuditLogHashChainTests : IClassFixture<MsSqlMigrationFixture>
var chain = new AuditLogHashChain(ctx);
for (var i = 0; i < 1000; i++)
{
var r = await chain.ChainPendingAsync(500);
var r = await chain.ChainPendingAsync(500, chainBacklog: true);
if (r.RowsChained == 0 && !r.LockBusy)
{
return;
@@ -393,7 +393,7 @@ public sealed class AuditLogPartitionArchiveTests : IClassFixture<MsSqlMigration
var chain = new AuditLogHashChain(ctx);
for (var i = 0; i < 1000; i++)
{
var r = await chain.ChainPendingAsync(500);
var r = await chain.ChainPendingAsync(500, chainBacklog: true);
if (r.RowsChained == 0 && !r.LockBusy)
{
return;
@@ -109,7 +109,7 @@ IF SUSER_ID(N'{MaintenanceLogin}') IS NOT NULL DROP LOGIN [{MaintenanceLogin}];"
// Negative controls: the writer login can neither chain nor delete.
await using (var writerCtx = Context(writerCs))
{
await Assert.ThrowsAsync<SqlException>(() => new AuditLogHashChain(writerCtx).ChainPendingAsync(100));
await Assert.ThrowsAsync<SqlException>(() => new AuditLogHashChain(writerCtx).ChainPendingAsync(100, chainBacklog: true));
await Assert.ThrowsAsync<SqlException>(() =>
new AuditLogRepository(writerCtx).PurgeChannelOlderThanAsync("DbOutbound", month.AddDays(1), 100));
}
@@ -133,7 +133,7 @@ IF SUSER_ID(N'{MaintenanceLogin}') IS NOT NULL DROP LOGIN [{MaintenanceLogin}];"
Assert.Equal(WriterLogin, await LoginNameAsync(scope.ServiceProvider.GetRequiredService<ScadaBridgeDbContext>()));
var chain = scope.ServiceProvider.GetRequiredService<IAuditLogHashChain>();
Assert.Equal(5, (await chain.ChainPendingAsync(100)).RowsChained);
Assert.Equal(5, (await chain.ChainPendingAsync(100, chainBacklog: true)).RowsChained);
Assert.Equal(AuditChainStatus.Intact, (await chain.VerifyMonthAsync(month)).Status);
var head = Assert.Single(await scope.ServiceProvider.GetRequiredService<IAuditChainHeadStore>().ReadHeadsAsync());
Assert.Equal(5, head.LastSeq);
@@ -50,6 +50,27 @@ public class VerifyAuditChainCommandTests : TestKit
new DateTime(2026, 5, 1, 0, 0, 0, DateTimeKind.Utc), Arg.Any<CancellationToken>());
}
/// <summary>
/// ScadaBridge#139: the cutoff, the pre-cutoff row count and the new break kinds reach the
/// wire (the CLI renders them; the kinds as names like every other kind).
/// </summary>
[Fact]
public void CutoffFields_AndCutoffBreakKind_ReachTheWire()
{
var chainFrom = new DateTime(2026, 11, 13, 9, 30, 0, DateTimeKind.Utc);
var brk = new AuditChainBreak("2026-11", 4, Guid.NewGuid(), AuditChainBreakKind.ChainedBeforeCutoff, "moved back");
_chain.VerifyMonthAsync(Arg.Any<DateTime>(), Arg.Any<CancellationToken>())
.Returns(new AuditChainVerificationResult("2026-11", AuditChainStatus.Broken, 3, 0, null, 9, 0, brk, 3441, chainFrom));
CreateActor().Tell(Envelope(new VerifyAuditChainCommand("2026-11"), [], "Viewer"));
var response = ExpectMsg<ManagementSuccess>(TimeSpan.FromSeconds(5));
using var json = JsonDocument.Parse(response.JsonData);
Assert.Equal(3441, json.RootElement.GetProperty("preCutoffRows").GetInt64());
Assert.Equal(chainFrom, json.RootElement.GetProperty("chainFromUtc").GetDateTime().ToUniversalTime());
Assert.Equal("ChainedBeforeCutoff", json.RootElement.GetProperty("firstBreak").GetProperty("kind").GetString());
}
/// <summary>A Designer/Deployer without an OperationalAudit role is refused.</summary>
[Fact]
public void NonAuditRole_IsUnauthorized()