Files
ScadaBridge/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditChainHeadStoreTests.cs
T
Joseph Doherty c4d7ec9b96 feat(audit): off-box chain-head export, verify-heads, maintenance login (ScadaBridge#107)
Item 1 (owner decision: no signing). AuditLog:HashChain:HeadExport (default
off): both centrals periodically read every month's chain head (LastSeq,
LastHash, purge anchor, chained rows, tombstones; one snapshot under the
chain lock SHARED) and append the changed heads to a sink the DB login cannot
write - File (append-only JSONL per node per day, local or UNC) or Syslog
(RFC 5424 over UDP / TCP octet counting). `audit verify-heads --path` reads
those records back (file or syslog-collector lines) and the new
VerifyAuditChainHeads command checks each against the database: Matches /
Extended / Purged, or Mismatch / Regressed / MissingHead / MissingLink /
CountMismatch (exit 3). AuditChainHeadStoreTests forges a chain with the
production hasher: verify-chain says Intact, verify-heads says Mismatch.

Item 3. AuditLog:MaintenanceConnectionString (${secret:}-capable, unset =
today's single login) runs the chain job, chain verify / head export, the
partition switch-out + archive and the per-channel purge on
AuditMaintenanceContext (keyed audit-maintenance repository/archive, purge
actor prefers them). AuditMaintenanceLoginTests runs the hardened setup on
SQL Server: writer (datareader/datawriter/audit_writer) ingests and is
refused chaining and deleting; the chainer+purger login does the full cycle.
Two-login grants documented in Component-AuditLog.md.

Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
2026-11-09 23:16:05 -05:00

313 lines
14 KiB
C#

using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Maintenance;
/// <summary>
/// SQL Server tests for the chain-head export store (ScadaBridge#107 item 1): heads read with
/// their counts, and exported heads checked against the database. The key case is the forgery
/// the export exists for: rows rewritten AND every later link and the head recomputed — which
/// <c>verify-chain</c> accepts — is caught because the exported hash no longer matches.
/// </summary>
/// <remarks>Each test owns one month (partition) of the shared fixture database.</remarks>
public sealed class AuditChainHeadStoreTests : IClassFixture<MsSqlMigrationFixture>
{
private readonly MsSqlMigrationFixture _fixture;
/// <summary>Creates the test class with the shared migrated SQL Server fixture.</summary>
/// <param name="fixture">The shared migration fixture.</param>
public AuditChainHeadStoreTests(MsSqlMigrationFixture fixture) => _fixture = fixture;
/// <summary>The head snapshot carries the chain end and the live row + tombstone counts.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task ReadHeads_ReportsTheChainEnd_AndRowAndTombstoneCounts()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var month = Month(2026, 3);
await InsertAsync(Enumerable.Range(1, 5).Select(i => NewEvent(month.AddDays(i))));
await InsertAsync([NewEvent(month.AddHours(1), AuditChannel.DbOutbound)]);
await DrainAsync();
await using (var ctx = CreateContext())
{
Assert.Equal(1, await new AuditLogRepository(ctx).PurgeChannelOlderThanAsync("DbOutbound", month.AddDays(1), 100));
}
var head = await HeadAsync(month);
Assert.Equal(6, head.LastSeq);
Assert.Equal(5, head.ChainedRows);
Assert.Equal(1, head.Tombstones);
Assert.Null(head.PurgedThroughSeq);
}
/// <summary>
/// Unchanged chain: Matches. The chain grew: Extended. Both centrals' duplicate exports
/// collapse to one check.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task ExportedHeads_MatchTheUnchangedChain_AndAreExtendedByLaterLinks()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var month = Month(2026, 4);
await InsertAsync(Enumerable.Range(1, 4).Select(i => NewEvent(month.AddDays(i))));
await DrainAsync();
var exported = Export(await HeadAsync(month));
var same = await VerifyAsync(exported, exported with { Node = "central-b" });
Assert.True(same.Verified);
Assert.Equal(AuditChainHeadStatus.Matches, Assert.Single(same.Checks).Status);
await InsertAsync([NewEvent(month.AddDays(10))]);
await DrainAsync();
var grown = await VerifyAsync(exported);
Assert.True(grown.Verified);
var check = Assert.Single(grown.Checks);
Assert.Equal(AuditChainHeadStatus.Extended, check.Status);
Assert.Equal(5, check.DatabaseSeq);
}
/// <summary>
/// The forgery: a chained row's content is rewritten and every link from it plus the head
/// are recomputed with the real hasher. <c>VerifyMonthAsync</c> reports the forged chain
/// Intact (positive control: the in-database chain alone cannot see it); the exported head
/// does not match it.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task ARewrittenAndRehashedChain_PassesVerifyChain_ButFailsTheExportedHead()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var month = Month(2026, 5);
await InsertAsync(Enumerable.Range(1, 6).Select(i => NewEvent(month.AddDays(i))));
await DrainAsync();
var exported = Export(await HeadAsync(month));
await ForgeChainAsync(month, fromSeq: 3);
await using (var ctx = CreateContext())
{
var chain = await new AuditLogHashChain(ctx).VerifyMonthAsync(month);
Assert.Equal(AuditChainStatus.Intact, chain.Status);
}
var result = await VerifyAsync(exported);
Assert.False(result.Verified);
Assert.Equal(AuditChainHeadStatus.Mismatch, Assert.Single(result.Checks).Status);
}
/// <summary>Each structural tampering maps to its own verdict.</summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task TamperedHeadsAndRows_AreReported_WithTheirOwnVerdicts()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// Head rolled back below the export (tail deletion + head rewrite).
var regressed = Month(2026, 6);
await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(regressed.AddDays(i))));
await DrainAsync();
var regressedExport = Export(await HeadAsync(regressed));
await ExecAsync("UPDATE dbo.AuditLogChainHead SET LastSeq = LastSeq - 1 WHERE ChainMonth = @m;", regressed);
// Head row deleted.
var missingHead = Month(2026, 7);
await InsertAsync([NewEvent(missingHead.AddDays(1))]);
await DrainAsync();
var missingHeadExport = Export(await HeadAsync(missingHead));
await ExecAsync("DELETE FROM dbo.AuditLogChainHead WHERE ChainMonth = @m;", missingHead);
// A chained row deleted with no tombstone, head unchanged: counts no longer add up.
var counted = Month(2026, 8);
await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(counted.AddDays(i))));
await DrainAsync();
var countedExport = Export(await HeadAsync(counted));
await ExecAsync("DELETE FROM dbo.AuditLog WHERE ChainSeq = 2 AND OccurredAtUtc >= @m AND OccurredAtUtc < DATEADD(MONTH, 1, @m);", counted);
// The exported link itself deleted after the chain grew past it.
var missingLink = Month(2026, 9);
await InsertAsync(Enumerable.Range(1, 2).Select(i => NewEvent(missingLink.AddDays(i))));
await DrainAsync();
var missingLinkExport = Export(await HeadAsync(missingLink));
await InsertAsync([NewEvent(missingLink.AddDays(5))]);
await DrainAsync();
await ExecAsync("DELETE FROM dbo.AuditLog WHERE ChainSeq = 2 AND OccurredAtUtc >= @m AND OccurredAtUtc < DATEADD(MONTH, 1, @m);", missingLink);
var result = await VerifyAsync(regressedExport, missingHeadExport, countedExport, missingLinkExport);
Assert.False(result.Verified);
var byMonth = result.Checks.ToDictionary(c => c.Month, c => c.Status);
Assert.Equal(AuditChainHeadStatus.Regressed, byMonth["2026-06"]);
Assert.Equal(AuditChainHeadStatus.MissingHead, byMonth["2026-07"]);
Assert.Equal(AuditChainHeadStatus.CountMismatch, byMonth["2026-08"]);
Assert.Equal(AuditChainHeadStatus.MissingLink, byMonth["2026-09"]);
}
/// <summary>
/// Sanctioned purges are not tamper evidence: a per-channel purge (row to tombstone) keeps
/// Matches; after a partition switch the exported head is the anchor, so the check still
/// passes, and an older exported sequence the switch removed is reported Purged.
/// </summary>
/// <returns>A task that represents the asynchronous operation.</returns>
[SkippableFact]
public async Task SanctionedPurges_AreNotFailures()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var month = Month(2026, 10);
await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(month.AddDays(i))));
await DrainAsync();
var early = Export(await HeadAsync(month));
await InsertAsync([NewEvent(month.AddDays(5), AuditChannel.Notification)]);
await DrainAsync();
var late = Export(await HeadAsync(month));
await using (var ctx = CreateContext())
{
Assert.Equal(1, await new AuditLogRepository(ctx).PurgeChannelOlderThanAsync("Notification", month.AddDays(6), 100));
}
Assert.Equal(AuditChainHeadStatus.Matches, Assert.Single((await VerifyAsync(late)).Checks).Status);
await using (var ctx = CreateContext())
{
await new AuditLogRepository(ctx).SwitchOutPartitionAsync(month);
}
var afterSwitch = await VerifyAsync(late, early);
Assert.True(afterSwitch.Verified);
var statuses = afterSwitch.Checks.ToDictionary(c => c.ExportedSeq, c => c.Status);
Assert.Equal(AuditChainHeadStatus.Matches, statuses[4]);
Assert.Equal(AuditChainHeadStatus.Purged, statuses[3]);
}
private static AuditChainHeadRecord Export(AuditChainHeadSnapshot head) => new(
AuditChainHeadRecord.CurrentVersion,
AuditChainHasher.PartitionLabel(head.Month),
head.LastSeq,
AuditChainHeadComparer.ToHex(head.LastHash),
head.PurgedThroughSeq,
head.PurgedThroughHash is null ? null : AuditChainHeadComparer.ToHex(head.PurgedThroughHash),
head.ChainedRows,
head.Tombstones,
head.UpdatedAtUtc,
DateTime.UtcNow,
"central-a");
private async Task<AuditChainHeadSnapshot> HeadAsync(DateTime month)
{
await using var ctx = CreateContext();
var heads = await new AuditLogHashChain(ctx).ReadHeadsAsync();
return Assert.Single(heads, h => h.Month == month);
}
private async Task<AuditChainHeadVerificationResult> VerifyAsync(params AuditChainHeadRecord[] records)
{
await using var ctx = CreateContext();
return await new AuditLogHashChain(ctx).VerifyExportedHeadsAsync(records);
}
/// <summary>
/// Rewrites the row at <paramref name="fromSeq"/> and re-links it and every later row plus the
/// head with the production hasher — what a principal with full write access could do.
/// </summary>
private async Task ForgeChainAsync(DateTime month, long fromSeq)
{
await using var conn = _fixture.OpenConnection();
await using (var edit = new SqlCommand(
"UPDATE dbo.AuditLog SET Target = N'forged' WHERE ChainSeq = @s AND OccurredAtUtc >= @m AND OccurredAtUtc < DATEADD(MONTH, 1, @m);", conn))
{
edit.Parameters.AddWithValue("@s", fromSeq);
edit.Parameters.Add(new SqlParameter("@m", System.Data.SqlDbType.DateTime2) { Scale = 7, Value = month });
Assert.Equal(1, await edit.ExecuteNonQueryAsync());
}
var rows = new List<AuditChainRow>();
await using (var read = new SqlCommand(
$"SELECT {AuditLogHashChain.RowColumns}, ChainSeq, RowHash FROM dbo.AuditLog " +
"WHERE ChainSeq IS NOT NULL AND OccurredAtUtc >= @m AND OccurredAtUtc < DATEADD(MONTH, 1, @m) ORDER BY ChainSeq;", conn))
{
read.Parameters.Add(new SqlParameter("@m", System.Data.SqlDbType.DateTime2) { Scale = 7, Value = month });
await using var reader = await read.ExecuteReaderAsync();
while (await reader.ReadAsync())
{
rows.Add(AuditLogHashChain.ReadRow(reader, withLink: true));
}
}
var previous = rows.Single(r => r.ChainSeq == fromSeq - 1).RowHash!;
foreach (var row in rows.Where(r => r.ChainSeq >= fromSeq))
{
var hash = AuditChainHasher.ComputeRowHash(previous, month, row.ChainSeq!.Value, row);
await using var update = new SqlCommand("UPDATE dbo.AuditLog SET RowHash = @h WHERE EventId = @e AND OccurredAtUtc = @o;", conn);
update.Parameters.AddWithValue("@h", hash);
update.Parameters.AddWithValue("@e", row.EventId);
update.Parameters.Add(new SqlParameter("@o", System.Data.SqlDbType.DateTime2) { Scale = 7, Value = row.OccurredAtUtc });
Assert.Equal(1, await update.ExecuteNonQueryAsync());
previous = hash;
}
await using var head = new SqlCommand("UPDATE dbo.AuditLogChainHead SET LastHash = @h WHERE ChainMonth = @m;", conn);
head.Parameters.AddWithValue("@h", previous);
head.Parameters.Add(new SqlParameter("@m", System.Data.SqlDbType.Date) { Value = month });
Assert.Equal(1, await head.ExecuteNonQueryAsync());
}
private async Task ExecAsync(string sql, DateTime month)
{
await using var conn = _fixture.OpenConnection();
await using var cmd = new SqlCommand(sql, conn);
cmd.Parameters.Add(new SqlParameter("@m", System.Data.SqlDbType.DateTime2) { Scale = 7, Value = month });
Assert.True(await cmd.ExecuteNonQueryAsync() >= 1);
}
private async Task InsertAsync(IEnumerable<AuditEvent> events)
{
await using var ctx = CreateContext();
var repo = new AuditLogRepository(ctx);
foreach (var e in events)
{
await repo.InsertIfNotExistsAsync(e);
}
}
private async Task DrainAsync()
{
await using var ctx = CreateContext();
var chain = new AuditLogHashChain(ctx);
for (var i = 0; i < 1000; i++)
{
var r = await chain.ChainPendingAsync(500);
if (r.RowsChained == 0 && !r.LockBusy)
{
return;
}
}
throw new InvalidOperationException("Chain did not drain.");
}
private static DateTime Month(int year, int month) => new(year, month, 1, 0, 0, 0, DateTimeKind.Utc);
private ScadaBridgeDbContext CreateContext()
=> new(new DbContextOptionsBuilder<ScadaBridgeDbContext>().UseSqlServer(_fixture.ConnectionString).Options);
private static AuditEvent NewEvent(DateTime occurredAtUtc, AuditChannel channel = AuditChannel.ApiOutbound) =>
ScadaBridgeAuditEventFactory.Create(
channel: channel,
kind: channel == AuditChannel.DbOutbound ? AuditKind.DbWrite : AuditKind.ApiCall,
status: AuditStatus.Delivered,
occurredAtUtc: occurredAtUtc,
actor: "head-test",
target: "erp",
sourceNode: "central-a",
sourceSiteId: "head-test",
correlationId: Guid.NewGuid());
}