feat(audit): SHA-256 hash chain over the central AuditLog (ScadaBridge#41)
Audit roadmap T1 (alog.md §11.4): one chain per UTC calendar month of OccurredAtUtc. Each chained row carries ChainSeq (gap-free within its month) and RowHash = SHA-256(prev.RowHash || canonical(row)); the v1 canonical form (AuditChainHasher) is pinned by a known-answer vector computed independently. - Migration AddAuditLogHashChain: nullable ChainSeq/RowHash on dbo.AuditLog (metadata-only add), AuditLogChainHead (per-month chain end + partition-purge checkpoint anchor), AuditLogChainTombstone, two partition-aligned filtered indexes, scadabridge_audit_chainer role with a column-level UPDATE grant. Production script docs/plans/sql/AddAuditLogHashChain.sql. - AuditLogHashChain chains unchained rows oldest month first under an exclusive sp_getapplock (zero timeout, so a dual-active central pair still yields one chain), with an optimistic head check and a ChainSeq IS NULL row-count check that roll a pass back instead of leaving a gap. The ingest path is untouched. - Purges keep the chain verifiable: the partition switch records the month's checkpoint anchor and drops its tombstones; the per-channel DELETE writes a tombstone per chained row in the same transaction (OUTPUT INTO). Both, and the SourceNode backfill, hold the chain lock; the backfill skips chained rows. The switch staging table gains the two new columns. - AuditChainVerifier reports the first break (partition, ChainSeq, EventId, kind) for edited, deleted, reordered rows and tail truncation. - AuditLogHashChainService (central, default OFF via AuditLog:HashChain). Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,125 @@
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Central;
|
||||
|
||||
/// <summary>
|
||||
/// Policy tests for <see cref="AuditLogHashChainService"/> (ScadaBridge#41) over a scripted
|
||||
/// <see cref="IAuditLogHashChain"/>: default-off, draining a backlog within the per-tick
|
||||
/// cap, yielding to the peer node's lock, and containing failures. The SQL chaining itself
|
||||
/// is covered by <c>AuditLogHashChainTests</c> in ConfigurationDatabase.Tests.
|
||||
/// </summary>
|
||||
public class AuditLogHashChainServiceTests
|
||||
{
|
||||
private sealed class ScriptedChain(params AuditChainBatchResult[] script) : IAuditLogHashChain
|
||||
{
|
||||
private int _next;
|
||||
public int Calls => _next;
|
||||
public Exception? Throw { get; init; }
|
||||
|
||||
public Task<AuditChainBatchResult> ChainPendingAsync(int batchSize, CancellationToken ct = default)
|
||||
{
|
||||
if (Throw is not null) throw Throw;
|
||||
var i = _next++;
|
||||
return Task.FromResult(i < script.Length ? script[i] : new AuditChainBatchResult(0, false));
|
||||
}
|
||||
|
||||
public Task<AuditChainVerificationResult> VerifyMonthAsync(DateTime monthStartUtc, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
}
|
||||
|
||||
private static AuditLogHashChainService Create(IAuditLogHashChain chain, AuditLogHashChainOptions options)
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddScoped(_ => chain);
|
||||
return new AuditLogHashChainService(
|
||||
services.BuildServiceProvider().GetRequiredService<IServiceScopeFactory>(),
|
||||
Options.Create(options),
|
||||
NullLogger<AuditLogHashChainService>.Instance);
|
||||
}
|
||||
|
||||
/// <summary>The job is off by default and starting it then does nothing.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Disabled_ByDefault_StartsNoLoop()
|
||||
{
|
||||
var chain = new ScriptedChain(new AuditChainBatchResult(5, false));
|
||||
Assert.False(new AuditLogHashChainOptions().Enabled);
|
||||
using var service = Create(chain, new AuditLogHashChainOptions());
|
||||
|
||||
await service.StartAsync(CancellationToken.None);
|
||||
await Task.Delay(50);
|
||||
await service.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, chain.Calls);
|
||||
}
|
||||
|
||||
/// <summary>A tick drains batches until one comes back empty.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Tick_DrainsUntilEmpty()
|
||||
{
|
||||
var chain = new ScriptedChain(
|
||||
new AuditChainBatchResult(10, false), new AuditChainBatchResult(10, false), new AuditChainBatchResult(3, false));
|
||||
using var service = Create(chain, new AuditLogHashChainOptions { Enabled = true, BatchSize = 10 });
|
||||
|
||||
Assert.Equal(23, await service.TickAsync(CancellationToken.None));
|
||||
Assert.Equal(4, chain.Calls); // the 4th returns 0 and ends the tick
|
||||
}
|
||||
|
||||
/// <summary>The peer node holding the lock ends the tick at once.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Tick_StopsWhenPeerHoldsTheLock()
|
||||
{
|
||||
var chain = new ScriptedChain(new AuditChainBatchResult(0, LockBusy: true), new AuditChainBatchResult(10, false));
|
||||
using var service = Create(chain, new AuditLogHashChainOptions { Enabled = true });
|
||||
|
||||
Assert.Equal(0, await service.TickAsync(CancellationToken.None));
|
||||
Assert.Equal(1, chain.Calls);
|
||||
}
|
||||
|
||||
/// <summary>A backlog yields after MaxBatchesPerTick batches.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Tick_RespectsMaxBatchesPerTick()
|
||||
{
|
||||
var chain = new ScriptedChain(Enumerable.Repeat(new AuditChainBatchResult(10, false), 50).ToArray());
|
||||
using var service = Create(chain, new AuditLogHashChainOptions { Enabled = true, MaxBatchesPerTick = 3 });
|
||||
|
||||
Assert.Equal(30, await service.TickAsync(CancellationToken.None));
|
||||
Assert.Equal(3, chain.Calls);
|
||||
}
|
||||
|
||||
/// <summary>A failing tick is contained (logged), never thrown out of the loop.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Tick_ContainsFailures()
|
||||
{
|
||||
var chain = new ScriptedChain { Throw = new InvalidOperationException("sql down") };
|
||||
using var service = Create(chain, new AuditLogHashChainOptions { Enabled = true });
|
||||
|
||||
Assert.Equal(0, await service.TickAsync(CancellationToken.None));
|
||||
}
|
||||
|
||||
/// <summary>Invalid tuning fails startup validation.</summary>
|
||||
/// <param name="interval">IntervalSeconds.</param>
|
||||
/// <param name="batch">BatchSize.</param>
|
||||
/// <param name="maxBatches">MaxBatchesPerTick.</param>
|
||||
[Theory]
|
||||
[InlineData(0, 1000, 100)]
|
||||
[InlineData(60, 0, 100)]
|
||||
[InlineData(60, 10_001, 100)]
|
||||
[InlineData(60, 1000, 0)]
|
||||
public void Validator_RejectsInvalidTuning(int interval, int batch, int maxBatches)
|
||||
{
|
||||
var result = new AuditLogHashChainOptionsValidator().Validate(null,
|
||||
new AuditLogHashChainOptions { IntervalSeconds = interval, BatchSize = batch, MaxBatchesPerTick = maxBatches });
|
||||
Assert.False(result.Succeeded);
|
||||
Assert.True(new AuditLogHashChainOptionsValidator().Validate(null, new AuditLogHashChainOptions()).Succeeded);
|
||||
}
|
||||
}
|
||||
+6
-1
@@ -32,7 +32,7 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests;
|
||||
/// Additionally, the scan is line-oriented: DML where the keyword and table name appear
|
||||
/// on separate lines is an accepted, undetected edge case.
|
||||
///
|
||||
/// <b>Allow-list.</b> Two narrow maintenance-path exemptions carry the exact
|
||||
/// <b>Allow-list.</b> Three narrow exemptions carry the exact
|
||||
/// <see cref="AuditPurgeAllowedMarker"/> trailing comment:
|
||||
/// <list type="bullet">
|
||||
/// <item><description>
|
||||
@@ -47,6 +47,11 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests;
|
||||
/// one-time ops procedure; the append-only invariant still applies to all
|
||||
/// other columns and all other UPDATE forms.
|
||||
/// </description></item>
|
||||
/// <item><description>
|
||||
/// <c>AuditLogHashChain.WriteLinksAsync</c> (ScadaBridge#41): the chaining job's
|
||||
/// <c>UPDATE dbo.AuditLog SET ChainSeq, RowHash</c>, guarded by
|
||||
/// <c>ChainSeq IS NULL</c> — it only ever fills the two hash-chain link columns once.
|
||||
/// </description></item>
|
||||
/// </list>
|
||||
/// The allow-list is applied in the file-scan test only
|
||||
/// (<see cref="ConfigurationDatabase_ShouldNotContainAuditLogMutations"/>) — the
|
||||
|
||||
+5
-2
@@ -97,8 +97,9 @@ public class AuditLogEntityTypeConfigurationTests : IDisposable
|
||||
// C5: 10 canonical columns (EventId, OccurredAtUtc, Actor, Action, Outcome,
|
||||
// Channel[=Category], Target, SourceNode, CorrelationId, DetailsJson) + 6
|
||||
// persisted computed columns (Kind, Status, SourceSiteId, ExecutionId,
|
||||
// ParentExecutionId, IngestedAtUtc) = 16.
|
||||
Assert.Equal(16, properties.Count);
|
||||
// ParentExecutionId, IngestedAtUtc) + 2 hash-chain link columns (ChainSeq,
|
||||
// RowHash — ScadaBridge#41) = 18.
|
||||
Assert.Equal(18, properties.Count);
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the channel property maps to the canonical category column.</summary>
|
||||
@@ -177,6 +178,7 @@ public class AuditLogEntityTypeConfigurationTests : IDisposable
|
||||
var expected = new[]
|
||||
{
|
||||
"IX_AuditLog_Actor_Occurred",
|
||||
"IX_AuditLog_ChainSeq", // hash chain (ScadaBridge#41)
|
||||
"IX_AuditLog_Channel_Status_Occurred",
|
||||
"IX_AuditLog_CorrelationId",
|
||||
"IX_AuditLog_Execution",
|
||||
@@ -186,6 +188,7 @@ public class AuditLogEntityTypeConfigurationTests : IDisposable
|
||||
"IX_AuditLog_ParentExecution",
|
||||
"IX_AuditLog_Site_Occurred",
|
||||
"IX_AuditLog_Target_Occurred",
|
||||
"IX_AuditLog_Unchained", // hash chain (ScadaBridge#41)
|
||||
// UX_AuditLog_EventId is intentionally absent — dropped by
|
||||
// AlignAuditLogEventIdUniqueness; the aligned clustered PK carries it.
|
||||
};
|
||||
|
||||
+244
@@ -0,0 +1,244 @@
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Maintenance;
|
||||
|
||||
/// <summary>
|
||||
/// Storage-free tests for the AuditLog hash-chain hasher + verifier (ScadaBridge#41):
|
||||
/// tamper detection for an edited, a deleted and a reordered row, tombstone and
|
||||
/// partition-anchor bridging, and a pinned known-answer vector for the v1 canonical form.
|
||||
/// </summary>
|
||||
public class AuditChainVerifierTests
|
||||
{
|
||||
private static readonly DateTime Month = new(2026, 5, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
/// <summary>An untouched chain verifies intact and covers every row.</summary>
|
||||
[Fact]
|
||||
public void IntactChain_Verifies()
|
||||
{
|
||||
var (rows, head) = BuildChain(5);
|
||||
var result = Verify(rows, head);
|
||||
|
||||
Assert.Equal(AuditChainStatus.Intact, result.Status);
|
||||
Assert.Equal(5, result.RowsVerified);
|
||||
Assert.Null(result.FirstBreak);
|
||||
}
|
||||
|
||||
/// <summary>Editing any stored column of a chained row is a HashMismatch at that row.</summary>
|
||||
[Fact]
|
||||
public void EditedRow_IsHashMismatch_AtThatRow()
|
||||
{
|
||||
var (rows, head) = BuildChain(5);
|
||||
rows[2] = rows[2] with { Target = "tampered" };
|
||||
|
||||
var result = Verify(rows, head);
|
||||
|
||||
Assert.Equal(AuditChainStatus.Broken, result.Status);
|
||||
Assert.Equal(AuditChainBreakKind.HashMismatch, result.FirstBreak!.Kind);
|
||||
Assert.Equal(3, result.FirstBreak.ChainSeq);
|
||||
Assert.Equal(rows[2].EventId, result.FirstBreak.EventId);
|
||||
Assert.Equal("2026-05", result.FirstBreak.Partition);
|
||||
Assert.Equal(2, result.RowsVerified);
|
||||
}
|
||||
|
||||
/// <summary>A row deleted from the middle (no tombstone) is a MissingRow at its sequence number.</summary>
|
||||
[Fact]
|
||||
public void DeletedMiddleRow_IsMissingRow()
|
||||
{
|
||||
var (rows, head) = BuildChain(5);
|
||||
var deleted = rows[1];
|
||||
rows.RemoveAt(1);
|
||||
|
||||
var result = Verify(rows, head);
|
||||
|
||||
Assert.Equal(AuditChainStatus.Broken, result.Status);
|
||||
Assert.Equal(AuditChainBreakKind.MissingRow, result.FirstBreak!.Kind);
|
||||
Assert.Equal(deleted.ChainSeq, result.FirstBreak.ChainSeq);
|
||||
Assert.Equal(rows[1].EventId, result.FirstBreak.EventId); // the next surviving row
|
||||
}
|
||||
|
||||
/// <summary>Deleting the newest rows leaves a self-consistent prefix; the head catches it.</summary>
|
||||
[Fact]
|
||||
public void DeletedTailRows_IsHeadMismatch()
|
||||
{
|
||||
var (rows, head) = BuildChain(5);
|
||||
rows.RemoveRange(3, 2);
|
||||
|
||||
var result = Verify(rows, head);
|
||||
|
||||
Assert.Equal(AuditChainStatus.Broken, result.Status);
|
||||
Assert.Equal(AuditChainBreakKind.HeadMismatch, result.FirstBreak!.Kind);
|
||||
Assert.Equal(4, result.FirstBreak.ChainSeq);
|
||||
Assert.Null(result.FirstBreak.EventId);
|
||||
}
|
||||
|
||||
/// <summary>Swapping two rows' sequence numbers is a HashMismatch at the first moved row.</summary>
|
||||
[Fact]
|
||||
public void ReorderedRows_IsHashMismatch()
|
||||
{
|
||||
var (rows, head) = BuildChain(5);
|
||||
var a = rows[1];
|
||||
var b = rows[3];
|
||||
rows[1] = b with { ChainSeq = a.ChainSeq };
|
||||
rows[3] = a with { ChainSeq = b.ChainSeq };
|
||||
|
||||
var result = Verify(rows, head);
|
||||
|
||||
Assert.Equal(AuditChainStatus.Broken, result.Status);
|
||||
Assert.Equal(AuditChainBreakKind.HashMismatch, result.FirstBreak!.Kind);
|
||||
Assert.Equal(2, result.FirstBreak.ChainSeq);
|
||||
Assert.Equal(b.EventId, result.FirstBreak.EventId);
|
||||
}
|
||||
|
||||
/// <summary>A duplicated sequence number is reported, not silently skipped.</summary>
|
||||
[Fact]
|
||||
public void DuplicateSequence_IsReported()
|
||||
{
|
||||
var (rows, head) = BuildChain(3);
|
||||
rows.Insert(2, rows[1] with { EventId = Guid.NewGuid() });
|
||||
|
||||
var result = Verify(rows, head);
|
||||
|
||||
Assert.Equal(AuditChainBreakKind.DuplicateSequence, result.FirstBreak!.Kind);
|
||||
}
|
||||
|
||||
/// <summary>A sanctioned per-channel purge leaves a tombstone that bridges the gap.</summary>
|
||||
[Fact]
|
||||
public void TombstonedRow_BridgesTheGap()
|
||||
{
|
||||
var (rows, head) = BuildChain(5);
|
||||
var purged = rows[2];
|
||||
rows.RemoveAt(2);
|
||||
var tombstones = new Dictionary<long, byte[]> { [purged.ChainSeq!.Value] = purged.RowHash! };
|
||||
|
||||
var result = Verify(rows, head, tombstones);
|
||||
|
||||
Assert.Equal(AuditChainStatus.Intact, result.Status);
|
||||
Assert.Equal(4, result.RowsVerified);
|
||||
Assert.Equal(1, result.TombstonesApplied);
|
||||
}
|
||||
|
||||
/// <summary>A tombstone with the wrong hash does not hide a deletion.</summary>
|
||||
[Fact]
|
||||
public void ForgedTombstone_IsDetectedAtTheNextRow()
|
||||
{
|
||||
var (rows, head) = BuildChain(5);
|
||||
var purged = rows[2];
|
||||
rows.RemoveAt(2);
|
||||
var tombstones = new Dictionary<long, byte[]> { [purged.ChainSeq!.Value] = new byte[32] };
|
||||
|
||||
var result = Verify(rows, head, tombstones);
|
||||
|
||||
Assert.Equal(AuditChainBreakKind.HashMismatch, result.FirstBreak!.Kind);
|
||||
Assert.Equal(4, result.FirstBreak.ChainSeq);
|
||||
}
|
||||
|
||||
/// <summary>After a partition purge, verification resumes at the checkpoint anchor.</summary>
|
||||
[Fact]
|
||||
public void PartitionPurgeAnchor_VerifiesTheRest()
|
||||
{
|
||||
var (rows, head) = BuildChain(6);
|
||||
var anchor = rows[3];
|
||||
var survivors = rows.Skip(4).ToList();
|
||||
var anchored = head with { PurgedThroughSeq = anchor.ChainSeq, PurgedThroughHash = anchor.RowHash };
|
||||
|
||||
var result = Verify(survivors, anchored);
|
||||
|
||||
Assert.Equal(AuditChainStatus.Intact, result.Status);
|
||||
Assert.Equal(2, result.RowsVerified);
|
||||
Assert.Equal(4, result.PurgedThroughSeq);
|
||||
}
|
||||
|
||||
/// <summary>A month with neither rows nor head reports NotChained.</summary>
|
||||
[Fact]
|
||||
public void EmptyMonth_IsNotChained()
|
||||
{
|
||||
var result = Verify([], null, unchained: 7);
|
||||
|
||||
Assert.Equal(AuditChainStatus.NotChained, result.Status);
|
||||
Assert.Equal(7, result.UnchainedRows);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Pins the v1 canonical form: a change to the serialisation silently invalidates every
|
||||
/// chain already in a database, so it must fail a test first. The expected value was
|
||||
/// computed by an independent implementation of the documented v1 encoding (Python
|
||||
/// hashlib/struct), not by this code.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void KnownAnswerVector_V1()
|
||||
{
|
||||
var row = new AuditChainRow(
|
||||
Guid.Parse("0f8fad5b-d9cb-469f-a165-70867728950e"),
|
||||
new DateTime(2026, 5, 20, 10, 30, 0, DateTimeKind.Utc).AddTicks(1234567),
|
||||
"alice", "ApiOutbound.ApiCall", "Success", "ApiOutbound", "Erp", "central-a",
|
||||
Guid.Parse("7c9e6679-7425-40de-944b-e07fc1f90ae7"), "{\"kind\":\"ApiCall\",\"status\":\"Delivered\"}");
|
||||
|
||||
var hash = AuditChainHasher.ComputeRowHash(AuditChainHasher.GenesisHash, Month, 1, row);
|
||||
|
||||
Assert.Equal("a3137c9961f35f931c7d84de6bced66ce1fe8bbb1dc500f9740b97efd9983ad4", Convert.ToHexStringLower(hash));
|
||||
}
|
||||
|
||||
/// <summary>NULL and the empty string hash differently (presence byte), as do adjacent-field shifts.</summary>
|
||||
[Fact]
|
||||
public void Canonical_DistinguishesNullEmptyAndFieldBoundaries()
|
||||
{
|
||||
var baseRow = NewRow(1);
|
||||
var nullTarget = baseRow with { Target = null };
|
||||
var emptyTarget = baseRow with { Target = "" };
|
||||
var shifted = baseRow with { Actor = "ab", Target = "c" };
|
||||
var shifted2 = baseRow with { Actor = "a", Target = "bc" };
|
||||
|
||||
Assert.NotEqual(Hash(nullTarget), Hash(emptyTarget));
|
||||
Assert.NotEqual(Hash(shifted), Hash(shifted2));
|
||||
}
|
||||
|
||||
private static string Hash(AuditChainRow row)
|
||||
=> Convert.ToHexString(AuditChainHasher.ComputeRowHash(AuditChainHasher.GenesisHash, Month, 1, row));
|
||||
|
||||
private static AuditChainVerificationResult Verify(
|
||||
IEnumerable<AuditChainRow> rows,
|
||||
AuditChainHeadState? head,
|
||||
IReadOnlyDictionary<long, byte[]>? tombstones = null,
|
||||
long unchained = 0)
|
||||
{
|
||||
var verifier = new AuditChainVerifier(Month, head, tombstones ?? new Dictionary<long, byte[]>());
|
||||
foreach (var row in rows)
|
||||
{
|
||||
if (!verifier.Accept(row))
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return verifier.Complete(unchained);
|
||||
}
|
||||
|
||||
private static (List<AuditChainRow> Rows, AuditChainHeadState Head) BuildChain(int count)
|
||||
{
|
||||
var rows = new List<AuditChainRow>();
|
||||
var previous = AuditChainHasher.GenesisHash.ToArray();
|
||||
for (var seq = 1; seq <= count; seq++)
|
||||
{
|
||||
var row = NewRow(seq);
|
||||
var hash = AuditChainHasher.ComputeRowHash(previous, Month, seq, row);
|
||||
rows.Add(row with { ChainSeq = seq, RowHash = hash });
|
||||
previous = hash;
|
||||
}
|
||||
|
||||
return (rows, new AuditChainHeadState(count, previous, null, null));
|
||||
}
|
||||
|
||||
private static AuditChainRow NewRow(int i)
|
||||
=> new(
|
||||
Guid.NewGuid(),
|
||||
Month.AddDays(i).AddTicks(i),
|
||||
$"actor-{i}",
|
||||
"ApiOutbound.ApiCall",
|
||||
"Success",
|
||||
"ApiOutbound",
|
||||
$"target-{i}",
|
||||
"central-a",
|
||||
null,
|
||||
$"{{\"kind\":\"ApiCall\",\"n\":{i}}}");
|
||||
}
|
||||
+346
@@ -0,0 +1,346 @@
|
||||
using Microsoft.Data.SqlClient;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using ZB.MOM.WW.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Maintenance;
|
||||
|
||||
/// <summary>
|
||||
/// SQL Server integration tests for the AuditLog hash chain (ScadaBridge#41) against the
|
||||
/// real migrated, partitioned schema: continuity under concurrent writers and chainers,
|
||||
/// detection of an edited / deleted / reordered row, and survival of both purge paths.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The fixture database is shared by the class and the chain is per month across the whole
|
||||
/// table, so every test owns a distinct month (xunit runs one class's tests sequentially).
|
||||
/// Each test drains the chain before tampering so no other month is left unchained.
|
||||
/// </remarks>
|
||||
public class AuditLogHashChainTests : IClassFixture<MsSqlMigrationFixture>
|
||||
{
|
||||
private readonly MsSqlMigrationFixture _fixture;
|
||||
|
||||
/// <summary>Creates the test class with the shared migrated SQL Server fixture.</summary>
|
||||
/// <param name="fixture">The shared migration fixture.</param>
|
||||
public AuditLogHashChainTests(MsSqlMigrationFixture fixture) => _fixture = fixture;
|
||||
|
||||
/// <summary>
|
||||
/// Many concurrent INSERTs (the two central nodes + site ingest) racing two concurrent
|
||||
/// chainers (a dual-active central pair) still produce one gap-free, intact chain.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[SkippableFact]
|
||||
public async Task ConcurrentWritersAndChainers_ProduceOneIntactGapFreeChain()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
var month = Month(2026, 2);
|
||||
const int writers = 8;
|
||||
const int rowsPerWriter = 40;
|
||||
|
||||
using var stop = new CancellationTokenSource();
|
||||
var lockBusySeen = 0;
|
||||
async Task ChainLoop()
|
||||
{
|
||||
while (!stop.IsCancellationRequested)
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
var r = await new AuditLogHashChain(ctx).ChainPendingAsync(17);
|
||||
if (r.LockBusy) Interlocked.Increment(ref lockBusySeen);
|
||||
if (r.RowsChained == 0) await Task.Delay(5);
|
||||
}
|
||||
}
|
||||
|
||||
var chainers = new[] { Task.Run(ChainLoop), Task.Run(ChainLoop) };
|
||||
var writerTasks = Enumerable.Range(0, writers).Select(w => Task.Run(async () =>
|
||||
{
|
||||
for (var i = 0; i < rowsPerWriter; i++)
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
// Out-of-order OccurredAtUtc on purpose: late-arriving site rows.
|
||||
var at = month.AddDays((w * 7 + i * 13) % 27).AddSeconds(i);
|
||||
await new AuditLogRepository(ctx).InsertIfNotExistsAsync(NewEvent(at, sourceNode: $"node-{w}"));
|
||||
}
|
||||
})).ToArray();
|
||||
|
||||
await Task.WhenAll(writerTasks);
|
||||
await Task.Delay(200);
|
||||
await stop.CancelAsync();
|
||||
await Task.WhenAll(chainers);
|
||||
await DrainAsync();
|
||||
|
||||
var result = await VerifyAsync(month);
|
||||
Assert.Equal(AuditChainStatus.Intact, result.Status);
|
||||
Assert.Equal(writers * rowsPerWriter, result.RowsVerified);
|
||||
Assert.Equal(writers * rowsPerWriter, result.HeadSeq);
|
||||
Assert.Equal(0, result.UnchainedRows);
|
||||
|
||||
var seqs = await ChainSeqsAsync(month);
|
||||
Assert.Equal(Enumerable.Range(1, writers * rowsPerWriter).Select(i => (long)i), seqs);
|
||||
}
|
||||
|
||||
/// <summary>Editing a chained row's content is reported at that row.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[SkippableFact]
|
||||
public async Task EditedRow_IsDetected_WithRowIdAndPartition()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
var month = Month(2026, 3);
|
||||
var events = await SeedAndChainAsync(month, 6);
|
||||
Assert.Equal(AuditChainStatus.Intact, (await VerifyAsync(month)).Status);
|
||||
|
||||
var victim = events[3];
|
||||
await ExecAsync($"UPDATE dbo.AuditLog SET Target = N'forged' WHERE EventId = '{victim}';");
|
||||
|
||||
var result = await VerifyAsync(month);
|
||||
Assert.Equal(AuditChainStatus.Broken, result.Status);
|
||||
Assert.Equal(AuditChainBreakKind.HashMismatch, result.FirstBreak!.Kind);
|
||||
Assert.Equal("2026-03", result.FirstBreak.Partition);
|
||||
Assert.Equal(victim, result.FirstBreak.EventId);
|
||||
Assert.Equal(await SeqOfAsync(victim), result.FirstBreak.ChainSeq);
|
||||
}
|
||||
|
||||
/// <summary>Deleting a chained row outside the purge path is reported as a missing sequence number.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[SkippableFact]
|
||||
public async Task DeletedRow_IsDetected()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
var month = Month(2026, 4);
|
||||
var events = await SeedAndChainAsync(month, 6);
|
||||
var victim = events[2];
|
||||
var victimSeq = await SeqOfAsync(victim);
|
||||
|
||||
await ExecAsync($"DELETE FROM dbo.AuditLog WHERE EventId = '{victim}';");
|
||||
|
||||
var result = await VerifyAsync(month);
|
||||
Assert.Equal(AuditChainStatus.Broken, result.Status);
|
||||
Assert.Equal(AuditChainBreakKind.MissingRow, result.FirstBreak!.Kind);
|
||||
Assert.Equal(victimSeq, result.FirstBreak.ChainSeq);
|
||||
Assert.Equal("2026-04", result.FirstBreak.Partition);
|
||||
}
|
||||
|
||||
/// <summary>Swapping two rows' chain positions is reported.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[SkippableFact]
|
||||
public async Task ReorderedRows_AreDetected()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
var month = Month(2026, 5);
|
||||
var events = await SeedAndChainAsync(month, 6);
|
||||
var a = events[1];
|
||||
var b = events[4];
|
||||
var seqA = await SeqOfAsync(a);
|
||||
var seqB = await SeqOfAsync(b);
|
||||
|
||||
await ExecAsync(
|
||||
$"UPDATE dbo.AuditLog SET ChainSeq = CASE WHEN EventId = '{a}' THEN {seqB} ELSE {seqA} END " +
|
||||
$"WHERE EventId IN ('{a}', '{b}');");
|
||||
|
||||
var result = await VerifyAsync(month);
|
||||
Assert.Equal(AuditChainStatus.Broken, result.Status);
|
||||
Assert.Equal(AuditChainBreakKind.HashMismatch, result.FirstBreak!.Kind);
|
||||
Assert.Equal(Math.Min(seqA, seqB), result.FirstBreak.ChainSeq);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The per-channel retention DELETE removes chained rows from the middle of a month and
|
||||
/// the chain still verifies (tombstones); a later chaining pass continues the chain.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[SkippableFact]
|
||||
public async Task PerChannelPurge_LeavesAVerifiableChain()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
var month = Month(2026, 6);
|
||||
var rows = new List<AuditEvent>();
|
||||
for (var i = 0; i < 10; i++)
|
||||
{
|
||||
rows.Add(NewEvent(month.AddDays(i + 1),
|
||||
channel: i % 2 == 0 ? AuditChannel.DbOutbound : AuditChannel.ApiOutbound,
|
||||
kind: i % 2 == 0 ? AuditKind.DbWrite : AuditKind.ApiCall));
|
||||
}
|
||||
|
||||
await InsertAsync(rows);
|
||||
await DrainAsync();
|
||||
|
||||
await using (var ctx = CreateContext())
|
||||
{
|
||||
var purged = await new AuditLogRepository(ctx).PurgeChannelOlderThanAsync(
|
||||
nameof(AuditChannel.DbOutbound), month.AddDays(8), batchSize: 2);
|
||||
Assert.Equal(4, purged); // days 1,3,5,7
|
||||
}
|
||||
|
||||
var result = await VerifyAsync(month);
|
||||
Assert.Equal(AuditChainStatus.Intact, result.Status);
|
||||
Assert.Equal(4, result.TombstonesApplied);
|
||||
Assert.Equal(6, result.RowsVerified);
|
||||
|
||||
// A row arriving after the purge chains on from the existing head.
|
||||
await InsertAsync([NewEvent(month.AddDays(20))]);
|
||||
await DrainAsync();
|
||||
result = await VerifyAsync(month);
|
||||
Assert.Equal(AuditChainStatus.Intact, result.Status);
|
||||
Assert.Equal(11, result.HeadSeq);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The monthly partition-switch purge removes the whole month; the head's checkpoint
|
||||
/// anchor keeps the month verifiable, including for a late row chained afterwards.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[SkippableFact]
|
||||
public async Task PartitionSwitchPurge_LeavesAnAnchoredVerifiableChain()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
var month = Month(2026, 7);
|
||||
await SeedAndChainAsync(month, 5);
|
||||
|
||||
await using (var ctx = CreateContext())
|
||||
{
|
||||
await new AuditLogRepository(ctx).SwitchOutPartitionAsync(month);
|
||||
}
|
||||
|
||||
var result = await VerifyAsync(month);
|
||||
Assert.Equal(AuditChainStatus.Intact, result.Status);
|
||||
Assert.Equal(5, result.PurgedThroughSeq);
|
||||
Assert.Equal(0, result.RowsVerified);
|
||||
|
||||
await InsertAsync([NewEvent(month.AddDays(3))]);
|
||||
await DrainAsync();
|
||||
result = await VerifyAsync(month);
|
||||
Assert.Equal(AuditChainStatus.Intact, result.Status);
|
||||
Assert.Equal(1, result.RowsVerified);
|
||||
Assert.Equal(6, result.HeadSeq);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The SourceNode backfill (the other sanctioned AuditLog UPDATE) must not touch a chained
|
||||
/// row — SourceNode is inside its hash.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[SkippableFact]
|
||||
public async Task SourceNodeBackfill_SkipsChainedRows()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
var month = Month(2026, 8);
|
||||
await InsertAsync([NewEvent(month.AddDays(2), sourceNode: null)]);
|
||||
await DrainAsync();
|
||||
var unchained = NewEvent(month.AddDays(3), sourceNode: null);
|
||||
|
||||
// A row written after the drain stays unchained until the next pass.
|
||||
await InsertAsync([unchained]);
|
||||
await using (var ctx = CreateContext())
|
||||
{
|
||||
await new AuditLogRepository(ctx).BackfillSourceNodeAsync("unknown", month.AddMonths(1), batchSize: 10);
|
||||
}
|
||||
|
||||
await DrainAsync();
|
||||
var result = await VerifyAsync(month);
|
||||
Assert.Equal(AuditChainStatus.Intact, result.Status);
|
||||
Assert.Equal(2, result.RowsVerified);
|
||||
Assert.Equal("unknown", await ScalarAsync<string>($"SELECT SourceNode FROM dbo.AuditLog WHERE EventId = '{unchained.EventId}';"));
|
||||
}
|
||||
|
||||
// ── helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
private static DateTime Month(int year, int month) => new(year, month, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
private async Task<List<Guid>> SeedAndChainAsync(DateTime month, int count)
|
||||
{
|
||||
var events = Enumerable.Range(1, count).Select(i => NewEvent(month.AddDays(i).AddMinutes(i))).ToList();
|
||||
await InsertAsync(events);
|
||||
await DrainAsync();
|
||||
return events.Select(e => e.EventId).ToList();
|
||||
}
|
||||
|
||||
private async Task InsertAsync(IEnumerable<AuditEvent> events)
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
var repo = new AuditLogRepository(ctx);
|
||||
foreach (var e in events)
|
||||
{
|
||||
await repo.InsertIfNotExistsAsync(e);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DrainAsync()
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
var chain = new AuditLogHashChain(ctx);
|
||||
for (var i = 0; i < 1000; i++)
|
||||
{
|
||||
var r = await chain.ChainPendingAsync(500);
|
||||
if (r.RowsChained == 0 && !r.LockBusy)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
throw new InvalidOperationException("Chain did not drain.");
|
||||
}
|
||||
|
||||
private async Task<AuditChainVerificationResult> VerifyAsync(DateTime month)
|
||||
{
|
||||
await using var ctx = CreateContext();
|
||||
return await new AuditLogHashChain(ctx).VerifyMonthAsync(month);
|
||||
}
|
||||
|
||||
private async Task<long> SeqOfAsync(Guid eventId)
|
||||
=> await ScalarAsync<long>($"SELECT ChainSeq FROM dbo.AuditLog WHERE EventId = '{eventId}';");
|
||||
|
||||
private async Task<List<long>> ChainSeqsAsync(DateTime month)
|
||||
{
|
||||
await using var conn = new SqlConnection(_fixture.ConnectionString);
|
||||
await conn.OpenAsync();
|
||||
await using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = "SELECT ChainSeq FROM dbo.AuditLog WHERE OccurredAtUtc >= @s AND OccurredAtUtc < @e ORDER BY ChainSeq;";
|
||||
cmd.Parameters.Add(new SqlParameter("@s", System.Data.SqlDbType.DateTime2) { Value = month });
|
||||
cmd.Parameters.Add(new SqlParameter("@e", System.Data.SqlDbType.DateTime2) { Value = month.AddMonths(1) });
|
||||
var list = new List<long>();
|
||||
await using var reader = await cmd.ExecuteReaderAsync();
|
||||
while (await reader.ReadAsync())
|
||||
{
|
||||
list.Add(reader.GetInt64(0));
|
||||
}
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
private async Task ExecAsync(string sql)
|
||||
{
|
||||
await using var conn = new SqlConnection(_fixture.ConnectionString);
|
||||
await conn.OpenAsync();
|
||||
await using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = sql;
|
||||
await cmd.ExecuteNonQueryAsync();
|
||||
}
|
||||
|
||||
private async Task<T> ScalarAsync<T>(string sql)
|
||||
{
|
||||
await using var conn = new SqlConnection(_fixture.ConnectionString);
|
||||
await conn.OpenAsync();
|
||||
await using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = sql;
|
||||
return (T)(await cmd.ExecuteScalarAsync())!;
|
||||
}
|
||||
|
||||
private ScadaBridgeDbContext CreateContext()
|
||||
=> new(new DbContextOptionsBuilder<ScadaBridgeDbContext>().UseSqlServer(_fixture.ConnectionString).Options);
|
||||
|
||||
private static AuditEvent NewEvent(
|
||||
DateTime occurredAtUtc,
|
||||
AuditChannel channel = AuditChannel.ApiOutbound,
|
||||
AuditKind kind = AuditKind.ApiCall,
|
||||
string? sourceNode = "central-a") =>
|
||||
ScadaBridgeAuditEventFactory.Create(
|
||||
channel: channel,
|
||||
kind: kind,
|
||||
status: AuditStatus.Delivered,
|
||||
occurredAtUtc: occurredAtUtc,
|
||||
target: "erp",
|
||||
sourceNode: sourceNode,
|
||||
sourceSiteId: "hash-chain-test");
|
||||
}
|
||||
Reference in New Issue
Block a user