dohertj2/ScadaBridge
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(audit): M5.6 SourceNode sentinel backfill (purge-role) + CLI + runbook note (T5)

Browse Source
This commit is contained in:
Joseph Doherty
2026-06-16 22:02:21 -04:00
parent de2968b03d
commit 55630b48b6
12 changed files with 1399 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/AuditLogAppendOnlyGuardTests.cs
+154 -4
View File
@@ -31,9 +31,40 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests;
/// targeting the AuditLog entity are NOT covered and must never be introduced.
/// Additionally, the scan is line-oriented: DML where the keyword and table name appear
/// on separate lines is an accepted, undetected edge case.
///
/// <b>Allow-list.</b> Two narrow maintenance-path exemptions carry the exact
/// <see cref="AuditPurgeAllowedMarker"/> trailing comment:
/// <list type="bullet">
/// <item><description>
/// M5.5 (T3) — <c>AuditLogRepository.PurgeChannelOlderThanAsync</c>: the
/// one sanctioned batched <c>DELETE TOP (@batch) FROM dbo.AuditLog</c>,
/// running on the purge/maintenance connection.
/// </description></item>
/// <item><description>
/// M5.6 (T5) — <c>AuditLogRepository.BackfillSourceNodeAsync</c>: the
/// one sanctioned batched <c>UPDATE TOP (@batch) dbo.AuditLog SET SourceNode</c>,
/// running on the maintenance connection. The sentinel backfill is a
/// one-time ops procedure; the append-only invariant still applies to all
/// other columns and all other UPDATE forms.
/// </description></item>
/// </list>
/// The allow-list is applied in the file-scan test only
/// (<see cref="ConfigurationDatabase_ShouldNotContainAuditLogMutations"/>) — the
/// raw mutation matcher (<see cref="ContainsAuditLogMutation"/>) is marker-blind,
/// so the matcher's self-tests remain honest and any OTHER UPDATE/DELETE against
/// AuditLog (or any DML lacking the marker) still fails the build.
/// </summary>
public class AuditLogAppendOnlyGuardTests
{
/// <summary>
/// The exact trailing-comment marker that exempts a single sanctioned
/// maintenance-path DML line from the append-only guard. Carried at the END of
/// the SQL constant string in both <c>AuditLogRepository.PurgeChannelOlderThanAsync</c>
/// (M5.5 T3 batched DELETE) and <c>AuditLogRepository.BackfillSourceNodeAsync</c>
/// (M5.6 T5 batched UPDATE). Kept deliberately specific so it cannot be pasted
/// onto an unrelated mutation without a reviewer noticing.
/// </summary>
internal const string AuditPurgeAllowedMarker = "AUDIT-PURGE-ALLOWED";
// ---------------------------------------------------------------------------
// Source root location — same walk-up pattern used by ArchitecturalConstraintTests
// in the Commons.Tests project.
@@ -133,11 +164,38 @@ public class AuditLogAppendOnlyGuardTests
return AuditLogMutationPattern.IsMatch(text);
}
// The DELETE branch tolerates an optional TOP (...) batch-size clause between
// DELETE and the (optional) FROM — e.g. "DELETE TOP (@batch) FROM dbo.AuditLog"
// (the M5.5 T3 batched purge shape). Without this the guard would silently miss a
// batched row DELETE against AuditLog, which is exactly the kind of mutation it
// must catch. The TOP sub-pattern is (?:TOP\s*\(.*?\)\s+)? — optional, lazy inside
// the parens so it never swallows past the matching ')'.
//
// The UPDATE branch similarly tolerates an optional TOP (...) clause between
// UPDATE and (optional schema.) AuditLog — e.g.
// "UPDATE TOP (@batch) dbo.AuditLog SET SourceNode = @sentinel …"
// (the M5.6 T5 batched backfill shape).
private static readonly Regex AuditLogMutationPattern = new(
@"\bUPDATE\s+(?:\[?dbo\]?\.)?(?:\[?AuditLog\]?)\b" +
@"|\bDELETE\s+(?:FROM\s+)?(?:\[?dbo\]?\.)?(?:\[?AuditLog\]?)\b",
@"\bUPDATE\s+(?:TOP\s*\(.*?\)\s+)?(?:\[?dbo\]?\.)?(?:\[?AuditLog\]?)\b" +
@"|\bDELETE\s+(?:TOP\s*\(.*?\)\s+)?(?:FROM\s+)?(?:\[?dbo\]?\.)?(?:\[?AuditLog\]?)\b",
RegexOptions.IgnoreCase | RegexOptions.Compiled);
/// <summary>
/// Returns <see langword="true"/> when <paramref name="line"/> carries the narrow
/// <see cref="AuditPurgeAllowedMarker"/> exemption. Sanctioned uses are:
/// <list type="bullet">
/// <item><description>M5.5 T3 — the per-channel maintenance-path batched DELETE.</description></item>
/// <item><description>M5.6 T5 — the SourceNode sentinel batched UPDATE.</description></item>
/// </list>
/// A flagged line that lacks the marker is NOT allow-listed. The mutation matcher
/// itself stays marker-blind; the allow-list is applied only by the file-scan test,
/// so the matcher's self-tests still observe the raw mutation.
/// </summary>
/// <param name="line">A single source line already known to contain a mutation.</param>
/// <returns><see langword="true"/> if the line is a sanctioned maintenance-path exemption.</returns>
internal static bool IsAllowListed(string line) =>
line.Contains(AuditPurgeAllowedMarker, StringComparison.Ordinal);
// ---------------------------------------------------------------------------
// Guard test: scan every *.cs file in ConfigurationDatabase (excluding
// Designer/Snapshot EF artefacts and the obj/ directory).
@@ -168,7 +226,7 @@ public class AuditLogAppendOnlyGuardTests
var lines = content.Split('\n');
for (var i = 0; i < lines.Length; i++)
{
if (ContainsAuditLogMutation(lines[i]))
if (ContainsAuditLogMutation(lines[i]) && !IsAllowListed(lines[i]))
{
var relativePath = Path.GetRelativePath(sourceDir, file);
violations.Add($"{relativePath}:{i + 1}: {lines[i].Trim()}");
@@ -179,7 +237,7 @@ public class AuditLogAppendOnlyGuardTests
Assert.True(violations.Count == 0,
"AuditLog append-only guard: found UPDATE/DELETE targeting dbo.AuditLog " +
"in ConfigurationDatabase source. AuditLog is APPEND-ONLY (retention uses " +
"partition-switch DDL, not row DELETE). Violation(s):\n" +
"partition-switch DDL, not row DELETE/UPDATE). Violation(s):\n" +
string.Join("\n", violations));
}
@@ -285,6 +343,27 @@ public class AuditLogAppendOnlyGuardTests
// DELETE FROM [AuditLog] — bracketed table, no schema prefix.
Assert.True(ContainsAuditLogMutation(
"DELETE FROM [AuditLog] WHERE OccurredAtUtc < @threshold;"));
// ---- Batched DELETE TOP (...) forms (M5.5 T3 purge shape) ----
// The matcher must catch a batched DELETE against AuditLog regardless of the
// marker — the allow-list (IsAllowListed) is what forgives the ONE sanctioned
// line, not the matcher.
Assert.True(ContainsAuditLogMutation(
"DELETE TOP (@batch) FROM dbo.AuditLog WHERE Category = @channel AND OccurredAtUtc < @threshold;"));
Assert.True(ContainsAuditLogMutation(
"DELETE TOP (5000) FROM dbo.AuditLog WHERE OccurredAtUtc < @threshold;"));
Assert.True(ContainsAuditLogMutation(
"DELETE TOP(100) FROM [dbo].[AuditLog] WHERE Status = 'Parked';"));
// ---- Batched UPDATE TOP (...) forms (M5.6 T5 backfill shape) ----
// The matcher must also catch a batched UPDATE against AuditLog, regardless of
// the marker — the allow-list is what forgives the ONE sanctioned backfill line.
Assert.True(ContainsAuditLogMutation(
"UPDATE TOP (@batch) dbo.AuditLog SET SourceNode = @sentinel WHERE SourceNode IS NULL AND OccurredAtUtc < @before;"));
Assert.True(ContainsAuditLogMutation(
"UPDATE TOP (500) dbo.AuditLog SET SourceNode = 'unknown' WHERE SourceNode IS NULL;"));
Assert.True(ContainsAuditLogMutation(
"UPDATE TOP(100) [dbo].[AuditLog] SET SourceNode = @s WHERE SourceNode IS NULL;"));
}
[Fact]
@@ -315,4 +394,75 @@ public class AuditLogAppendOnlyGuardTests
Assert.False(ContainsAuditLogMutation(
"DELETE FROM dbo.SiteCalls WHERE TerminalAtUtc < @cutoff;"));
}
// ---------------------------------------------------------------------------
// Allow-list self-tests (M5.5 T3 / M5.6 T5) — prove the narrow exemption only
// forgives the marked maintenance-path DML and still blocks everything else.
// ---------------------------------------------------------------------------
[Fact]
public void AllowList_ForgivesMarkedPurgeDelete_ButMatcherStillTrips()
{
// The sanctioned per-channel purge DELETE — verbatim shape from
// AuditLogRepository.PurgeChannelOlderThanAsync, carrying the trailing marker.
const string sanctioned =
"\"DELETE TOP (@batch) FROM dbo.AuditLog WHERE Category = @channel AND OccurredAtUtc < @threshold;\"; " +
"// AUDIT-PURGE-ALLOWED: per-channel retention override (M5.5 T3), maintenance path";
// The raw matcher STILL sees the mutation (the matcher is marker-blind) ...
Assert.True(ContainsAuditLogMutation(sanctioned));
// ... but the allow-list forgives it because of the trailing marker.
Assert.True(IsAllowListed(sanctioned));
}
[Fact]
public void AllowList_ForgivesMarkedBackfillUpdate_ButMatcherStillTrips()
{
// The sanctioned SourceNode sentinel backfill UPDATE — verbatim shape from
// AuditLogRepository.BackfillSourceNodeAsync, carrying the trailing marker.
const string sanctioned =
"\"UPDATE TOP (@batch) dbo.AuditLog SET SourceNode = @sentinel WHERE SourceNode IS NULL AND OccurredAtUtc < @before;\"; " +
"// AUDIT-PURGE-ALLOWED: SourceNode sentinel backfill (M5.6 T5), maintenance path";
// The raw matcher STILL sees the mutation (the matcher is marker-blind) ...
Assert.True(ContainsAuditLogMutation(sanctioned));
// ... but the allow-list forgives it because of the trailing marker.
Assert.True(IsAllowListed(sanctioned));
}
[Fact]
public void AllowList_DoesNotForgive_UnmarkedStrayDelete()
{
// A stray DELETE against AuditLog WITHOUT the marker — exactly the kind of
// regression the guard exists to catch. It must be flagged (matcher) AND not
// forgiven (allow-list), so the file-scan test would record it as a violation.
const string stray = "DELETE FROM dbo.AuditLog WHERE Status = 'Parked';";
Assert.True(ContainsAuditLogMutation(stray));
Assert.False(IsAllowListed(stray),
"A DELETE against AuditLog without the AUDIT-PURGE-ALLOWED marker must NOT be allow-listed.");
}
[Fact]
public void AllowList_DoesNotForgive_UnmarkedStrayUpdate()
{
// A stray UPDATE against AuditLog WITHOUT the marker — must still trip the guard.
const string stray = "UPDATE dbo.AuditLog SET Status = 'Corrected' WHERE EventId = @id;";
Assert.True(ContainsAuditLogMutation(stray));
Assert.False(IsAllowListed(stray),
"An UPDATE against AuditLog without the AUDIT-PURGE-ALLOWED marker must NOT be allow-listed.");
}
[Fact]
public void AllowList_DoesNotForgive_BatchedUpdateWithoutMarker()
{
// A batched UPDATE TOP ... AuditLog without the marker — the TOP clause variant
// must also be caught and not forgiven without the explicit marker.
const string stray = "UPDATE TOP (500) dbo.AuditLog SET SourceNode = 'unknown' WHERE SourceNode IS NULL;";
Assert.True(ContainsAuditLogMutation(stray));
Assert.False(IsAllowListed(stray),
"A batched UPDATE against AuditLog without the AUDIT-PURGE-ALLOWED marker must NOT be allow-listed.");
}
}
tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/BackfillSourceNodeTests.cs
+237
View File
@@ -0,0 +1,237 @@
using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Maintenance;
/// <summary>
/// Integration tests for <see cref="AuditLogRepository.BackfillSourceNodeAsync"/>
/// (M5.6 T5 — SourceNode sentinel backfill).
///
/// <para>
/// These tests exercise the real <see cref="AuditLogRepository"/> against a
/// per-class <see cref="MsSqlMigrationFixture"/> database, mirroring the
/// style of <c>PartitionPurgeTests</c>. All tests are guarded with
/// <c>[SkippableFact]</c> and skipped when the MSSQL container is absent.
/// </para>
/// </summary>
public class BackfillSourceNodeTests : IClassFixture<MsSqlMigrationFixture>
{
private readonly MsSqlMigrationFixture _fixture;
public BackfillSourceNodeTests(MsSqlMigrationFixture fixture)
{
_fixture = fixture;
}
private ScadaBridgeDbContext CreateContext() =>
new(new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer(_fixture.ConnectionString).Options);
private AuditLogRepository CreateRepo(ScadaBridgeDbContext ctx) => new(ctx);
// ------------------------------------------------------------------
// Seed helper: direct INSERT bypassing the writer role, same pattern
// as PartitionPurgeTests.DirectInsertAsync.
// ------------------------------------------------------------------
private async Task SeedRowAsync(
SqlConnection conn,
Guid eventId,
DateTime occurredAtUtc,
string? sourceNode)
{
await using var cmd = conn.CreateCommand();
// Supply SourceNode explicitly (NULL or a value) so the test controls
// which rows are eligible for backfill.
cmd.CommandText = @"
INSERT INTO dbo.AuditLog
(EventId, OccurredAtUtc, Actor, Action, Outcome, Category, Target, SourceNode, CorrelationId, DetailsJson)
VALUES
(@EventId, @OccurredAtUtc, NULL, 'ApiOutbound.ApiCall', 'Success', 'ApiOutbound', NULL, @SourceNode, NULL,
@DetailsJson);";
cmd.Parameters.Add("@EventId", System.Data.SqlDbType.UniqueIdentifier).Value = eventId;
var occurredParam = cmd.Parameters.Add("@OccurredAtUtc", System.Data.SqlDbType.DateTime2);
occurredParam.Scale = 7;
occurredParam.Value = occurredAtUtc;
var sourceNodeParam = cmd.Parameters.Add("@SourceNode", System.Data.SqlDbType.VarChar, 64);
sourceNodeParam.Value = (object?)sourceNode ?? DBNull.Value;
var detailsJson =
"{\"channel\":\"ApiOutbound\",\"kind\":\"ApiCall\",\"status\":\"Delivered\"," +
"\"payloadTruncated\":false}";
cmd.Parameters.Add("@DetailsJson", System.Data.SqlDbType.NVarChar, -1).Value = detailsJson;
await cmd.ExecuteNonQueryAsync();
}
private async Task<string?> ReadSourceNodeAsync(SqlConnection conn, Guid eventId)
{
await using var cmd = conn.CreateCommand();
cmd.CommandText = "SELECT SourceNode FROM dbo.AuditLog WHERE EventId = @EventId;";
cmd.Parameters.Add("@EventId", System.Data.SqlDbType.UniqueIdentifier).Value = eventId;
var raw = await cmd.ExecuteScalarAsync();
return raw == DBNull.Value ? null : (string?)raw;
}
// ------------------------------------------------------------------
// 1. SetsNullRowsBeforeThreshold
// ------------------------------------------------------------------
[SkippableFact]
public async Task BackfillSourceNode_SetsNullRowsBeforeThreshold()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var before = new DateTime(2026, 3, 1, 0, 0, 0, DateTimeKind.Utc);
var eligibleId = Guid.NewGuid(); // NULL, occurred before threshold
var tooNewId = Guid.NewGuid(); // NULL, occurred after threshold
await using var seedConn = _fixture.OpenConnection();
await SeedRowAsync(seedConn, eligibleId,
new DateTime(2026, 1, 15, 0, 0, 0, DateTimeKind.Utc), sourceNode: null);
await SeedRowAsync(seedConn, tooNewId,
new DateTime(2026, 4, 1, 0, 0, 0, DateTimeKind.Utc), sourceNode: null);
await using var ctx = CreateContext();
var repo = CreateRepo(ctx);
var rows = await repo.BackfillSourceNodeAsync("unknown", before, batchSize: 1000);
Assert.True(rows >= 1, $"Expected at least 1 row updated; got {rows}.");
// eligible row: must now have the sentinel
var eligibleNode = await ReadSourceNodeAsync(seedConn, eligibleId);
Assert.Equal("unknown", eligibleNode);
// too-new row: must still be NULL
var tooNewNode = await ReadSourceNodeAsync(seedConn, tooNewId);
Assert.Null(tooNewNode);
}
// ------------------------------------------------------------------
// 2. LeavesNonNullRowsUntouched
// ------------------------------------------------------------------
[SkippableFact]
public async Task BackfillSourceNode_LeavesNonNullRowsUntouched()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var before = new DateTime(2026, 3, 1, 0, 0, 0, DateTimeKind.Utc);
var alreadySetId = Guid.NewGuid(); // already has a SourceNode value
await using var seedConn = _fixture.OpenConnection();
await SeedRowAsync(seedConn, alreadySetId,
new DateTime(2026, 1, 10, 0, 0, 0, DateTimeKind.Utc), sourceNode: "node-a");
await using var ctx = CreateContext();
var repo = CreateRepo(ctx);
await repo.BackfillSourceNodeAsync("unknown", before, batchSize: 1000);
// "node-a" must still be "node-a", not overwritten
var node = await ReadSourceNodeAsync(seedConn, alreadySetId);
Assert.Equal("node-a", node);
}
// ------------------------------------------------------------------
// 3. Idempotent_SecondRunUpdatesZeroRows
// ------------------------------------------------------------------
[SkippableFact]
public async Task BackfillSourceNode_Idempotent_SecondRunUpdatesZeroRows()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var before = new DateTime(2026, 3, 1, 0, 0, 0, DateTimeKind.Utc);
var idempotentId = Guid.NewGuid();
await using var seedConn = _fixture.OpenConnection();
await SeedRowAsync(seedConn, idempotentId,
new DateTime(2026, 1, 20, 0, 0, 0, DateTimeKind.Utc), sourceNode: null);
await using var ctx1 = CreateContext();
var repo1 = CreateRepo(ctx1);
var firstRun = await repo1.BackfillSourceNodeAsync("unknown", before, batchSize: 1000);
Assert.True(firstRun >= 1, "First run should update at least 1 row.");
// Second run: no NULL rows remain for this threshold — must update 0.
await using var ctx2 = CreateContext();
var repo2 = CreateRepo(ctx2);
var secondRun = await repo2.BackfillSourceNodeAsync("unknown", before, batchSize: 1000);
// The second run must not update the already-sentinel row again.
// We cannot assert exactly 0 because other tests share the same fixture DB
// and may have left unrelated NULL rows; but the idempotentId row must not
// have been touched (it already has "unknown", so the WHERE SourceNode IS NULL
// filter excludes it).
var node = await ReadSourceNodeAsync(seedConn, idempotentId);
Assert.Equal("unknown", node);
// The second run returning 0 would be true if no other NULL rows exist —
// we assert the contract from the repo's perspective by checking the row.
_ = secondRun; // acknowledged: value consumed
}
// ------------------------------------------------------------------
// 4. CustomSentinelIsWritten
// ------------------------------------------------------------------
[SkippableFact]
public async Task BackfillSourceNode_CustomSentinel_IsWritten()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var before = new DateTime(2026, 6, 1, 0, 0, 0, DateTimeKind.Utc);
var customId = Guid.NewGuid();
await using var seedConn = _fixture.OpenConnection();
await SeedRowAsync(seedConn, customId,
new DateTime(2026, 2, 5, 0, 0, 0, DateTimeKind.Utc), sourceNode: null);
await using var ctx = CreateContext();
var repo = CreateRepo(ctx);
await repo.BackfillSourceNodeAsync("pre-feature", before, batchSize: 1000);
var node = await ReadSourceNodeAsync(seedConn, customId);
Assert.Equal("pre-feature", node);
}
// ------------------------------------------------------------------
// 5. ArgumentValidation
// ------------------------------------------------------------------
[Fact]
public async Task BackfillSourceNode_EmptySentinel_Throws()
{
// Guard fires even without a DB connection — no Skip needed.
// Use a null/empty context via a degenerate connection string; the
// argument check fires before any SQL runs.
await using var ctx = new ScadaBridgeDbContext(
new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer("Server=.;Database=dummy;Connect Timeout=0;")
.Options);
var repo = new AuditLogRepository(ctx);
await Assert.ThrowsAsync<ArgumentException>(
() => repo.BackfillSourceNodeAsync("", DateTime.UtcNow, 1000));
}
[Fact]
public async Task BackfillSourceNode_ZeroBatchSize_Throws()
{
await using var ctx = new ScadaBridgeDbContext(
new DbContextOptionsBuilder<ScadaBridgeDbContext>()
.UseSqlServer("Server=.;Database=dummy;Connect Timeout=0;")
.Options);
var repo = new AuditLogRepository(ctx);
await Assert.ThrowsAsync<ArgumentOutOfRangeException>(
() => repo.BackfillSourceNodeAsync("unknown", DateTime.UtcNow, 0));
}
}