using System.Data; using System.Data.Common; using System.Text; using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance; /// /// The SQL Server application lock that serialises every writer of AuditLog hash-chain /// state (ScadaBridge#41): the chaining job and both purge paths take it EXCLUSIVE; /// verification takes it SHARED so it reads a chain no writer is half-way through. /// public static class AuditChainLock { /// The sp_getapplock resource name. public const string Resource = "ScadaBridge.AuditLogHashChain"; } /// /// SQL Server implementation of (ScadaBridge#41, /// alog.md §11.4). See the interface for the chain model. /// /// /// /// Concurrency. Two central nodes and the site-ingest path INSERT into /// dbo.AuditLog concurrently; none of them touches the chain. The chaining pass runs /// in one transaction that first takes the exclusive with a /// zero timeout (the peer node's pass simply skips its tick), then reads the head row with /// UPDLOCK, HOLDLOCK and advances it with an optimistic LastSeq check. Every /// link UPDATE is guarded by ChainSeq IS NULL and its row count is checked, so a row /// purged or chained underneath the pass rolls the whole pass back instead of leaving a gap. /// A row INSERTed while the pass runs is invisible to it (READ COMMITTED) and is chained on /// a later pass with a later sequence number — chain order is chaining order, not /// OccurredAtUtc order, which is why late-arriving site rows need no special case. /// /// /// Append-only. The only AuditLog columns this class writes are ChainSeq and /// RowHash, and only while they are NULL. In production that UPDATE runs under the /// scadabridge_audit_chainer role, which holds a COLUMN-level UPDATE grant on just /// those two columns; the ingest writer role keeps its table-wide DENY UPDATE. /// /// public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore { // 4 parameters per row; 200 rows = 800 parameters, far under SQL Server's 2,100 cap. private const int UpdateChunkRows = 200; /// The ten writable AuditLog columns, in table order — the hashed row content. internal const string RowColumns = "EventId, OccurredAtUtc, Actor, Action, Outcome, Category, Target, SourceNode, CorrelationId, DetailsJson"; private readonly ScadaBridgeDbContext _context; private readonly ILogger _logger; /// Initializes a new instance of the class. /// The database context whose connection is used. /// Optional logger. public AuditLogHashChain(ScadaBridgeDbContext context, ILogger? logger = null) { _context = context ?? throw new ArgumentNullException(nameof(context)); _logger = logger ?? NullLogger.Instance; } /// /// Command timeout for verification statements, in seconds. A month can hold millions /// of rows; the management Ask (long-running, 5 min) is the real deadline. /// public int VerifyCommandTimeoutSeconds { get; init; } = 300; /// How long verification waits for an in-flight chaining/purge pass to release the lock. public TimeSpan VerifyLockTimeout { get; init; } = TimeSpan.FromSeconds(60); /// public async Task ChainPendingAsync(int batchSize, CancellationToken ct = default) { if (batchSize <= 0) { throw new ArgumentOutOfRangeException(nameof(batchSize), batchSize, "Batch size must be > 0."); } var conn = _context.Database.GetDbConnection(); var openedHere = await OpenAsync(conn, ct).ConfigureAwait(false); try { await using var tx = await conn.BeginTransactionAsync(IsolationLevel.ReadCommitted, ct).ConfigureAwait(false); if (!await TryAcquireLockAsync(conn, tx, "Exclusive", TimeSpan.Zero, ct).ConfigureAwait(false)) { await tx.RollbackAsync(ct).ConfigureAwait(false); return new AuditChainBatchResult(0, LockBusy: true); } // Oldest month that still has unchained rows. One month per pass keeps the // candidate read partition-eliminated and index-ordered (no cross-partition sort). DateTime? oldest; await using (var cmd = Command(conn, tx, "SELECT TOP (1) OccurredAtUtc FROM dbo.AuditLog WHERE ChainSeq IS NULL ORDER BY OccurredAtUtc;")) { var scalar = await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false); oldest = scalar is DateTime dt ? DateTime.SpecifyKind(dt, DateTimeKind.Utc) : null; } if (oldest is null) { await tx.CommitAsync(ct).ConfigureAwait(false); return new AuditChainBatchResult(0, LockBusy: false); } var month = AuditChainHasher.MonthOf(oldest.Value); var candidates = new List(Math.Min(batchSize, 4096)); await using (var cmd = Command(conn, tx, $"SELECT TOP (@batch) {RowColumns} FROM dbo.AuditLog " + "WHERE ChainSeq IS NULL AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd " + "ORDER BY OccurredAtUtc, EventId;")) { AddInt(cmd, "@batch", batchSize); AddMonthRange(cmd, month); await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); while (await reader.ReadAsync(ct).ConfigureAwait(false)) { candidates.Add(ReadRow(reader, withLink: false)); } } // Current chain end for the month, locked until commit. long? headSeq = null; byte[] previous = AuditChainHasher.GenesisHash.ToArray(); await using (var cmd = Command(conn, tx, "SELECT LastSeq, LastHash FROM dbo.AuditLogChainHead WITH (UPDLOCK, HOLDLOCK) WHERE ChainMonth = @month;")) { AddDate(cmd, "@month", month); await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); if (await reader.ReadAsync(ct).ConfigureAwait(false)) { headSeq = reader.GetInt64(0); previous = (byte[])reader.GetValue(1); } } var seq = headSeq ?? 0; var links = new List<(AuditChainRow Row, long Seq, byte[] Hash)>(candidates.Count); foreach (var row in candidates) { seq++; var hash = AuditChainHasher.ComputeRowHash(previous, month, seq, row); links.Add((row, seq, hash)); previous = hash; } for (var offset = 0; offset < links.Count; offset += UpdateChunkRows) { var chunk = links.Skip(offset).Take(UpdateChunkRows).ToList(); var updated = await WriteLinksAsync(conn, tx, chunk, ct).ConfigureAwait(false); if (updated != chunk.Count) { // A candidate was purged (or chained by a pass that bypassed the lock) // between the read and the write. Linking the rest would leave a gap. _logger.LogWarning( "AuditLog hash chain: {Updated} of {Expected} links written for {Partition}; rolling the pass back to retry.", updated, chunk.Count, AuditChainHasher.PartitionLabel(month)); await tx.RollbackAsync(ct).ConfigureAwait(false); return new AuditChainBatchResult(0, LockBusy: false); } } if (links.Count > 0 && !await AdvanceHeadAsync(conn, tx, month, headSeq, seq, previous, ct).ConfigureAwait(false)) { _logger.LogWarning( "AuditLog hash chain: head for {Partition} moved underneath the pass; rolling back to retry.", AuditChainHasher.PartitionLabel(month)); await tx.RollbackAsync(ct).ConfigureAwait(false); return new AuditChainBatchResult(0, LockBusy: false); } await tx.CommitAsync(ct).ConfigureAwait(false); return new AuditChainBatchResult(links.Count, LockBusy: false); } finally { if (openedHere) { await conn.CloseAsync().ConfigureAwait(false); } } } /// public async Task VerifyMonthAsync(DateTime monthStartUtc, CancellationToken ct = default) { var month = AuditChainHasher.MonthOf(DateTime.SpecifyKind(monthStartUtc, DateTimeKind.Utc)); var conn = _context.Database.GetDbConnection(); var openedHere = await OpenAsync(conn, ct).ConfigureAwait(false); try { await using var tx = await conn.BeginTransactionAsync(IsolationLevel.ReadCommitted, ct).ConfigureAwait(false); if (!await TryAcquireLockAsync(conn, tx, "Shared", VerifyLockTimeout, ct).ConfigureAwait(false)) { await tx.RollbackAsync(ct).ConfigureAwait(false); throw new TimeoutException( $"The AuditLog hash chain was busy (chaining or purge in progress) for {VerifyLockTimeout.TotalSeconds:0}s; retry."); } AuditChainHeadState? head = null; await using (var cmd = Command(conn, tx, "SELECT LastSeq, LastHash, PurgedThroughSeq, PurgedThroughHash FROM dbo.AuditLogChainHead WHERE ChainMonth = @month;")) { AddDate(cmd, "@month", month); await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); if (await reader.ReadAsync(ct).ConfigureAwait(false)) { head = new AuditChainHeadState( reader.GetInt64(0), (byte[])reader.GetValue(1), reader.IsDBNull(2) ? null : reader.GetInt64(2), reader.IsDBNull(3) ? null : (byte[])reader.GetValue(3)); } } var tombstones = new Dictionary(); await using (var cmd = Command(conn, tx, "SELECT ChainSeq, RowHash FROM dbo.AuditLogChainTombstone WHERE ChainMonth = @month;")) { AddDate(cmd, "@month", month); await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); while (await reader.ReadAsync(ct).ConfigureAwait(false)) { tombstones[reader.GetInt64(0)] = (byte[])reader.GetValue(1); } } long unchained; await using (var cmd = Command(conn, tx, "SELECT COUNT_BIG(*) FROM dbo.AuditLog WHERE ChainSeq IS NULL AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd;")) { cmd.CommandTimeout = VerifyCommandTimeoutSeconds; AddMonthRange(cmd, month); unchained = Convert.ToInt64(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false)); } var verifier = new AuditChainVerifier(month, head, tombstones); await using (var cmd = Command(conn, tx, $"SELECT {RowColumns}, ChainSeq, RowHash FROM dbo.AuditLog " + "WHERE ChainSeq IS NOT NULL AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd " + "ORDER BY ChainSeq;")) { cmd.CommandTimeout = VerifyCommandTimeoutSeconds; AddMonthRange(cmd, month); await using var reader = await cmd.ExecuteReaderAsync(CommandBehavior.SequentialAccess, ct).ConfigureAwait(false); while (await reader.ReadAsync(ct).ConfigureAwait(false)) { if (!verifier.Accept(ReadRow(reader, withLink: true))) { break; } } } await tx.CommitAsync(ct).ConfigureAwait(false); return verifier.Complete(unchained); } finally { if (openedHere) { await conn.CloseAsync().ConfigureAwait(false); } } } /// public async Task> ReadHeadsAsync(CancellationToken ct = default) { var conn = _context.Database.GetDbConnection(); var openedHere = await OpenAsync(conn, ct).ConfigureAwait(false); try { await using var tx = await conn.BeginTransactionAsync(IsolationLevel.ReadCommitted, ct).ConfigureAwait(false); await AcquireSharedOrThrowAsync(conn, tx, ct).ConfigureAwait(false); var heads = await ReadHeadsCoreAsync(conn, tx, ct).ConfigureAwait(false); await tx.CommitAsync(ct).ConfigureAwait(false); return heads; } finally { if (openedHere) { await conn.CloseAsync().ConfigureAwait(false); } } } /// public async Task VerifyExportedHeadsAsync( IReadOnlyList records, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(records); // Collapse duplicates (both centrals export, and every unchanged re-export repeats a // record): one check per distinct (month, sequence, hash) claim. var distinct = records .GroupBy(r => (r.Month, r.LastSeq, Hash: r.LastHash?.ToLowerInvariant())) .Select(g => g.OrderByDescending(r => r.ExportedAtUtc).First()) .ToList(); var conn = _context.Database.GetDbConnection(); var openedHere = await OpenAsync(conn, ct).ConfigureAwait(false); try { await using var tx = await conn.BeginTransactionAsync(IsolationLevel.ReadCommitted, ct).ConfigureAwait(false); await AcquireSharedOrThrowAsync(conn, tx, ct).ConfigureAwait(false); var heads = (await ReadHeadsCoreAsync(conn, tx, ct).ConfigureAwait(false)) .ToDictionary(h => AuditChainHasher.PartitionLabel(h.Month), StringComparer.Ordinal); var checks = new List(distinct.Count); foreach (var record in distinct) { if (!AuditChainHeadComparer.TryParseMonth(record.Month, out var month)) { checks.Add(new AuditChainHeadCheck(record.Month ?? "", record.LastSeq, record.ExportedAtUtc, record.Node, AuditChainHeadStatus.Mismatch, null, $"Exported month '{record.Month}' is not YYYY-MM.")); continue; } heads.TryGetValue(AuditChainHasher.PartitionLabel(month), out var head); checks.Add(await AuditChainHeadComparer.CompareAsync( record, head, seq => ReadStoredLinkAsync(conn, tx, month, seq, ct)).ConfigureAwait(false)); } await tx.CommitAsync(ct).ConfigureAwait(false); var ordered = checks .OrderByDescending(c => c.IsFailure) .ThenBy(c => c.Month, StringComparer.Ordinal) .ThenBy(c => c.ExportedSeq) .ToList(); return new AuditChainHeadVerificationResult(ordered.All(c => !c.IsFailure), ordered.Count, ordered); } finally { if (openedHere) { await conn.CloseAsync().ConfigureAwait(false); } } } private async Task AcquireSharedOrThrowAsync(DbConnection conn, DbTransaction tx, CancellationToken ct) { if (!await TryAcquireLockAsync(conn, tx, "Shared", VerifyLockTimeout, ct).ConfigureAwait(false)) { await tx.RollbackAsync(ct).ConfigureAwait(false); throw new TimeoutException( $"The AuditLog hash chain was busy (chaining or purge in progress) for {VerifyLockTimeout.TotalSeconds:0}s; retry."); } } private async Task> ReadHeadsCoreAsync(DbConnection conn, DbTransaction tx, CancellationToken ct) { // Per month: the head row, the chained rows still present (the filtered ChainSeq index, // partition-eliminated by the month range) and the tombstones. The head table holds one // row per month ever chained, so this is a few dozen rows. var heads = new List(); await using var cmd = Command(conn, tx, @" SELECT h.ChainMonth, h.LastSeq, h.LastHash, h.PurgedThroughSeq, h.PurgedThroughHash, h.UpdatedAtUtc, (SELECT COUNT_BIG(*) FROM dbo.AuditLog a WHERE a.ChainSeq IS NOT NULL AND a.OccurredAtUtc >= CAST(h.ChainMonth AS datetime2(7)) AND a.OccurredAtUtc < DATEADD(MONTH, 1, CAST(h.ChainMonth AS datetime2(7)))) AS ChainedRows, (SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone t WHERE t.ChainMonth = h.ChainMonth) AS Tombstones FROM dbo.AuditLogChainHead h ORDER BY h.ChainMonth;"); cmd.CommandTimeout = VerifyCommandTimeoutSeconds; await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); while (await reader.ReadAsync(ct).ConfigureAwait(false)) { heads.Add(new AuditChainHeadSnapshot( DateTime.SpecifyKind(reader.GetDateTime(0), DateTimeKind.Utc), reader.GetInt64(1), (byte[])reader.GetValue(2), reader.IsDBNull(3) ? null : reader.GetInt64(3), reader.IsDBNull(4) ? null : (byte[])reader.GetValue(4), reader.GetInt64(6), reader.GetInt64(7), DateTime.SpecifyKind(reader.GetDateTime(5), DateTimeKind.Utc))); } return heads; } private async Task ReadStoredLinkAsync( DbConnection conn, DbTransaction tx, DateTime month, long seq, CancellationToken ct) { byte[]? rowHash; await using (var cmd = Command(conn, tx, "SELECT TOP (1) RowHash FROM dbo.AuditLog WHERE ChainSeq = @seq AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd;")) { cmd.CommandTimeout = VerifyCommandTimeoutSeconds; cmd.Parameters.Add(new SqlParameter("@seq", SqlDbType.BigInt) { Value = seq }); AddMonthRange(cmd, month); rowHash = await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false) as byte[]; } byte[]? tombstoneHash = null; if (rowHash is null) { await using var cmd = Command(conn, tx, "SELECT RowHash FROM dbo.AuditLogChainTombstone WHERE ChainMonth = @month AND ChainSeq = @seq;"); AddDate(cmd, "@month", month); cmd.Parameters.Add(new SqlParameter("@seq", SqlDbType.BigInt) { Value = seq }); tombstoneHash = await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false) as byte[]; } return new AuditChainHeadComparer.StoredLink(rowHash, tombstoneHash); } /// /// Takes the chain application lock inside (released at commit / /// rollback). Returns false when the lock could not be granted within . /// internal static async Task TryAcquireLockAsync( DbConnection conn, DbTransaction tx, string mode, TimeSpan timeout, CancellationToken ct) { await using var cmd = Command(conn, tx, "DECLARE @result int; " + "EXEC @result = sp_getapplock @Resource = @resource, @LockMode = @mode, @LockOwner = 'Transaction', @LockTimeout = @timeoutMs; " + "SELECT @result;"); cmd.Parameters.Add(new SqlParameter("@resource", SqlDbType.NVarChar, 255) { Value = AuditChainLock.Resource }); cmd.Parameters.Add(new SqlParameter("@mode", SqlDbType.VarChar, 32) { Value = mode }); AddInt(cmd, "@timeoutMs", (int)Math.Min(int.MaxValue, timeout.TotalMilliseconds)); if (timeout > TimeSpan.Zero) { cmd.CommandTimeout = (int)Math.Ceiling(timeout.TotalSeconds) + 30; } var result = Convert.ToInt32(await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false)); return result >= 0; } private static async Task WriteLinksAsync( DbConnection conn, DbTransaction tx, IReadOnlyList<(AuditChainRow Row, long Seq, byte[] Hash)> chunk, CancellationToken ct) { var sql = new StringBuilder(); // The ONE sanctioned UPDATE against AuditLog outside the maintenance backfill: it // writes only the two chain columns, only while they are still NULL. sql.Append("UPDATE dbo.AuditLog SET ChainSeq = v.ChainSeq, RowHash = v.RowHash\n"); // AUDIT-PURGE-ALLOWED: hash-chain link (ChainSeq/RowHash only, NULL→value), chaining job sql.Append("FROM dbo.AuditLog INNER JOIN (VALUES "); await using var cmd = conn.CreateCommand(); cmd.Transaction = tx; for (var i = 0; i < chunk.Count; i++) { if (i > 0) { sql.Append(", "); } sql.Append($"(@e{i}, @o{i}, @s{i}, @h{i})"); cmd.Parameters.Add(new SqlParameter($"@e{i}", SqlDbType.UniqueIdentifier) { Value = chunk[i].Row.EventId }); cmd.Parameters.Add(new SqlParameter($"@o{i}", SqlDbType.DateTime2) { Scale = 7, Value = chunk[i].Row.OccurredAtUtc }); cmd.Parameters.Add(new SqlParameter($"@s{i}", SqlDbType.BigInt) { Value = chunk[i].Seq }); cmd.Parameters.Add(new SqlParameter($"@h{i}", SqlDbType.Binary, AuditChainHasher.HashLength) { Value = chunk[i].Hash }); } sql.Append(") AS v (EventId, OccurredAtUtc, ChainSeq, RowHash)\n"); sql.Append("ON dbo.AuditLog.EventId = v.EventId AND dbo.AuditLog.OccurredAtUtc = v.OccurredAtUtc\n"); sql.Append("WHERE dbo.AuditLog.ChainSeq IS NULL;"); cmd.CommandText = sql.ToString(); return await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false); } private static async Task AdvanceHeadAsync( DbConnection conn, DbTransaction tx, DateTime month, long? oldSeq, long newSeq, byte[] newHash, CancellationToken ct) { await using var cmd = Command(conn, tx, oldSeq is null ? "INSERT INTO dbo.AuditLogChainHead (ChainMonth, LastSeq, LastHash, UpdatedAtUtc) VALUES (@month, @seq, @hash, SYSUTCDATETIME());" : "UPDATE dbo.AuditLogChainHead SET LastSeq = @seq, LastHash = @hash, UpdatedAtUtc = SYSUTCDATETIME() " + "WHERE ChainMonth = @month AND LastSeq = @oldSeq;"); AddDate(cmd, "@month", month); cmd.Parameters.Add(new SqlParameter("@seq", SqlDbType.BigInt) { Value = newSeq }); cmd.Parameters.Add(new SqlParameter("@hash", SqlDbType.Binary, AuditChainHasher.HashLength) { Value = newHash }); if (oldSeq is not null) { cmd.Parameters.Add(new SqlParameter("@oldSeq", SqlDbType.BigInt) { Value = oldSeq.Value }); } return await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false) == 1; } /// /// Reads one row selected as (plus ChainSeq, RowHash when /// ), strictly in ordinal order so it is safe under SequentialAccess. /// internal static AuditChainRow ReadRow(DbDataReader reader, bool withLink) { // Ordinals follow RowColumns (+ ChainSeq, RowHash). Read strictly in order — the // verification reader is opened with SequentialAccess. var eventId = reader.GetGuid(0); var occurred = DateTime.SpecifyKind(reader.GetDateTime(1), DateTimeKind.Utc); var actor = reader.IsDBNull(2) ? null : reader.GetString(2); var action = reader.GetString(3); var outcome = reader.GetString(4); var category = reader.GetString(5); var target = reader.IsDBNull(6) ? null : reader.GetString(6); var sourceNode = reader.IsDBNull(7) ? null : reader.GetString(7); Guid? correlationId = reader.IsDBNull(8) ? null : reader.GetGuid(8); var details = reader.IsDBNull(9) ? null : reader.GetString(9); long? chainSeq = null; byte[]? rowHash = null; if (withLink) { chainSeq = reader.IsDBNull(10) ? null : reader.GetInt64(10); rowHash = reader.IsDBNull(11) ? null : (byte[])reader.GetValue(11); } return new AuditChainRow(eventId, occurred, actor, action, outcome, category, target, sourceNode, correlationId, details, chainSeq, rowHash); } private static DbCommand Command(DbConnection conn, DbTransaction tx, string sql) { var cmd = conn.CreateCommand(); cmd.Transaction = tx; cmd.CommandText = sql; return cmd; } private static void AddInt(DbCommand cmd, string name, int value) => cmd.Parameters.Add(new SqlParameter(name, SqlDbType.Int) { Value = value }); private static void AddDate(DbCommand cmd, string name, DateTime month) => cmd.Parameters.Add(new SqlParameter(name, SqlDbType.Date) { Value = month.Date }); private static void AddMonthRange(DbCommand cmd, DateTime month) { cmd.Parameters.Add(new SqlParameter("@monthStart", SqlDbType.DateTime2) { Scale = 7, Value = month }); cmd.Parameters.Add(new SqlParameter("@monthEnd", SqlDbType.DateTime2) { Scale = 7, Value = month.AddMonths(1) }); } internal static async Task OpenAsync(DbConnection conn, CancellationToken ct) { if (conn.State == ConnectionState.Open) { return false; } await conn.OpenAsync(ct).ConfigureAwait(false); return true; } }