dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

perf(central): set-based ingest, aligned partition purge, KPI query shapes, EF hygiene

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-14 21:07:12 -04:00
parent ee193cd2bb
commit 5db2a810c0
29 changed files with 3790 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Repositories/AuditLogRepository.cs
+241 -28
View File
@@ -1,5 +1,8 @@
using System.Data;
using System.Text;
using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.Audit;
@@ -17,15 +20,32 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
/// </summary>
public class AuditLogRepository : IAuditLogRepository
{
// SQL Server error numbers for duplicate-key violations on
// UX_AuditLog_EventId. 2601 is a unique-index violation; 2627 is a
// primary-key/unique-constraint violation. The IF NOT EXISTS … INSERT
// pattern has a check-then-act race window — two sessions can both pass
// the EXISTS check and then both attempt the INSERT — and the loser
// surfaces as one of these errors. Idempotency demands we swallow them.
// SQL Server error numbers for duplicate-key violations on the
// partition-aligned clustered PK_AuditLog (EventId, OccurredAtUtc).
// 2601 is a unique-index violation; 2627 is a primary-key/unique-constraint
// violation. The IF NOT EXISTS … INSERT pattern has a check-then-act race
// window — two sessions can both pass the EXISTS check and then both attempt
// the INSERT — and the loser surfaces as one of these errors. Idempotency
// demands we swallow them.
private const int SqlErrorUniqueIndexViolation = 2601;
private const int SqlErrorPrimaryKeyViolation = 2627;
// Rows per set-based ingest statement. Ten bound parameters per row against
// SQL Server's 2,100-parameter ceiling leaves ample headroom at 100 rows
// (1,000 parameters) while still collapsing a typical telemetry packet into
// a single round trip. Larger chunks buy little — the win is round-trip
// elimination, not statement size — and would push plan-cache churn up
// (one cached plan per distinct row count).
private const int IngestChunkRows = 100;
// Ordinal-stable column list shared by the single-row and set-based inserts.
// The five persisted computed columns (Kind/Status/SourceSiteId/ExecutionId/
// ParentExecutionId) plus the non-persisted IngestedAtUtc are derived
// server-side from DetailsJson and must NEVER appear here — writing a
// computed column is an error.
private const string CanonicalColumnList =
"EventId, OccurredAtUtc, Actor, Action, Outcome, Category, Target, SourceNode, CorrelationId, DetailsJson";
private readonly ScadaBridgeDbContext _context;
private readonly ILogger<AuditLogRepository> _logger;
@@ -83,7 +103,7 @@ VALUES
{
// Two concurrent sessions both passed the IF NOT EXISTS check and
// both attempted the INSERT — the loser raises 2601/2627 against
// UX_AuditLog_EventId. First-write-wins idempotency is already the
// the clustered PK. First-write-wins idempotency is already the
// documented contract for this method, so the race outcome is
// semantically a no-op. Swallow at Debug; other SqlExceptions
// bubble.
@@ -95,6 +115,196 @@ VALUES
}
}
/// <inheritdoc />
public async Task<int> InsertManyIfNotExistsAsync(
IReadOnlyList<AuditEvent> events,
TimeSpan? commandTimeout = null,
CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(events);
if (events.Count == 0)
{
return 0;
}
// De-duplicate WITHIN the packet before the statement is built. The
// set-based INSERT … SELECT … WHERE NOT EXISTS only tests rows that are
// ALREADY committed, so two copies of one EventId inside a single VALUES
// constructor would both pass the anti-semi-join and collide on the
// clustered PK — losing the whole statement to a duplicate-key fault.
// First-write-wins matches the single-row contract exactly (a later copy
// of the same EventId is by definition the same immutable append-only
// row), so keeping the first occurrence is not merely convenient, it is
// the documented semantics.
var seen = new HashSet<Guid>(events.Count);
var distinct = new List<AuditEvent>(events.Count);
foreach (var evt in events)
{
if (evt is not null && seen.Add(evt.EventId))
{
distinct.Add(evt);
}
}
var inserted = 0;
for (var offset = 0; offset < distinct.Count; offset += IngestChunkRows)
{
var length = Math.Min(IngestChunkRows, distinct.Count - offset);
var chunk = distinct.GetRange(offset, length);
try
{
inserted += await InsertChunkAsync(chunk, commandTimeout, ct).ConfigureAwait(false);
}
catch (SqlException ex) when (
ex.Number == SqlErrorUniqueIndexViolation
|| ex.Number == SqlErrorPrimaryKeyViolation)
{
// A CONCURRENT writer committed one of this chunk's EventIds
// between the anti-semi-join and the insert (the same
// check-then-act window the single-row path documents), and the
// whole set-based statement rolled back with it. Fall back to
// the per-row path so the rows that are genuinely new still land
// — the batch is a throughput optimisation, never a correctness
// dependency. Every row is retried, including the one that
// collided, because InsertIfNotExistsAsync swallows its own
// duplicate-key fault as a no-op.
_logger.LogDebug(
ex,
"Set-based audit ingest chunk of {Count} row(s) hit a duplicate-key violation (error {SqlErrorNumber}); falling back to per-row inserts.",
chunk.Count,
ex.Number);
foreach (var evt in chunk)
{
await InsertIfNotExistsAsync(evt, ct).ConfigureAwait(false);
}
}
}
return inserted;
}
/// <summary>
/// Executes one set-based idempotent insert: a VALUES table constructor
/// anti-semi-joined against the committed rows, so a whole telemetry packet
/// costs ONE round trip instead of one <c>IF NOT EXISTS … INSERT</c> per row.
/// </summary>
/// <remarks>
/// Raw ADO.NET (rather than <c>ExecuteSqlInterpolated</c>) because the
/// statement's parameter count varies with the chunk size and every parameter
/// needs an explicit <see cref="SqlDbType"/>: the VALUES constructor's column
/// types are inferred from the first row's parameters, so leaving a null
/// <c>Target</c>/<c>SourceNode</c> untyped would give the derived column the
/// wrong type and defeat the seek on the anti-semi-join. The command enlists
/// in the DbContext's ambient transaction when one is open — the cached
/// telemetry dual-write runs the audit insert and the SiteCalls upsert inside
/// a single transaction and both must commit or roll back together.
/// </remarks>
private async Task<int> InsertChunkAsync(
IReadOnlyList<AuditEvent> chunk, TimeSpan? commandTimeout, CancellationToken ct)
{
var sql = new StringBuilder(256 + (chunk.Count * 64));
sql.Append("INSERT INTO dbo.AuditLog (").Append(CanonicalColumnList).Append(")\n");
sql.Append("SELECT v.EventId, v.OccurredAtUtc, v.Actor, v.Action, v.Outcome, v.Category, ");
sql.Append("v.Target, v.SourceNode, v.CorrelationId, v.DetailsJson\n");
sql.Append("FROM (VALUES\n");
for (var i = 0; i < chunk.Count; i++)
{
if (i > 0)
{
sql.Append(",\n");
}
sql.Append(" (@e").Append(i)
.Append(",@t").Append(i)
.Append(",@a").Append(i)
.Append(",@n").Append(i)
.Append(",@o").Append(i)
.Append(",@c").Append(i)
.Append(",@g").Append(i)
.Append(",@s").Append(i)
.Append(",@r").Append(i)
.Append(",@d").Append(i)
.Append(')');
}
sql.Append("\n) AS v (").Append(CanonicalColumnList).Append(")\n");
sql.Append("WHERE NOT EXISTS (SELECT 1 FROM dbo.AuditLog x WHERE x.EventId = v.EventId);");
var conn = _context.Database.GetDbConnection();
var openedHere = false;
if (conn.State != ConnectionState.Open)
{
await conn.OpenAsync(ct).ConfigureAwait(false);
openedHere = true;
}
try
{
await using var cmd = conn.CreateCommand();
cmd.CommandText = sql.ToString();
cmd.Transaction = _context.Database.CurrentTransaction?.GetDbTransaction();
if (commandTimeout is { } timeout)
{
cmd.CommandTimeout = (int)timeout.TotalSeconds;
}
for (var i = 0; i < chunk.Count; i++)
{
var evt = chunk[i];
// Same canonical projection as the single-row path: UTC-kind
// OccurredAtUtc, empty Actor collapses to NULL, Outcome/Category
// bound as their varchar storage form.
var occurred = DateTime.SpecifyKind(evt.OccurredAtUtc.UtcDateTime, DateTimeKind.Utc);
object actor = string.IsNullOrEmpty(evt.Actor) ? DBNull.Value : evt.Actor;
AddParameter(cmd, "@e" + i, SqlDbType.UniqueIdentifier, size: 0, evt.EventId);
AddParameter(cmd, "@t" + i, SqlDbType.DateTime2, size: 0, occurred);
AddParameter(cmd, "@a" + i, SqlDbType.NVarChar, size: 256, actor);
AddParameter(cmd, "@n" + i, SqlDbType.VarChar, size: 64, evt.Action);
AddParameter(cmd, "@o" + i, SqlDbType.VarChar, size: 16, evt.Outcome.ToString());
AddParameter(cmd, "@c" + i, SqlDbType.VarChar, size: 32, evt.Category);
AddParameter(cmd, "@g" + i, SqlDbType.NVarChar, size: 256, evt.Target);
AddParameter(cmd, "@s" + i, SqlDbType.VarChar, size: 64, evt.SourceNode);
AddParameter(cmd, "@r" + i, SqlDbType.UniqueIdentifier, size: 0, evt.CorrelationId);
AddParameter(cmd, "@d" + i, SqlDbType.NVarChar, size: -1, evt.DetailsJson);
}
return await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
}
finally
{
if (openedHere)
{
await conn.CloseAsync().ConfigureAwait(false);
}
}
}
/// <summary>
/// Binds one explicitly-typed parameter. A null CLR value binds as
/// <see cref="DBNull"/> while KEEPING its declared <see cref="SqlDbType"/>,
/// which is what makes the VALUES constructor's derived column types stable
/// regardless of which rows happen to carry nulls.
/// </summary>
private static void AddParameter(
System.Data.Common.DbCommand cmd, string name, SqlDbType type, int size, object? value)
{
var p = (SqlParameter)cmd.CreateParameter();
p.ParameterName = name;
p.SqlDbType = type;
if (size != 0)
{
p.Size = size;
}
p.Value = value ?? DBNull.Value;
cmd.Parameters.Add(p);
}
/// <inheritdoc />
public async Task<IReadOnlyList<AuditEvent>> QueryAsync(
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default)
@@ -229,13 +439,23 @@ VALUES
/// <inheritdoc />
public async Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default)
{
// The drop-and-rebuild batch below runs via
// The switch batch below runs via
// ExecuteSqlRaw with NO EF user-transaction — it carries its own server-side
// BEGIN TRANSACTION / TRY-CATCH / ROLLBACK — so the DbContext's retrying
// execution strategy (EnableRetryOnFailure) MAY auto-replay the whole batch on
// a transient fault. That replay is safe: every step is IF-EXISTS / IF-NOT-EXISTS
// guarded and the staging table is GUID-suffixed, so a re-run is idempotent.
//
// ALIGNED UNIQUENESS (WP2.2): there is no longer an index drop/rebuild around
// the SWITCH. EventId uniqueness is now enforced solely by the clustered
// PK_AuditLog (EventId, OccurredAtUtc), which is partition-aligned on
// ps_AuditLog_Month(OccurredAtUtc) — so ALTER TABLE … SWITCH PARTITION has no
// non-aligned index to object to. The former dance dropped UX_AuditLog_EventId,
// switched, then rebuilt it OFFLINE inside the same transaction: a whole-table
// unique-index build blocking every writer for the duration of the purge, and a
// window in which the idempotency-supporting index did not exist at all. Both
// are gone. See migration AlignAuditLogEventIdUniqueness for the reasoning.
//
// Maintenance timeout in whole seconds (ADO.NET CommandTimeout unit). Null leaves the
// provider default in place. See AuditLogPurgeOptions.MaintenanceCommandTimeoutMinutes /
// arch-review 04 S2 for why the ~30s default is unsafe for the switch-out dance.
@@ -270,8 +490,13 @@ VALUES
BEGIN TRY
BEGIN TRANSACTION;
-- 1. Drop the non-aligned unique index. ALTER TABLE SWITCH refuses
-- to run while it exists.
-- 1. Defensive cleanup for databases created before
-- AlignAuditLogEventIdUniqueness: the migration drops the
-- non-aligned UX_AuditLog_EventId, but a database restored from an
-- older backup could still carry it and SWITCH refuses to run while
-- a non-aligned unique index exists. Dropping it here is idempotent
-- and permanent — the aligned clustered PK is the only uniqueness
-- enforcement the ingest path needs, so there is nothing to rebuild.
IF EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
DROP INDEX UX_AuditLog_EventId ON dbo.AuditLog;
@@ -313,31 +538,19 @@ VALUES
-- 4. Drop staging — the rows are discarded here. This is the purge.
DROP TABLE dbo.[{stagingTableName}];
-- 5. Rebuild the non-aligned unique index. Live traffic that hit the
-- table during steps 1-4 saw composite-PK uniqueness only; from
-- here on, single-column EventId uniqueness is restored.
CREATE UNIQUE NONCLUSTERED INDEX UX_AuditLog_EventId ON dbo.AuditLog (EventId) ON [PRIMARY];
COMMIT TRANSACTION;
END TRY
BEGIN CATCH
IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION;
-- Best-effort staging cleanup. The DROP INDEX in step 1 is now
-- rolled back (so the index is back), but the staging table from
-- step 2 may or may not survive the rollback depending on the
-- failure point. Guard the DROP so a missing staging table doesn't
-- mask the original error.
-- Best-effort staging cleanup. The staging table from step 2 may or
-- may not survive the rollback depending on the failure point. Guard
-- the DROP so a missing staging table doesn't mask the original error.
-- Nothing else needs repairing: uniqueness lives on the clustered PK,
-- which the switch never touches, so a failed purge can no longer
-- leave the live table without its idempotency enforcement.
IF OBJECT_ID('dbo.[{stagingTableName}]', 'U') IS NOT NULL DROP TABLE dbo.[{stagingTableName}];
-- Idempotent index rebuild — covers the niche case where ROLLBACK
-- failed to restore UX_AuditLog_EventId (or the failure happened
-- AFTER the COMMIT, which shouldn't be possible inside this TRY
-- but is cheap insurance). Without this, a failed switch could
-- leave the live table without its idempotency-supporting index.
IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
CREATE UNIQUE NONCLUSTERED INDEX UX_AuditLog_EventId ON dbo.AuditLog (EventId) ON [PRIMARY];
-- Surface the original error to the caller — the purge actor logs
-- and continues with the next boundary.
THROW;