Merge branch 'worktree-agent-a3b474b485c0288de' into arch-review-remediation

This commit is contained in:
Joseph Doherty
2026-08-14 21:15:31 -04:00
29 changed files with 3790 additions and 266 deletions
+10 -6
View File
@@ -78,13 +78,15 @@ builder.Property(e => e.ExecutionId)
"CAST(JSON_VALUE(DetailsJson,'$.executionId') AS uniqueidentifier)", stored: true) "CAST(JSON_VALUE(DetailsJson,'$.executionId') AS uniqueidentifier)", stored: true)
.ValueGeneratedOnAddOrUpdate(); .ValueGeneratedOnAddOrUpdate();
// Composite PK includes OccurredAtUtc for partition alignment // Composite PK includes OccurredAtUtc for partition alignment — and is the ONLY
// EventId uniqueness enforcement (WP2.2 / AlignAuditLogEventIdUniqueness). EventId
// is a GUID minted once at the site alongside OccurredAtUtc, so a given EventId
// always lands in one partition and pair-uniqueness is EventId-uniqueness.
builder.HasKey(e => new { e.EventId, e.OccurredAtUtc }); builder.HasKey(e => new { e.EventId, e.OccurredAtUtc });
builder.HasIndex(e => e.EventId).IsUnique()
.HasDatabaseName("UX_AuditLog_EventId");
``` ```
The predecessor `UX_AuditLog_EventId` — a single-column unique index on `[PRIMARY]`, deliberately NOT partition-aligned — was dropped: a non-aligned index blocks `ALTER TABLE … SWITCH PARTITION`, so the retention purge had to drop it, switch, and rebuild it offline inside the switch transaction on every run.
**`TemplateConfiguration`** (representative of the domain-area configs) sets up the self-referencing parent FK, folder FK, cascade-delete relationships to attributes/alarms/scripts/compositions/native alarm sources, and the filtered unique index that enforces name uniqueness only on non-derived (base) templates. **`TemplateConfiguration`** (representative of the domain-area configs) sets up the self-referencing parent FK, folder FK, cascade-delete relationships to attributes/alarms/scripts/compositions/native alarm sources, and the filtered unique index that enforces name uniqueness only on non-derived (base) templates.
**`SiteCallEntityTypeConfiguration`** maps `SiteCall` to `dbo.SiteCalls` with a `TrackedOperationId` PK stored as `varchar(36)` (GUID in `"D"` format) so the column shape matches the wire format and the site SQLite store — one consistent format for operational debugging. **`SiteCallEntityTypeConfiguration`** maps `SiteCall` to `dbo.SiteCalls` with a `TrackedOperationId` PK stored as `varchar(36)` (GUID in `"D"` format) so the column shape matches the wire format and the site SQLite store — one consistent format for operational debugging.
@@ -108,11 +110,13 @@ VALUES
`FormattableString` interpolation parameterises every value so there is no injection surface. SQL error numbers `2601` and `2627` (unique-index violation) are swallowed as no-ops because the IF NOT EXISTS check has a race window; both the check-loser and the retrying telemetry path are semantically correct duplicates. `FormattableString` interpolation parameterises every value so there is no injection surface. SQL error numbers `2601` and `2627` (unique-index violation) are swallowed as no-ops because the IF NOT EXISTS check has a race window; both the check-loser and the retrying telemetry path are semantically correct duplicates.
`InsertManyIfNotExistsAsync` is the set-based form the ingest hot path uses: one `INSERT … SELECT … FROM (VALUES …) WHERE NOT EXISTS` per chunk of 100 rows (ten bound parameters per row against SQL Server's 2,100-parameter ceiling), so a telemetry packet costs one round trip instead of one per event. Because the anti-semi-join only sees committed rows, the packet is de-duplicated by `EventId` in C# first (first-write-wins, matching the single-row contract); a duplicate-key fault from a concurrent writer falls back to the per-row path so the batch is never a correctness dependency. It uses raw ADO.NET with explicitly typed `SqlParameter`s — the `VALUES` constructor derives its column types from the first row's parameters, so an untyped null would give the derived column the wrong type — and enlists in the DbContext's ambient transaction when one is open (the cached-telemetry dual-write).
`QueryAsync` builds LINQ predicates over `AuditLogRow` using `AsNoTracking()`, translating filter dimensions (`Channels`, `Kinds`, `Statuses`, `SourceSiteIds`, `SourceNodes`, `ExecutionId`, `ParentExecutionId`, time range) to server-side SQL IN/equality predicates and using keyset pagination on `(OccurredAtUtc DESC, EventId DESC)`. `QueryAsync` builds LINQ predicates over `AuditLogRow` using `AsNoTracking()`, translating filter dimensions (`Channels`, `Kinds`, `Statuses`, `SourceSiteIds`, `SourceNodes`, `ExecutionId`, `ParentExecutionId`, time range) to server-side SQL IN/equality predicates and using keyset pagination on `(OccurredAtUtc DESC, EventId DESC)`.
`GetExecutionTreeAsync` walks the `ParentExecutionId` graph in two phases: a loop climbs to the root (bounded at 32 levels), then a recursive CTE descends the full tree and LEFT JOINs back to `AuditLog` so stub nodes (purged or row-less executions) still appear with `RowCount = 0`. `GetExecutionTreeAsync` walks the `ParentExecutionId` graph in two phases: a loop climbs to the root (bounded at 32 levels), then a recursive CTE descends the full tree and LEFT JOINs back to `AuditLog` so stub nodes (purged or row-less executions) still appear with `RowCount = 0`.
`SwitchOutPartitionAsync` executes a drop-and-rebuild dance — dropping `UX_AuditLog_EventId`, creating a byte-identical staging table (including the computed-column definitions), switching the target partition to staging, dropping staging, and rebuilding the unique index — all inside a single `BEGIN TRY / BEGIN CATCH` block that guarantees the index is present whether the switch succeeds or rolls back. `SwitchOutPartitionAsync` creates a byte-identical staging table (including the computed-column definitions), switches the target partition to staging, and drops staging, inside a single `BEGIN TRY / BEGIN CATCH` block whose CATCH cleans up the staging table on any failure. No index is dropped or rebuilt: uniqueness rides the partition-aligned clustered PK, so the switch is metadata-only. A guarded defensive `DROP INDEX UX_AuditLog_EventId` remains at the head of the batch purely so a database restored from a pre-alignment backup still purges.
### IAuditService — config-change audit ### IAuditService — config-change audit
@@ -241,7 +245,7 @@ The host is running in production mode and `GetPendingMigrationsAsync` found una
### AuditLog partition switch fails mid-operation ### AuditLog partition switch fails mid-operation
`SwitchOutPartitionAsync` wraps the drop-and-rebuild dance in `BEGIN TRY / BEGIN CATCH`. On failure the CATCH block drops the staging table if it exists and rebuilds `UX_AuditLog_EventId` if it was dropped before the failure. The original exception is re-thrown so the Audit Log purge actor logs it and retries on the next daily tick. Verify that the `scadabridge_audit_purger` role still holds `ALTER ON SCHEMA::dbo` if the operation fails with a permissions error. `SwitchOutPartitionAsync` wraps the staging/switch batch in `BEGIN TRY / BEGIN CATCH`. On failure the CATCH block drops the staging table if it exists, so no orphaned `AuditLog_Staging_*` object is left behind. There is no index to repair — uniqueness lives on the clustered PK, which the switch never touches. The original exception is re-thrown so the Audit Log purge actor logs it and retries on the next daily tick. Verify that the `scadabridge_audit_purger` role still holds `ALTER ON SCHEMA::dbo` if the operation fails with a permissions error.
### Design-time `dotnet ef` tooling cannot find a connection string ### Design-time `dotnet ef` tooling cannot find a connection string
@@ -0,0 +1,24 @@
BEGIN TRANSACTION;
IF NOT EXISTS (
SELECT * FROM [__EFMigrationsHistory]
WHERE [MigrationId] = N'20260815004957_AlignAuditLogEventIdUniqueness'
)
BEGIN
IF EXISTS (SELECT 1 FROM sys.indexes
WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
DROP INDEX UX_AuditLog_EventId ON dbo.AuditLog;
END;
IF NOT EXISTS (
SELECT * FROM [__EFMigrationsHistory]
WHERE [MigrationId] = N'20260815004957_AlignAuditLogEventIdUniqueness'
)
BEGIN
INSERT INTO [__EFMigrationsHistory] ([MigrationId], [ProductVersion])
VALUES (N'20260815004957_AlignAuditLogEventIdUniqueness', N'10.0.7');
END;
COMMIT;
GO
+32 -4
View File
@@ -291,6 +291,27 @@ stay `Pending` for the next sweep. Cadence is short (default 5 s) when
non-empty, longer (default 30 s) when idle; telemetry runs on a dedicated non-empty, longer (default 30 s) when idle; telemetry runs on a dedicated
dispatcher. dispatcher.
**Central-side ingest is set-based.** `AuditLogIngestActor` writes a whole
packet with ONE `InsertManyIfNotExistsAsync` statement rather than one
`IF NOT EXISTS … INSERT` round trip per event; the cached-telemetry dual-write
similarly runs the whole packet in ONE transaction (set-based audit insert plus
one single-statement `SiteCalls` upsert per entry) instead of a transaction per
entry. Idempotency is unchanged: duplicates that repeat *within* a packet
collapse first-write-wins before the statement is built, duplicates *across*
packets are eliminated by the anti-semi-join, and any failure falls back to the
per-row / per-entry path — so the documented invariant that one bad row cannot
sink the rest of the batch still holds, it is simply no longer paid for on the
healthy path.
**Timeout ladder.** The ingest budget is deliberately the smallest on the path:
the site's Ask and the central gRPC handler's Ask are both 30 s, the actor's
own database budget is 20 s and the per-statement SQL timeout is 15 s. Before
this the three were identical, so they expired at the same instant and the
caller learned nothing but "it took 30 s" — no partial ack, no way to tell a
slow database from a wedged singleton. With the inner budgets strictly smaller,
a slow batch is abandoned by the actor first and the accepted-so-far ids are
still replied while the outer Asks are still waiting.
### Reconciliation pull (self-healing for missed telemetry) ### Reconciliation pull (self-healing for missed telemetry)
A central `SiteAuditReconciliationActor` periodically (default 5 min per site) A central `SiteAuditReconciliationActor` periodically (default 5 min per site)
@@ -490,12 +511,19 @@ MS SQL for direct-write events). Unredacted secrets never persist.
silently failing retention job would otherwise be invisible until the table grew silently failing retention job would otherwise be invisible until the table grew
unbounded. Per-boundary/per-channel error isolation is unchanged — a single failure unbounded. Per-boundary/per-channel error isolation is unchanged — a single failure
still never abandons the rest of the tick. still never abandons the rest of the tick.
- **Maintenance command timeout:** the switch-out drop-and-rebuild dance and each - **Maintenance command timeout:** the switch-out staging batch and each
per-channel `DELETE TOP` batch run with an explicit command timeout per-channel `DELETE TOP` batch run with an explicit command timeout
(`AuditLog:Purge:MaintenanceCommandTimeoutMinutes`, default 30, floor 1 min) rather (`AuditLog:Purge:MaintenanceCommandTimeoutMinutes`, default 30, floor 1 min) rather
than the ~30 s ADO.NET default, which could abort the metadata-only SWITCH mid-dance than the ~30 s ADO.NET default, which could abort the metadata-only SWITCH mid-batch
on a large or contended partition and leave the live table without on a large or contended partition and leave an orphaned staging table for the next
`UX_AuditLog_EventId` until a later tick's CATCH branch rebuilt it. tick's CATCH branch to clean up.
- **Partition-aligned uniqueness:** the switch no longer drops and rebuilds an index.
`EventId` uniqueness rides the clustered `PK_AuditLog (EventId, OccurredAtUtc)`,
which is aligned on `ps_AuditLog_Month(OccurredAtUtc)`, so `SWITCH PARTITION` has no
non-aligned unique index to object to. The predecessor `UX_AuditLog_EventId` forced
an offline whole-table index build inside the switch transaction — blocking every
audit writer for its duration — and left a window in which the idempotency-supporting
index did not exist at all. See migration `AlignAuditLogEventIdUniqueness`.
- **Per-channel retention overrides (M5.5 T3):** `AuditLog:PerChannelRetentionDays` - **Per-channel retention overrides (M5.5 T3):** `AuditLog:PerChannelRetentionDays`
is a dictionary keyed by canonical channel name (`ApiOutbound`, `DbOutbound`, is a dictionary keyed by canonical channel name (`ApiOutbound`, `DbOutbound`,
`Notification`, `ApiInbound`, `SecuredWrite` — all five `AuditChannel` values are `Notification`, `ApiInbound`, `SecuredWrite` — all five `AuditChannel` values are
@@ -95,9 +95,11 @@ The configuration database stores all central system data, organized by domain a
- `ParentExecutionId``CAST(JSON_VALUE(DetailsJson,'$.parentExecutionId') AS uniqueidentifier)` PERSISTED — spawner's `ExecutionId`; null for top-level runs. - `ParentExecutionId``CAST(JSON_VALUE(DetailsJson,'$.parentExecutionId') AS uniqueidentifier)` PERSISTED — spawner's `ExecutionId`; null for top-level runs.
- `IngestedAtUtc``CAST(SWITCHOFFSET(CAST(JSON_VALUE(DetailsJson,'$.ingestedAtUtc') AS datetimeoffset), 0) AS datetime2(7))` — central ingest timestamp; **not** persisted (SQL Server rejects PERSISTED on the non-deterministic `SWITCHOFFSET` expression). - `IngestedAtUtc``CAST(SWITCHOFFSET(CAST(JSON_VALUE(DetailsJson,'$.ingestedAtUtc') AS datetimeoffset), 0) AS datetime2(7))` — central ingest timestamp; **not** persisted (SQL Server rejects PERSISTED on the non-deterministic `SWITCHOFFSET` expression).
*Clustered primary key:* `(EventId, OccurredAtUtc)` — composite so the key is partition-aligned. `UX_AuditLog_EventId` (unique, non-aligned on `[PRIMARY]`) enforces global `EventId` uniqueness for `InsertIfNotExistsAsync` idempotency. *Clustered primary key:* `(EventId, OccurredAtUtc)` — composite so the key is partition-aligned, and the **sole** `EventId` uniqueness enforcement backing `InsertIfNotExistsAsync` idempotency. `EventId` is a GUID minted once at the emitting site in the same operation that stamps `OccurredAtUtc`, and neither field is ever re-stamped downstream, so a given `EventId` always arrives with the same `OccurredAtUtc` and can only map to one partition — pair-uniqueness is `EventId`-uniqueness for every row the system produces. The idempotency probe (`WHERE EventId = @id`) still seeks the clustered key's leading column, at the cost of one seek per partition rather than one seek overall.
*Indexes* (all non-clustered, partition-aligned on `ps_AuditLog_Month(OccurredAtUtc)` except `UX_AuditLog_EventId`): A non-aligned single-column `UX_AuditLog_EventId` on `[PRIMARY]` used to carry that uniqueness; it was dropped by the `AlignAuditLogEventIdUniqueness` migration because a non-aligned index blocks `ALTER TABLE … SWITCH PARTITION`, forcing the retention purge to drop it, switch, and rebuild it **offline inside the switch transaction** on every run.
*Indexes* (all non-clustered, partition-aligned on `ps_AuditLog_Month(OccurredAtUtc)`):
- `IX_AuditLog_OccurredAtUtc` (primary time-range index for global scans) - `IX_AuditLog_OccurredAtUtc` (primary time-range index for global scans)
- `IX_AuditLog_Site_Occurred (SourceSiteId, OccurredAtUtc)` (per-site filters) - `IX_AuditLog_Site_Occurred (SourceSiteId, OccurredAtUtc)` (per-site filters)
- `IX_AuditLog_CorrelationId (CorrelationId) WHERE CorrelationId IS NOT NULL` (drilldown from a single operation) - `IX_AuditLog_CorrelationId (CorrelationId) WHERE CorrelationId IS NOT NULL` (drilldown from a single operation)
@@ -135,6 +135,28 @@ Pinned)` on the EventStream (transition-only, mirroring
health-observable condition rather than a silent log line, and the latch clears health-observable condition rather than a silent log line, and the latch clears
with `Pinned=false` once a later tick makes progress. with `Pinned=false` once a later tick makes progress.
**The drain runs off the mailbox.** A reconciliation pass is unbounded work —
every site, up to a page ceiling of network pulls each, one upsert per row — so
it runs as a background task with a `PipeTo`-delivered completion message and a
single-flight guard, not as an actor message handler. A handler occupies the
actor for its whole duration, so a post-outage catch-up used to park telemetry
ingest, UI queries and KPI Asks behind it; those callers time out rather than
queue, which made a slow site look like a dead central. The single-flight guard
is raised and lowered ON the actor thread, so the per-site cursor and pinned-latch
dictionaries the pass mutates are still only ever touched by one task at a time,
and the mailbox supplies the memory barrier between consecutive passes. The
daily terminal-row purge uses the same shape. (`NotificationOutboxActor`'s
dispatch sweep is the in-repo reference.)
**The central upsert is one statement.** `SiteCallAuditRepository.UpsertAsync`
issues a single batch that runs the monotonic UPDATE first and INSERTs only when
nothing matched *and* the row genuinely does not exist — instead of an
unconditional insert-if-absent followed by the update, which cost two round trips
on every packet and wasted the insert half for every packet after the first. The
existence re-check is load-bearing: a zero row count also means "the monotonic
guard rejected this packet", and inserting there would fork the mirror with a
second row for an id that already exists.
## Retry / Discard Relay ## Retry / Discard Relay
Parked cached calls live in the owning site's S&F buffer. Operator Retry/Discard Parked cached calls live in the owning site's S&F buffer. Operator Retry/Discard
@@ -18,8 +18,9 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// ingest timestamp into DetailsJson (there is no promoted IngestedAtUtc /// ingest timestamp into DetailsJson (there is no promoted IngestedAtUtc
/// column — the value is a DetailsJson field set via /// column — the value is a DetailsJson field set via
/// <see cref="AuditRowProjection.WithIngestedAtUtc"/>) and inserted idempotently /// <see cref="AuditRowProjection.WithIngestedAtUtc"/>) and inserted idempotently
/// via <see cref="IAuditLogRepository.InsertIfNotExistsAsync"/> — duplicates are /// via <see cref="IAuditLogRepository.InsertManyIfNotExistsAsync"/> — duplicates
/// silently swallowed (first-write-wins). /// are silently swallowed (first-write-wins), whether they repeat inside one
/// packet or across packets.
/// </summary> /// </summary>
/// <remarks> /// <remarks>
/// <para> /// <para>
@@ -28,9 +29,11 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// consistent and the site is free to flip its local row to <c>Forwarded</c>. /// consistent and the site is free to flip its local row to <c>Forwarded</c>.
/// </para> /// </para>
/// <para> /// <para>
/// Audit-write failures must NEVER abort the user-facing action. The actor /// Audit-write failures must NEVER abort the user-facing action. Each message
/// wraps each repository call in its own try/catch so a single bad row cannot /// is written as ONE set-based statement (or one transaction, for the cached
/// cause the rest of the batch to be lost, and it guards scope/repository /// dual-write); if that fails the actor retries the same work row-by-row inside
/// per-row try/catch, so a single bad row still cannot cause the rest of the
/// batch to be lost. It also guards scope/repository
/// resolution so a transient DI fault cannot restart the singleton — those /// resolution so a transient DI fault cannot restart the singleton — those
/// catches are what keep this actor alive across handler throws, not the /// catches are what keep this actor alive across handler throws, not the
/// supervisor strategy. The <see cref="SupervisorStrategy"/> override returns /// supervisor strategy. The <see cref="SupervisorStrategy"/> override returns
@@ -50,6 +53,31 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// </remarks> /// </remarks>
public class AuditLogIngestActor : ReceiveActor public class AuditLogIngestActor : ReceiveActor
{ {
/// <summary>
/// Overall budget for one ingest message's database work, deliberately
/// SHORTER than the gRPC Ask that wraps it.
/// </summary>
/// <remarks>
/// The path used to stack three identical 30 s budgets — the site's Ask
/// (<c>CommunicationOptions.NotificationForwardTimeout</c>), the central gRPC
/// handler's Ask (<c>SiteStreamGrpcServer.AuditIngestAskTimeout</c>) and the
/// ADO.NET command default — so they all expired at the same instant. The
/// caller therefore learned nothing except "it took 30 s": no partial ack, no
/// distinction between a slow database and a wedged singleton. Making the
/// innermost budget strictly smallest means a slow batch is abandoned by the
/// actor FIRST, with the accepted-so-far ids still replied, while the outer
/// Asks are still waiting.
/// </remarks>
internal static readonly TimeSpan IngestBudget = TimeSpan.FromSeconds(20);
/// <summary>
/// Per-statement SQL timeout for the ingest write, strictly inside
/// <see cref="IngestBudget"/> so a single wedged statement surfaces as a SQL
/// timeout the per-batch catch can log, rather than consuming the whole
/// actor-level budget and starving the rest of the batch.
/// </summary>
internal static readonly TimeSpan IngestSqlCommandTimeout = TimeSpan.FromSeconds(15);
private readonly IServiceProvider? _serviceProvider; private readonly IServiceProvider? _serviceProvider;
private readonly IAuditLogRepository? _injectedRepository; private readonly IAuditLogRepository? _injectedRepository;
private readonly ILogger<AuditLogIngestActor> _logger; private readonly ILogger<AuditLogIngestActor> _logger;
@@ -185,39 +213,75 @@ public class AuditLogIngestActor : ReceiveActor
DateTime nowUtc, DateTime nowUtc,
List<Guid> accepted) List<Guid> accepted)
{ {
// Stamp the ingest timestamp here, not at the site. Redact BEFORE the
// IngestedAtUtc stamp so the redacted copy carries the central-side
// ingest timestamp. The redactor is contract-bound to never throw; a null
// redactor (test composition root, no IAuditRedactor registered) falls
// back to the SafeDefault rather than pass-through, so HTTP header
// redaction always runs. IngestedAtUtc is a DetailsJson field on the
// canonical record, so stamp it via the projection helper.
var safeRedactor = redactor ?? SafeDefaultAuditRedactor.Instance;
var projected = new List<AuditEvent>(cmd.Events.Count);
foreach (var evt in cmd.Events) foreach (var evt in cmd.Events)
{ {
var filtered = safeRedactor.Apply(evt);
projected.Add(AuditRowProjection.WithIngestedAtUtc(filtered, nowUtc));
}
// ONE set-based statement for the whole packet (WP2.2) instead of one
// IF NOT EXISTS … INSERT round trip per event. A telemetry packet of 200
// rows used to cost 200 sequential round trips on the central singleton's
// dispatcher; it now costs two. Idempotency is unchanged — duplicates
// within the packet collapse first-write-wins, duplicates against
// committed rows are eliminated by the anti-semi-join, and a concurrent
// writer degrades to the repository's per-row fallback.
using var budget = new CancellationTokenSource(IngestBudget);
try try
{ {
// Stamp the ingest timestamp here, not at the site. The await repository
// repository's duplicate-key hardening already swallows .InsertManyIfNotExistsAsync(projected, IngestSqlCommandTimeout, budget.Token)
// duplicate-key races, so the same id arriving twice (site .ConfigureAwait(false);
// retry, reconciliation) is a silent no-op.
// Redact BEFORE the IngestedAtUtc stamp so the redacted // A batch that returned without throwing means every row is now
// copy carries the central-side ingest timestamp. The redactor // present — inserted here or already committed by an earlier delivery.
// is contract-bound to never throw. A null // Both count as accepted: the storage state is consistent and the site
// redactor (test composition root, no IAuditRedactor // is free to flip its local rows to Forwarded.
// registered) now falls back to the SafeDefault rather than foreach (var evt in cmd.Events)
// pass-through, so HTTP header redaction always runs. {
// IngestedAtUtc is a DetailsJson field on
// the canonical record, so stamp it via the projection helper.
var safeRedactor = redactor ?? SafeDefaultAuditRedactor.Instance;
var filtered = safeRedactor.Apply(evt);
var ingested = AuditRowProjection.WithIngestedAtUtc(filtered, nowUtc);
await repository.InsertIfNotExistsAsync(ingested).ConfigureAwait(false);
accepted.Add(evt.EventId); accepted.Add(evt.EventId);
} }
}
catch (Exception ex) catch (Exception ex)
{
// The batch is a throughput optimisation, NOT a change to the
// failure grain. The documented invariant — "a single bad row cannot
// cause the rest of the batch to be lost" — is preserved by falling
// back to the per-row path here, so a poison row still costs only
// itself. Re-running rows the failed batch may already have committed
// is safe: every insert is idempotent on EventId.
_logger.LogWarning(ex,
"Set-based ingest of {Count} audit event(s) failed; falling back to per-row inserts so one bad row does not sink the batch.",
cmd.Events.Count);
for (var i = 0; i < projected.Count; i++)
{
try
{
await repository.InsertIfNotExistsAsync(projected[i], budget.Token).ConfigureAwait(false);
accepted.Add(cmd.Events[i].EventId);
}
catch (Exception rowEx)
{ {
// Per-row catch — one bad row never sinks the whole batch. // Per-row catch — one bad row never sinks the whole batch.
// The row stays Pending at the site; the next drain retries. // The row stays Pending at the site; the next drain retries.
// Bump the central health counter so a // Bump the central health counter so a sustained insert-throw
// sustained insert-throw failure surfaces on the dashboard. // failure surfaces on the dashboard.
try { failureCounter?.Increment(); } try { failureCounter?.Increment(); }
catch { /* counter must never throw — defence in depth */ } catch { /* counter must never throw — defence in depth */ }
_logger.LogError(ex, _logger.LogError(rowEx,
"Failed to persist audit event {EventId} during batch ingest; row will be retried by the site.", "Failed to persist audit event {EventId} during batch ingest; row will be retried by the site.",
evt.EventId); cmd.Events[i].EventId);
}
} }
} }
} }
@@ -262,6 +326,24 @@ public class AuditLogIngestActor : ReceiveActor
var strategy = dbContext.Database.CreateExecutionStrategy(); var strategy = dbContext.Database.CreateExecutionStrategy();
// Fast path (WP2.2): the WHOLE packet in ONE transaction — one
// set-based audit insert plus one single-statement upsert per entry.
// The predecessor opened a transaction PER entry and issued three
// statements inside it (IF NOT EXISTS insert, then the two-statement
// SiteCalls upsert), so a 50-entry packet cost ~200 round trips and 50
// commits. If anything faults, the per-entry loop below re-runs the
// packet with its original per-entry isolation, so the documented
// "one entry's failure does not abort the others" invariant survives —
// it is simply no longer paid for on the healthy path.
using var budget = new CancellationTokenSource(IngestBudget);
if (await TryIngestCachedBatchAsync(
strategy, dbContext, auditRepo, siteCallRepo, redactor, cmd, accepted, budget.Token)
.ConfigureAwait(false))
{
replyTo.Tell(new IngestCachedTelemetryReply(accepted));
return;
}
foreach (var entry in cmd.Entries) foreach (var entry in cmd.Entries)
{ {
try try
@@ -330,6 +412,90 @@ public class AuditLogIngestActor : ReceiveActor
replyTo.Tell(new IngestCachedTelemetryReply(accepted)); replyTo.Tell(new IngestCachedTelemetryReply(accepted));
} }
/// <summary>
/// Attempts the whole cached-telemetry packet as ONE transaction: a single
/// set-based audit insert followed by one monotonic <c>SiteCalls</c> upsert
/// per entry. Returns <see langword="true"/> when it committed (and only then
/// appends to <paramref name="accepted"/>), <see langword="false"/> when the
/// caller should fall back to the per-entry transaction loop.
/// </summary>
/// <remarks>
/// The batch is all-or-nothing by construction — it is a single transaction —
/// which is why a failure MUST fall back rather than report partial success:
/// the per-entry loop is what turns one poison entry into one lost entry
/// instead of a lost packet. Nothing is appended to
/// <paramref name="accepted"/> until the commit returns, so a failed attempt
/// leaves the caller's list untouched and the retry starts from a clean slate.
/// </remarks>
private async Task<bool> TryIngestCachedBatchAsync(
Microsoft.EntityFrameworkCore.Storage.IExecutionStrategy strategy,
ScadaBridgeDbContext dbContext,
IAuditLogRepository auditRepo,
ISiteCallAuditRepository siteCallRepo,
IAuditRedactor? redactor,
IngestCachedTelemetryCommand cmd,
List<Guid> accepted,
CancellationToken ct)
{
try
{
await strategy.ExecuteAsync(async () =>
{
await using var tx = await dbContext.Database
.BeginTransactionAsync(ct)
.ConfigureAwait(false);
// One central-side instant for the whole packet so a join across
// the two tables sees matching timestamps (debugging convenience,
// not a correctness invariant).
var ingestedAt = DateTime.UtcNow;
var safeRedactor = redactor ?? SafeDefaultAuditRedactor.Instance;
var auditRows = new List<AuditEvent>(cmd.Entries.Count);
foreach (var entry in cmd.Entries)
{
// Only the AuditLog row's payload columns are redactable;
// SiteCalls carries operational state only (status, retry
// count) and is left untouched.
var filteredAudit = safeRedactor.Apply(entry.Audit);
auditRows.Add(AuditRowProjection.WithIngestedAtUtc(filteredAudit, ingestedAt));
}
await auditRepo
.InsertManyIfNotExistsAsync(auditRows, IngestSqlCommandTimeout, ct)
.ConfigureAwait(false);
foreach (var entry in cmd.Entries)
{
await siteCallRepo
.UpsertAsync(entry.SiteCall with { IngestedAtUtc = ingestedAt }, ct)
.ConfigureAwait(false);
}
await tx.CommitAsync(ct).ConfigureAwait(false);
}).ConfigureAwait(false);
}
catch (Exception ex)
{
// No health-counter bump here — the per-entry retry is the authority
// on whether this packet genuinely failed, and double-counting a
// batch that the fallback then writes successfully would make the
// dashboard read as a sustained fault during normal contention.
_logger.LogWarning(
ex,
"Batched cached-telemetry dual-write of {Count} entr(ies) failed; falling back to per-entry transactions.",
cmd.Entries.Count);
return false;
}
foreach (var entry in cmd.Entries)
{
accepted.Add(entry.Audit.EventId);
}
return true;
}
/// <summary> /// <summary>
/// Fallback handler installed on the single-repository test ctor — that /// Fallback handler installed on the single-repository test ctor — that
/// ctor has no DbContext and no <see cref="ISiteCallAuditRepository"/>, so /// ctor has no DbContext and no <see cref="ISiteCallAuditRepository"/>, so
@@ -17,8 +17,9 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// for monthly boundaries whose latest <c>OccurredAtUtc</c> is older /// for monthly boundaries whose latest <c>OccurredAtUtc</c> is older
/// than <c>DateTime.UtcNow - RetentionDays</c>.</item> /// than <c>DateTime.UtcNow - RetentionDays</c>.</item>
/// <item>For each eligible boundary, calls /// <item>For each eligible boundary, calls
/// <see cref="IAuditLogRepository.SwitchOutPartitionAsync"/> which runs /// <see cref="IAuditLogRepository.SwitchOutPartitionAsync"/>, a
/// the drop-and-rebuild dance around <c>UX_AuditLog_EventId</c>.</item> /// metadata-only staging-table switch (WP2.2 removed the index
/// drop/rebuild that used to bracket it).</item>
/// <item>Publishes <see cref="AuditLogPurgedEvent"/> on the actor-system /// <item>Publishes <see cref="AuditLogPurgedEvent"/> on the actor-system
/// EventStream so the central health collector + ops surfaces /// EventStream so the central health collector + ops surfaces
/// can subscribe without coupling to this actor.</item> /// can subscribe without coupling to this actor.</item>
@@ -26,11 +27,10 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// </summary> /// </summary>
/// <remarks> /// <remarks>
/// <para> /// <para>
/// <b>Daily cadence.</b> Partition switch is metadata-only but the /// <b>Daily cadence.</b> The partition switch is metadata-only, but it still
/// drop-and-rebuild dance briefly removes <c>UX_AuditLog_EventId</c>; running /// takes schema-modification locks on a table the ingest path is writing to;
/// more often than necessary trades unique-index rebuild outages for /// running more often than necessary trades contention for negligible freshness
/// negligible freshness wins. The default 24-hour interval matches /// wins. The default 24-hour interval matches alog.md §10's retention policy.
/// alog.md §10's retention policy.
/// </para> /// </para>
/// <para> /// <para>
/// <b>Continue-on-error.</b> A single boundary that throws (transient SQL /// <b>Continue-on-error.</b> A single boundary that throws (transient SQL
@@ -10,12 +10,12 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// </summary> /// </summary>
/// <remarks> /// <remarks>
/// <para> /// <para>
/// The purge actor is a daily-cadence singleton, not a hot-loop, because /// The purge actor is a daily-cadence singleton, not a hot-loop. The
/// partition-switch I/O is metadata-only but the drop-and-rebuild dance /// partition switch itself is metadata-only (since WP2.2's
/// briefly removes the <c>UX_AuditLog_EventId</c> unique index — running /// <c>AlignAuditLogEventIdUniqueness</c> it no longer drops and rebuilds a
/// more often than necessary trades index-rebuild outages for marginal /// non-aligned unique index around it), but it still takes schema-modification
/// freshness gains. Lower this only when an operator can prove they need /// locks and competes with ingest for the same table. Lower this only when an
/// sub-daily purge granularity. /// operator can prove they need sub-daily purge granularity.
/// </para> /// </para>
/// <para> /// <para>
/// <see cref="IntervalOverride"/> exists for tests to drop the cadence to /// <see cref="IntervalOverride"/> exists for tests to drop the cadence to
@@ -58,15 +58,14 @@ public sealed class AuditLogPurgeOptions
/// <summary> /// <summary>
/// Per-command timeout (in minutes) for the maintenance SQL the purge tick issues — /// Per-command timeout (in minutes) for the maintenance SQL the purge tick issues —
/// both the partition switch-out drop-and-rebuild dance /// both the partition switch-out staging batch
/// (<see cref="ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories.IAuditLogRepository.SwitchOutPartitionAsync"/>) /// (<see cref="ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories.IAuditLogRepository.SwitchOutPartitionAsync"/>)
/// and each per-channel <c>DELETE TOP</c> batch. Default 30 minutes. /// and each per-channel <c>DELETE TOP</c> batch. Default 30 minutes.
/// </summary> /// </summary>
/// <remarks> /// <remarks>
/// The ADO.NET default command timeout is ~30 seconds. On a large or contended partition the /// The ADO.NET default command timeout is ~30 seconds. On a large or contended partition the
/// SWITCH dance (which briefly drops <c>UX_AuditLog_EventId</c>) can exceed that and abort /// SWITCH batch can exceed that and abort mid-flight, leaving an orphaned staging table for the
/// mid-flight — leaving the live table without its idempotency-supporting unique index until a /// next tick's CATCH branch to clean up (arch-review 04, S2). A generous maintenance timeout
/// later tick's CATCH branch rebuilds it (arch-review 04, S2). A generous maintenance timeout
/// lets the metadata-only switch complete rather than self-locking. Resolved via /// lets the metadata-only switch complete rather than self-locking. Resolved via
/// <see cref="ResolvedMaintenanceCommandTimeout"/>, clamped to a 1-minute floor. /// <see cref="ResolvedMaintenanceCommandTimeout"/>, clamped to a 1-minute floor.
/// </remarks> /// </remarks>
@@ -34,6 +34,62 @@ public interface IAuditLogRepository
/// <returns>A task that represents the asynchronous operation.</returns> /// <returns>A task that represents the asynchronous operation.</returns>
Task InsertIfNotExistsAsync(AuditEvent evt, CancellationToken ct = default); Task InsertIfNotExistsAsync(AuditEvent evt, CancellationToken ct = default);
/// <summary>
/// Set-based form of <see cref="InsertIfNotExistsAsync"/>: inserts every
/// event in <paramref name="events"/> that does not already exist, in as few
/// round trips as the provider allows, and returns the number of rows
/// actually written. First-write-wins idempotency is unchanged — an EventId
/// already present is silently skipped.
/// </summary>
/// <remarks>
/// <para>
/// <b>Duplicate EventIds are tolerated both within and across packets.</b>
/// Duplicates inside one call collapse to the first occurrence before the
/// statement is built (an append-only row is immutable, so later copies of an
/// EventId carry the same content); duplicates against already-committed rows
/// are eliminated by the anti-semi-join. A concurrent writer that commits a
/// row mid-statement is handled by falling back to the per-row path, so the
/// batch is a throughput optimisation and never a correctness dependency.
/// </para>
/// <para>
/// <b>Default implementation.</b> The interface supplies a per-row loop so a
/// test double or an alternative store keeps working unchanged; the EF Core
/// implementation overrides it with a genuine set-based statement. Callers on
/// the ingest hot path should always prefer this method — the per-row form
/// cost one <c>IF NOT EXISTS … INSERT</c> round trip per audit event.
/// </para>
/// </remarks>
/// <param name="events">Audit events to insert; may contain duplicates.</param>
/// <param name="commandTimeout">
/// Optional per-statement timeout. The ingest actor passes a budget strictly
/// SHORTER than the gRPC Ask that wraps it, so the reply is produced before
/// the caller gives up rather than at the same instant.
/// </param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A task that resolves to the number of rows inserted (duplicates excluded).</returns>
async Task<int> InsertManyIfNotExistsAsync(
IReadOnlyList<AuditEvent> events,
TimeSpan? commandTimeout = null,
CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(events);
var inserted = 0;
var seen = new HashSet<Guid>(events.Count);
foreach (var evt in events)
{
if (evt is null || !seen.Add(evt.EventId))
{
continue;
}
await InsertIfNotExistsAsync(evt, ct).ConfigureAwait(false);
inserted++;
}
return inserted;
}
/// <summary> /// <summary>
/// Returns up to <see cref="AuditLogPaging.PageSize"/> rows matching /// Returns up to <see cref="AuditLogPaging.PageSize"/> rows matching
/// <paramref name="filter"/>, ordered by <c>(OccurredAtUtc DESC, EventId DESC)</c>. /// <paramref name="filter"/>, ordered by <c>(OccurredAtUtc DESC, EventId DESC)</c>.
@@ -60,36 +116,32 @@ public interface IAuditLogRepository
/// </summary> /// </summary>
/// <remarks> /// <remarks>
/// <para> /// <para>
/// <b>Drop-and-rebuild dance.</b> <c>UX_AuditLog_EventId</c> is intentionally /// <b>Partition-aligned uniqueness — no index drop.</b> EventId uniqueness is
/// non-partition-aligned (it lives on <c>[PRIMARY]</c> so single-column /// enforced by the clustered <c>PK_AuditLog (EventId, OccurredAtUtc)</c>,
/// EventId uniqueness — required by <see cref="InsertIfNotExistsAsync"/> — /// which is aligned on <c>ps_AuditLog_Month(OccurredAtUtc)</c>, so
/// can be enforced cheaply). SQL Server rejects /// <c>ALTER TABLE … SWITCH PARTITION</c> has no non-aligned unique index to
/// <c>ALTER TABLE … SWITCH PARTITION</c> while a non-aligned unique index /// object to. The implementation creates a staging table with byte-identical
/// is present, so the implementation drops the index, creates a staging /// schema, switches the partition's data into staging, and drops staging
/// table with byte-identical schema, switches the partition's data into /// (discarding the rows). Nothing is dropped from the live table, so there is
/// staging, drops staging (discarding the rows), and rebuilds the unique /// no window in which the idempotency enforcement is absent and no offline
/// index. The CATCH branch guarantees the index is rebuilt even on partial /// index rebuild inside the switch transaction.
/// failure so the table never returns to live traffic without its
/// idempotency-supporting index.
/// </para> /// </para>
/// <para> /// <para>
/// <b>Outage window.</b> The dance briefly removes the unique index, so /// The predecessor design carried a non-aligned
/// concurrent <see cref="InsertIfNotExistsAsync"/> calls during the switch /// <c>UX_AuditLog_EventId</c> on <c>[PRIMARY]</c> that had to be dropped and
/// could in principle race past the IF NOT EXISTS check without the index /// rebuilt around every switch. A defensive guarded <c>DROP INDEX</c> remains
/// catching the duplicate. This is acceptable for the daily purge cadence /// in the batch so a database restored from a pre-alignment backup still
/// — the inserts that the IF NOT EXISTS check guards are themselves rare /// purges; it is a one-way cleanup, never rebuilt.
/// enough that a sub-second collision window is operationally negligible,
/// and the composite PK still rejects same-(EventId, OccurredAtUtc) rows.
/// </para> /// </para>
/// </remarks> /// </remarks>
/// <param name="monthBoundary">Lower-bound datetime of the monthly partition to switch out.</param> /// <param name="monthBoundary">Lower-bound datetime of the monthly partition to switch out.</param>
/// <param name="commandTimeout"> /// <param name="commandTimeout">
/// Optional per-command timeout for the maintenance SQL (the row-count sample plus the /// Optional per-command timeout for the maintenance SQL (the row-count sample plus the
/// drop-and-rebuild dance). When null the provider default applies. The purge actor passes /// staging/switch batch). When null the provider default applies. The purge actor passes
/// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout"/> /// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout"/>
/// (default 30 min) because the ~30s ADO.NET default can abort the switch mid-dance on a large /// (default 30 min) because the ~30s ADO.NET default can abort the switch mid-batch on a large
/// or contended partition, leaving the table without <c>UX_AuditLog_EventId</c> until the next /// or contended partition, leaving an orphaned staging table for the next tick's CATCH to clean
/// tick recovers (arch-review 04, S2). /// up (arch-review 04, S2).
/// </param> /// </param>
/// <param name="ct">Cancellation token.</param> /// <param name="ct">Cancellation token.</param>
/// <returns>A task that resolves to the approximate number of rows discarded by the partition switch.</returns> /// <returns>A task that resolves to the approximate number of rows discarded by the partition switch.</returns>
@@ -46,10 +46,25 @@ public interface INotificationOutboxRepository
Task<Notification?> GetByIdAsync(string notificationId, CancellationToken cancellationToken = default); Task<Notification?> GetByIdAsync(string notificationId, CancellationToken cancellationToken = default);
/// <summary> /// <summary>
/// Marks <paramref name="n"/> modified and persists it (status transitions). /// Persists <paramref name="n"/>'s <b>delivery-state</b> columns —
/// Commits internally — this call is its own transaction. /// <c>Status</c>, <c>RetryCount</c>, <c>LastError</c>, <c>ResolvedTargets</c>,
/// <c>LastAttemptAt</c>, <c>NextAttemptAt</c>, <c>DeliveredAt</c>. Commits
/// internally — this call is its own transaction.
/// </summary> /// </summary>
/// <param name="n">The notification to update.</param> /// <remarks>
/// <b>Scope is deliberately narrow.</b> Those seven columns are the ONLY
/// mutable state a notification has: everything else (identity, type, list,
/// subject, body, type data, source/origin attribution, enqueue and creation
/// timestamps) is written once at ingest and is immutable by contract. Every
/// caller — the dispatcher's per-attempt write and the operator retry/discard
/// one-shots — touches only this set. Implementations are therefore free to
/// issue a targeted UPDATE of these columns rather than rewriting the whole
/// row, which matters because the row carries <c>nvarchar(max)</c> body and
/// payload columns and the dispatcher writes on EVERY attempt. A future
/// caller that needs to change an immutable column must add its own method
/// rather than widening this one.
/// </remarks>
/// <param name="n">The notification whose delivery state should be persisted.</param>
/// <param name="cancellationToken">Cancellation token.</param> /// <param name="cancellationToken">Cancellation token.</param>
/// <returns>A task that completes when the notification has been persisted.</returns> /// <returns>A task that completes when the notification has been persisted.</returns>
Task UpdateAsync(Notification n, CancellationToken cancellationToken = default); Task UpdateAsync(Notification n, CancellationToken cancellationToken = default);
@@ -172,15 +172,18 @@ public class AuditLogEntityTypeConfiguration : IEntityTypeConfiguration<AuditLog
// ── Keys + indexes ─────────────────────────────────────────────────── // ── Keys + indexes ───────────────────────────────────────────────────
// Composite PK includes OccurredAtUtc — required by the monthly partition scheme // Composite PK includes OccurredAtUtc — required by the monthly partition scheme
// (ps_AuditLog_Month) so the clustered key is partition-aligned. EventId still // (ps_AuditLog_Month) so the clustered key is partition-aligned. It is ALSO the
// needs to be globally unique for InsertIfNotExistsAsync idempotency, so a // only uniqueness enforcement the ingest path needs: EventId is a GUID minted
// separate (non-aligned) unique index is declared on EventId alone. // once at the emitting site and never re-stamped, so a given EventId always
// arrives with the same OccurredAtUtc and can only ever land in one partition.
// Uniqueness of the pair is therefore uniqueness of EventId in practice, and the
// idempotency probe (WHERE EventId = @id) still seeks the clustered key's leading
// column. The predecessor non-aligned UX_AuditLog_EventId on [PRIMARY] was
// dropped by AlignAuditLogEventIdUniqueness — it existed only to give
// single-column uniqueness and its non-alignment forced an offline drop/rebuild
// around every partition-switch purge.
builder.HasKey(e => new { e.EventId, e.OccurredAtUtc }); builder.HasKey(e => new { e.EventId, e.OccurredAtUtc });
builder.HasIndex(e => e.EventId)
.IsUnique()
.HasDatabaseName("UX_AuditLog_EventId");
// Index names are locked for reconciliation/migration discoverability. The // Index names are locked for reconciliation/migration discoverability. The
// column SETS migrate to the canonical/computed shape (alog.md §4 semantics // column SETS migrate to the canonical/computed shape (alog.md §4 semantics
// preserved): Channel→Category, Site/Node/Execution/ParentExecution now read // preserved): Channel→Category, Site/Node/Execution/ParentExecution now read
@@ -0,0 +1,93 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
{
/// <summary>
/// Makes <c>dbo.AuditLog</c>'s EventId uniqueness <b>partition-aligned</b> by
/// dropping the non-aligned <c>UX_AuditLog_EventId</c> and leaving the clustered
/// <c>PK_AuditLog (EventId, OccurredAtUtc)</c> — already aligned on
/// <c>ps_AuditLog_Month(OccurredAtUtc)</c> — as the sole enforcement.
/// </summary>
/// <remarks>
/// <para>
/// <b>Why.</b> <c>ALTER TABLE … SWITCH PARTITION</c> refuses to run while a
/// non-aligned index exists on the table, so the monthly retention purge
/// (<c>AuditLogRepository.SwitchOutPartitionAsync</c>) had to DROP
/// <c>UX_AuditLog_EventId</c>, switch, and then CREATE it again — an OFFLINE
/// whole-table unique-index build, inside the switch transaction, blocking every
/// audit writer for its duration. It also opened a window in which the index that
/// backs ingest idempotency did not exist at all, and a mid-dance failure could
/// leave the live table without it until a later tick's CATCH branch repaired it.
/// With alignment there is nothing to drop, so the switch is metadata-only and the
/// purge stops competing with ingest.
/// </para>
/// <para>
/// <b>Why dropping it is safe — EventId is globally unique by construction.</b>
/// The composite key enforces uniqueness of the PAIR, not of EventId alone, so in
/// principle the same EventId could now be stored twice under two different
/// <c>OccurredAtUtc</c> values (in two different partitions). That cannot happen
/// here: <c>EventId</c> is a GUID minted ONCE at the emitting site, in the same
/// operation that stamps <c>OccurredAtUtc</c>, and both travel together verbatim
/// through telemetry and reconciliation — nothing downstream re-stamps either
/// field. A given EventId therefore always arrives with the same OccurredAtUtc and
/// can only ever map to one partition, which makes pair-uniqueness equivalent to
/// EventId-uniqueness for every row this system produces. GUID collision across
/// partitions is not a real risk.
/// </para>
/// <para>
/// <b>The idempotency probe still seeks.</b> Both ingest forms test
/// <c>WHERE EventId = @id</c>, which is the LEADING column of the clustered PK, so
/// the probe remains an index seek. The cost changes shape rather than order: it
/// becomes one seek per partition (the partition column is not in the predicate, so
/// SQL Server cannot eliminate partitions) instead of a single seek on a
/// non-partitioned index. Against a monthly scheme that is a couple of dozen
/// shallow B-tree seeks — cheap, and paid on a path that now issues one statement
/// per telemetry packet rather than one per row.
/// </para>
/// <para>
/// <b>Edition note.</b> The alternative remedy — keeping the non-aligned index and
/// rebuilding it with <c>ONLINE = ON</c> outside the switch transaction — requires
/// Enterprise (or Azure SQL / Developer) edition; online index rebuild is not
/// available on Standard, which this deployment does not guarantee. Alignment
/// needs no edition-specific feature and removes the rebuild entirely, so it is
/// preferred regardless of edition.
/// </para>
/// <para>
/// <b>Down is a faithful reverse</b> and recreates the index on <c>[PRIMARY]</c>
/// exactly as <c>CollapseAuditLogToCanonical</c> created it. Reverting also
/// reinstates the SWITCH incompatibility, so the purge's guarded defensive
/// <c>DROP INDEX</c> (retained in <c>SwitchOutPartitionAsync</c> for databases
/// restored from pre-alignment backups) would remove it again on the next purge.
/// The partition function/scheme (<c>pf_AuditLog_Month</c> /
/// <c>ps_AuditLog_Month</c>) and every aligned index are untouched by both
/// directions.
/// </para>
/// </remarks>
public partial class AlignAuditLogEventIdUniqueness : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
// Raw, existence-guarded SQL rather than the scaffolded DropIndex: the
// AuditLog table is raw-SQL managed (partition scheme, persisted computed
// columns, append-only role grants), so its migrations stay explicit and
// re-runnable. The guard also lets this apply cleanly to a database whose
// index was already removed by the purge path's defensive cleanup.
migrationBuilder.Sql(@"
IF EXISTS (SELECT 1 FROM sys.indexes
WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
DROP INDEX UX_AuditLog_EventId ON dbo.AuditLog;");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.Sql(@"
IF NOT EXISTS (SELECT 1 FROM sys.indexes
WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
CREATE UNIQUE NONCLUSTERED INDEX UX_AuditLog_EventId ON dbo.AuditLog (EventId) ON [PRIMARY];");
}
}
}
@@ -1806,10 +1806,6 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
.HasDatabaseName("IX_AuditLog_CorrelationId") .HasDatabaseName("IX_AuditLog_CorrelationId")
.HasFilter("[CorrelationId] IS NOT NULL"); .HasFilter("[CorrelationId] IS NOT NULL");
b.HasIndex("EventId")
.IsUnique()
.HasDatabaseName("UX_AuditLog_EventId");
b.HasIndex("ExecutionId") b.HasIndex("ExecutionId")
.HasDatabaseName("IX_AuditLog_Execution"); .HasDatabaseName("IX_AuditLog_Execution");
@@ -1,5 +1,8 @@
using System.Data;
using System.Text;
using Microsoft.Data.SqlClient; using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.Audit; using ZB.MOM.WW.Audit;
@@ -17,15 +20,32 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
/// </summary> /// </summary>
public class AuditLogRepository : IAuditLogRepository public class AuditLogRepository : IAuditLogRepository
{ {
// SQL Server error numbers for duplicate-key violations on // SQL Server error numbers for duplicate-key violations on the
// UX_AuditLog_EventId. 2601 is a unique-index violation; 2627 is a // partition-aligned clustered PK_AuditLog (EventId, OccurredAtUtc).
// primary-key/unique-constraint violation. The IF NOT EXISTS … INSERT // 2601 is a unique-index violation; 2627 is a primary-key/unique-constraint
// pattern has a check-then-act race window — two sessions can both pass // violation. The IF NOT EXISTS … INSERT pattern has a check-then-act race
// the EXISTS check and then both attempt the INSERT — and the loser // window — two sessions can both pass the EXISTS check and then both attempt
// surfaces as one of these errors. Idempotency demands we swallow them. // the INSERT — and the loser surfaces as one of these errors. Idempotency
// demands we swallow them.
private const int SqlErrorUniqueIndexViolation = 2601; private const int SqlErrorUniqueIndexViolation = 2601;
private const int SqlErrorPrimaryKeyViolation = 2627; private const int SqlErrorPrimaryKeyViolation = 2627;
// Rows per set-based ingest statement. Ten bound parameters per row against
// SQL Server's 2,100-parameter ceiling leaves ample headroom at 100 rows
// (1,000 parameters) while still collapsing a typical telemetry packet into
// a single round trip. Larger chunks buy little — the win is round-trip
// elimination, not statement size — and would push plan-cache churn up
// (one cached plan per distinct row count).
private const int IngestChunkRows = 100;
// Ordinal-stable column list shared by the single-row and set-based inserts.
// The five persisted computed columns (Kind/Status/SourceSiteId/ExecutionId/
// ParentExecutionId) plus the non-persisted IngestedAtUtc are derived
// server-side from DetailsJson and must NEVER appear here — writing a
// computed column is an error.
private const string CanonicalColumnList =
"EventId, OccurredAtUtc, Actor, Action, Outcome, Category, Target, SourceNode, CorrelationId, DetailsJson";
private readonly ScadaBridgeDbContext _context; private readonly ScadaBridgeDbContext _context;
private readonly ILogger<AuditLogRepository> _logger; private readonly ILogger<AuditLogRepository> _logger;
@@ -83,7 +103,7 @@ VALUES
{ {
// Two concurrent sessions both passed the IF NOT EXISTS check and // Two concurrent sessions both passed the IF NOT EXISTS check and
// both attempted the INSERT — the loser raises 2601/2627 against // both attempted the INSERT — the loser raises 2601/2627 against
// UX_AuditLog_EventId. First-write-wins idempotency is already the // the clustered PK. First-write-wins idempotency is already the
// documented contract for this method, so the race outcome is // documented contract for this method, so the race outcome is
// semantically a no-op. Swallow at Debug; other SqlExceptions // semantically a no-op. Swallow at Debug; other SqlExceptions
// bubble. // bubble.
@@ -95,6 +115,196 @@ VALUES
} }
} }
/// <inheritdoc />
public async Task<int> InsertManyIfNotExistsAsync(
IReadOnlyList<AuditEvent> events,
TimeSpan? commandTimeout = null,
CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(events);
if (events.Count == 0)
{
return 0;
}
// De-duplicate WITHIN the packet before the statement is built. The
// set-based INSERT … SELECT … WHERE NOT EXISTS only tests rows that are
// ALREADY committed, so two copies of one EventId inside a single VALUES
// constructor would both pass the anti-semi-join and collide on the
// clustered PK — losing the whole statement to a duplicate-key fault.
// First-write-wins matches the single-row contract exactly (a later copy
// of the same EventId is by definition the same immutable append-only
// row), so keeping the first occurrence is not merely convenient, it is
// the documented semantics.
var seen = new HashSet<Guid>(events.Count);
var distinct = new List<AuditEvent>(events.Count);
foreach (var evt in events)
{
if (evt is not null && seen.Add(evt.EventId))
{
distinct.Add(evt);
}
}
var inserted = 0;
for (var offset = 0; offset < distinct.Count; offset += IngestChunkRows)
{
var length = Math.Min(IngestChunkRows, distinct.Count - offset);
var chunk = distinct.GetRange(offset, length);
try
{
inserted += await InsertChunkAsync(chunk, commandTimeout, ct).ConfigureAwait(false);
}
catch (SqlException ex) when (
ex.Number == SqlErrorUniqueIndexViolation
|| ex.Number == SqlErrorPrimaryKeyViolation)
{
// A CONCURRENT writer committed one of this chunk's EventIds
// between the anti-semi-join and the insert (the same
// check-then-act window the single-row path documents), and the
// whole set-based statement rolled back with it. Fall back to
// the per-row path so the rows that are genuinely new still land
// — the batch is a throughput optimisation, never a correctness
// dependency. Every row is retried, including the one that
// collided, because InsertIfNotExistsAsync swallows its own
// duplicate-key fault as a no-op.
_logger.LogDebug(
ex,
"Set-based audit ingest chunk of {Count} row(s) hit a duplicate-key violation (error {SqlErrorNumber}); falling back to per-row inserts.",
chunk.Count,
ex.Number);
foreach (var evt in chunk)
{
await InsertIfNotExistsAsync(evt, ct).ConfigureAwait(false);
}
}
}
return inserted;
}
/// <summary>
/// Executes one set-based idempotent insert: a VALUES table constructor
/// anti-semi-joined against the committed rows, so a whole telemetry packet
/// costs ONE round trip instead of one <c>IF NOT EXISTS … INSERT</c> per row.
/// </summary>
/// <remarks>
/// Raw ADO.NET (rather than <c>ExecuteSqlInterpolated</c>) because the
/// statement's parameter count varies with the chunk size and every parameter
/// needs an explicit <see cref="SqlDbType"/>: the VALUES constructor's column
/// types are inferred from the first row's parameters, so leaving a null
/// <c>Target</c>/<c>SourceNode</c> untyped would give the derived column the
/// wrong type and defeat the seek on the anti-semi-join. The command enlists
/// in the DbContext's ambient transaction when one is open — the cached
/// telemetry dual-write runs the audit insert and the SiteCalls upsert inside
/// a single transaction and both must commit or roll back together.
/// </remarks>
private async Task<int> InsertChunkAsync(
IReadOnlyList<AuditEvent> chunk, TimeSpan? commandTimeout, CancellationToken ct)
{
var sql = new StringBuilder(256 + (chunk.Count * 64));
sql.Append("INSERT INTO dbo.AuditLog (").Append(CanonicalColumnList).Append(")\n");
sql.Append("SELECT v.EventId, v.OccurredAtUtc, v.Actor, v.Action, v.Outcome, v.Category, ");
sql.Append("v.Target, v.SourceNode, v.CorrelationId, v.DetailsJson\n");
sql.Append("FROM (VALUES\n");
for (var i = 0; i < chunk.Count; i++)
{
if (i > 0)
{
sql.Append(",\n");
}
sql.Append(" (@e").Append(i)
.Append(",@t").Append(i)
.Append(",@a").Append(i)
.Append(",@n").Append(i)
.Append(",@o").Append(i)
.Append(",@c").Append(i)
.Append(",@g").Append(i)
.Append(",@s").Append(i)
.Append(",@r").Append(i)
.Append(",@d").Append(i)
.Append(')');
}
sql.Append("\n) AS v (").Append(CanonicalColumnList).Append(")\n");
sql.Append("WHERE NOT EXISTS (SELECT 1 FROM dbo.AuditLog x WHERE x.EventId = v.EventId);");
var conn = _context.Database.GetDbConnection();
var openedHere = false;
if (conn.State != ConnectionState.Open)
{
await conn.OpenAsync(ct).ConfigureAwait(false);
openedHere = true;
}
try
{
await using var cmd = conn.CreateCommand();
cmd.CommandText = sql.ToString();
cmd.Transaction = _context.Database.CurrentTransaction?.GetDbTransaction();
if (commandTimeout is { } timeout)
{
cmd.CommandTimeout = (int)timeout.TotalSeconds;
}
for (var i = 0; i < chunk.Count; i++)
{
var evt = chunk[i];
// Same canonical projection as the single-row path: UTC-kind
// OccurredAtUtc, empty Actor collapses to NULL, Outcome/Category
// bound as their varchar storage form.
var occurred = DateTime.SpecifyKind(evt.OccurredAtUtc.UtcDateTime, DateTimeKind.Utc);
object actor = string.IsNullOrEmpty(evt.Actor) ? DBNull.Value : evt.Actor;
AddParameter(cmd, "@e" + i, SqlDbType.UniqueIdentifier, size: 0, evt.EventId);
AddParameter(cmd, "@t" + i, SqlDbType.DateTime2, size: 0, occurred);
AddParameter(cmd, "@a" + i, SqlDbType.NVarChar, size: 256, actor);
AddParameter(cmd, "@n" + i, SqlDbType.VarChar, size: 64, evt.Action);
AddParameter(cmd, "@o" + i, SqlDbType.VarChar, size: 16, evt.Outcome.ToString());
AddParameter(cmd, "@c" + i, SqlDbType.VarChar, size: 32, evt.Category);
AddParameter(cmd, "@g" + i, SqlDbType.NVarChar, size: 256, evt.Target);
AddParameter(cmd, "@s" + i, SqlDbType.VarChar, size: 64, evt.SourceNode);
AddParameter(cmd, "@r" + i, SqlDbType.UniqueIdentifier, size: 0, evt.CorrelationId);
AddParameter(cmd, "@d" + i, SqlDbType.NVarChar, size: -1, evt.DetailsJson);
}
return await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
}
finally
{
if (openedHere)
{
await conn.CloseAsync().ConfigureAwait(false);
}
}
}
/// <summary>
/// Binds one explicitly-typed parameter. A null CLR value binds as
/// <see cref="DBNull"/> while KEEPING its declared <see cref="SqlDbType"/>,
/// which is what makes the VALUES constructor's derived column types stable
/// regardless of which rows happen to carry nulls.
/// </summary>
private static void AddParameter(
System.Data.Common.DbCommand cmd, string name, SqlDbType type, int size, object? value)
{
var p = (SqlParameter)cmd.CreateParameter();
p.ParameterName = name;
p.SqlDbType = type;
if (size != 0)
{
p.Size = size;
}
p.Value = value ?? DBNull.Value;
cmd.Parameters.Add(p);
}
/// <inheritdoc /> /// <inheritdoc />
public async Task<IReadOnlyList<AuditEvent>> QueryAsync( public async Task<IReadOnlyList<AuditEvent>> QueryAsync(
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default) AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default)
@@ -229,13 +439,23 @@ VALUES
/// <inheritdoc /> /// <inheritdoc />
public async Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default) public async Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default)
{ {
// The drop-and-rebuild batch below runs via // The switch batch below runs via
// ExecuteSqlRaw with NO EF user-transaction — it carries its own server-side // ExecuteSqlRaw with NO EF user-transaction — it carries its own server-side
// BEGIN TRANSACTION / TRY-CATCH / ROLLBACK — so the DbContext's retrying // BEGIN TRANSACTION / TRY-CATCH / ROLLBACK — so the DbContext's retrying
// execution strategy (EnableRetryOnFailure) MAY auto-replay the whole batch on // execution strategy (EnableRetryOnFailure) MAY auto-replay the whole batch on
// a transient fault. That replay is safe: every step is IF-EXISTS / IF-NOT-EXISTS // a transient fault. That replay is safe: every step is IF-EXISTS / IF-NOT-EXISTS
// guarded and the staging table is GUID-suffixed, so a re-run is idempotent. // guarded and the staging table is GUID-suffixed, so a re-run is idempotent.
// //
// ALIGNED UNIQUENESS (WP2.2): there is no longer an index drop/rebuild around
// the SWITCH. EventId uniqueness is now enforced solely by the clustered
// PK_AuditLog (EventId, OccurredAtUtc), which is partition-aligned on
// ps_AuditLog_Month(OccurredAtUtc) — so ALTER TABLE … SWITCH PARTITION has no
// non-aligned index to object to. The former dance dropped UX_AuditLog_EventId,
// switched, then rebuilt it OFFLINE inside the same transaction: a whole-table
// unique-index build blocking every writer for the duration of the purge, and a
// window in which the idempotency-supporting index did not exist at all. Both
// are gone. See migration AlignAuditLogEventIdUniqueness for the reasoning.
//
// Maintenance timeout in whole seconds (ADO.NET CommandTimeout unit). Null leaves the // Maintenance timeout in whole seconds (ADO.NET CommandTimeout unit). Null leaves the
// provider default in place. See AuditLogPurgeOptions.MaintenanceCommandTimeoutMinutes / // provider default in place. See AuditLogPurgeOptions.MaintenanceCommandTimeoutMinutes /
// arch-review 04 S2 for why the ~30s default is unsafe for the switch-out dance. // arch-review 04 S2 for why the ~30s default is unsafe for the switch-out dance.
@@ -270,8 +490,13 @@ VALUES
BEGIN TRY BEGIN TRY
BEGIN TRANSACTION; BEGIN TRANSACTION;
-- 1. Drop the non-aligned unique index. ALTER TABLE SWITCH refuses -- 1. Defensive cleanup for databases created before
-- to run while it exists. -- AlignAuditLogEventIdUniqueness: the migration drops the
-- non-aligned UX_AuditLog_EventId, but a database restored from an
-- older backup could still carry it and SWITCH refuses to run while
-- a non-aligned unique index exists. Dropping it here is idempotent
-- and permanent the aligned clustered PK is the only uniqueness
-- enforcement the ingest path needs, so there is nothing to rebuild.
IF EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog')) IF EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
DROP INDEX UX_AuditLog_EventId ON dbo.AuditLog; DROP INDEX UX_AuditLog_EventId ON dbo.AuditLog;
@@ -313,31 +538,19 @@ VALUES
-- 4. Drop staging the rows are discarded here. This is the purge. -- 4. Drop staging the rows are discarded here. This is the purge.
DROP TABLE dbo.[{stagingTableName}]; DROP TABLE dbo.[{stagingTableName}];
-- 5. Rebuild the non-aligned unique index. Live traffic that hit the
-- table during steps 1-4 saw composite-PK uniqueness only; from
-- here on, single-column EventId uniqueness is restored.
CREATE UNIQUE NONCLUSTERED INDEX UX_AuditLog_EventId ON dbo.AuditLog (EventId) ON [PRIMARY];
COMMIT TRANSACTION; COMMIT TRANSACTION;
END TRY END TRY
BEGIN CATCH BEGIN CATCH
IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION; IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION;
-- Best-effort staging cleanup. The DROP INDEX in step 1 is now -- Best-effort staging cleanup. The staging table from step 2 may or
-- rolled back (so the index is back), but the staging table from -- may not survive the rollback depending on the failure point. Guard
-- step 2 may or may not survive the rollback depending on the -- the DROP so a missing staging table doesn't mask the original error.
-- failure point. Guard the DROP so a missing staging table doesn't -- Nothing else needs repairing: uniqueness lives on the clustered PK,
-- mask the original error. -- which the switch never touches, so a failed purge can no longer
-- leave the live table without its idempotency enforcement.
IF OBJECT_ID('dbo.[{stagingTableName}]', 'U') IS NOT NULL DROP TABLE dbo.[{stagingTableName}]; IF OBJECT_ID('dbo.[{stagingTableName}]', 'U') IS NOT NULL DROP TABLE dbo.[{stagingTableName}];
-- Idempotent index rebuild covers the niche case where ROLLBACK
-- failed to restore UX_AuditLog_EventId (or the failure happened
-- AFTER the COMMIT, which shouldn't be possible inside this TRY
-- but is cheap insurance). Without this, a failed switch could
-- leave the live table without its idempotency-supporting index.
IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
CREATE UNIQUE NONCLUSTERED INDEX UX_AuditLog_EventId ON dbo.AuditLog (EventId) ON [PRIMARY];
-- Surface the original error to the caller the purge actor logs -- Surface the original error to the caller the purge actor logs
-- and continues with the next boundary. -- and continues with the next boundary.
THROW; THROW;
@@ -128,6 +128,37 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
var groups = samples.GroupBy(s => new SeriesHourKey( var groups = samples.GroupBy(s => new SeriesHourKey(
s.Source, s.Metric, s.Scope, s.ScopeKey, TruncateToHour(s.CapturedAtUtc))); s.Source, s.Metric, s.Scope, s.ScopeKey, TruncateToHour(s.CapturedAtUtc)));
// Preload every rollup row already covering this window in ONE query and
// index it by series+hour (WP2.2). The predecessor issued a
// FirstOrDefaultAsync existence probe PER (series, hour) group — an N+1
// that scaled with the metric catalogue times the lookback: a 3 h re-fold
// over ~40 series cost ~120 sequential round trips before a single row was
// written. The window is bounded by the caller's small trailing lookback
// and the rollup table holds exactly one row per series-hour, so the
// preload is a narrow range seek on IX_KpiRollupHourly_Series.
//
// Deliberately TRACKED (not a projection): the re-fold path mutates the
// existing entity in place and relies on the change tracker to emit the
// UPDATE. The dictionary's ScopeKey comparison is ordinal where the
// previous SQL predicate used the database collation; both sides are
// written from the same KpiSample.ScopeKey values, so they are
// byte-identical in practice and a residual mismatch degrades to the
// already-handled upsert-race path rather than a wrong aggregate.
var existingRollups = await _context.KpiRollupHourly
.Where(r => r.HourStartUtc >= from && r.HourStartUtc < to)
.ToListAsync(cancellationToken);
var existingByKey = new Dictionary<SeriesHourKey, KpiRollupHourly>(existingRollups.Count);
foreach (var row in existingRollups)
{
existingByKey[new SeriesHourKey(
row.Source,
row.Metric,
row.Scope,
row.ScopeKey,
DateTime.SpecifyKind(row.HourStartUtc, DateTimeKind.Utc))] = row;
}
foreach (var group in groups) foreach (var group in groups)
{ {
var key = group.Key; var key = group.Key;
@@ -143,18 +174,11 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
var maxValue = group.Max(s => s.Value); var maxValue = group.Max(s => s.Value);
var sampleCount = group.Count(); var sampleCount = group.Count();
// Idempotent upsert on the unique series+hour key. The ScopeKey == key.ScopeKey // Idempotent upsert on the unique series+hour key, resolved against the
// comparison matches null against the Global-scope rows (IS NULL) exactly as the // preloaded dictionary. A null ScopeKey keys the Global-scope rows exactly
// UNIQUE IX_KpiRollupHourly_Series index treats a null key as participating. // as the UNIQUE IX_KpiRollupHourly_Series index treats a null key as
var existing = await _context.KpiRollupHourly.FirstOrDefaultAsync( // participating.
r => r.Source == key.Source if (!existingByKey.TryGetValue(key, out var existing))
&& r.Metric == key.Metric
&& r.Scope == key.Scope
&& r.ScopeKey == key.ScopeKey
&& r.HourStartUtc == key.HourStartUtc,
cancellationToken);
if (existing is null)
{ {
_context.KpiRollupHourly.Add(new KpiRollupHourly _context.KpiRollupHourly.Add(new KpiRollupHourly
{ {
@@ -140,7 +140,14 @@ VALUES
public async Task<IReadOnlyList<Notification>> GetDueAsync( public async Task<IReadOnlyList<Notification>> GetDueAsync(
DateTimeOffset now, int batchSize, CancellationToken cancellationToken = default) DateTimeOffset now, int batchSize, CancellationToken cancellationToken = default)
{ {
// AsNoTracking (WP2.2): the dispatcher mutates each row's delivery state in
// memory and persists it through UpdateAsync, which is now a targeted
// server-side ExecuteUpdate and needs no change tracker. Tracking a batch
// of notifications — each carrying an nvarchar(max) Body and TypeData —
// paid for a full snapshot copy per row plus a DetectChanges scan of the
// whole batch on every save.
return await _context.Notifications return await _context.Notifications
.AsNoTracking()
.Where(n => n.Status == NotificationStatus.Pending .Where(n => n.Status == NotificationStatus.Pending
|| (n.Status == NotificationStatus.Retrying || (n.Status == NotificationStatus.Retrying
&& n.NextAttemptAt != null && n.NextAttemptAt != null
@@ -150,22 +157,59 @@ VALUES
.ToListAsync(cancellationToken); .ToListAsync(cancellationToken);
} }
/// <inheritdoc />
public async Task<Notification?> GetByIdAsync(string notificationId, CancellationToken cancellationToken = default)
=> await _context.Notifications.FindAsync(new object[] { notificationId }, cancellationToken);
/// <inheritdoc /> /// <inheritdoc />
public async Task UpdateAsync(Notification n, CancellationToken cancellationToken = default) public async Task UpdateAsync(Notification n, CancellationToken cancellationToken = default)
{ {
_context.Notifications.Update(n); ArgumentNullException.ThrowIfNull(n);
await _context.SaveChangesAsync(cancellationToken);
// Targeted server-side UPDATE of the seven mutable delivery-state columns
// (WP2.2). The predecessor called DbSet.Update(n) + SaveChanges, which
// marks EVERY property modified and rewrites all 21 columns — including
// the immutable nvarchar(max) Body/TypeData payloads — on every single
// delivery attempt. ExecuteUpdate also bypasses the change tracker
// entirely, so it composes with the untracked GetDueAsync read.
//
// Immutable-by-contract columns (NotificationId, Type, ListName, Subject,
// Body, TypeData, Source*, Origin*, SiteEnqueuedAt, CreatedAt) are
// deliberately absent — see the interface contract: nothing in the
// notification lifecycle ever changes them, and omitting them is what
// makes the write narrow.
var status = n.Status;
var retryCount = n.RetryCount;
var lastError = n.LastError;
var resolvedTargets = n.ResolvedTargets;
var lastAttemptAt = n.LastAttemptAt;
var nextAttemptAt = n.NextAttemptAt;
var deliveredAt = n.DeliveredAt;
var notificationId = n.NotificationId;
await _context.Notifications
.Where(row => row.NotificationId == notificationId)
.ExecuteUpdateAsync(
setters => setters
.SetProperty(row => row.Status, status)
.SetProperty(row => row.RetryCount, retryCount)
.SetProperty(row => row.LastError, lastError)
.SetProperty(row => row.ResolvedTargets, resolvedTargets)
.SetProperty(row => row.LastAttemptAt, lastAttemptAt)
.SetProperty(row => row.NextAttemptAt, nextAttemptAt)
.SetProperty(row => row.DeliveredAt, deliveredAt),
cancellationToken);
} }
/// <inheritdoc />
public async Task<Notification?> GetByIdAsync(string notificationId, CancellationToken cancellationToken = default)
=> await _context.Notifications.FindAsync(new object[] { notificationId }, cancellationToken);
/// <inheritdoc /> /// <inheritdoc />
public async Task<(IReadOnlyList<Notification> Rows, int TotalCount)> QueryAsync( public async Task<(IReadOnlyList<Notification> Rows, int TotalCount)> QueryAsync(
NotificationOutboxFilter filter, int pageNumber, int pageSize, CancellationToken cancellationToken = default) NotificationOutboxFilter filter, int pageNumber, int pageSize, CancellationToken cancellationToken = default)
{ {
var query = _context.Notifications.AsQueryable(); // AsNoTracking (WP2.2): this is the Central UI's read-only list page. The
// rows are projected to the wire and never saved, so tracking them cost a
// snapshot copy of every nvarchar(max) Body in the page for nothing.
var query = _context.Notifications.AsNoTracking().AsQueryable();
if (filter.Status is { } status) if (filter.Status is { } status)
{ {
@@ -218,8 +262,14 @@ VALUES
var totalCount = await query.CountAsync(cancellationToken); var totalCount = await query.CountAsync(cancellationToken);
// NotificationId breaks CreatedAt ties so the OFFSET window is deterministic —
// without it two rows sharing a CreatedAt could appear on both pages or on
// neither. (This page keeps OFFSET paging rather than the sibling repos'
// keyset cursor because its contract surfaces a page number and a total
// count, neither of which a keyset cursor can express.)
var rows = await query var rows = await query
.OrderByDescending(n => n.CreatedAt) .OrderByDescending(n => n.CreatedAt)
.ThenByDescending(n => n.NotificationId)
.Skip((pageNumber - 1) * pageSize) .Skip((pageNumber - 1) * pageSize)
.Take(pageSize) .Take(pageSize)
.ToListAsync(cancellationToken); .ToListAsync(cancellationToken);
@@ -265,7 +315,24 @@ VALUES
// One conditional-aggregation pass replaces four sequential COUNT round trips: // One conditional-aggregation pass replaces four sequential COUNT round trips:
// each metric is a COUNT(CASE WHEN <predicate> THEN 1 END) over the same scan // each metric is a COUNT(CASE WHEN <predicate> THEN 1 END) over the same scan
// (arch-review 04). GroupBy(_ => 1) yields a single group (no rows → no group). // (arch-review 04). GroupBy(_ => 1) yields a single group (no rows → no group).
//
// WP2.2 — the aggregation is now PREDICATE-RESTRICTED instead of scanning the
// whole table. Every KPI here is about the live queue (Pending/Retrying), the
// parked backlog, or the last delivery interval; the overwhelming bulk of the
// Notifications table is historical Delivered and Discarded rows that
// contribute to NONE of them. The pre-filter below is the union of the
// metric-contributing predicates, which lets the optimizer seek the status
// index for the live/parked legs and the filtered
// (DeliveredAt) WHERE Status='Delivered' index (WP1.4) for the interval leg,
// instead of paying a full scan whose cost grows with retained history.
// Mirrors the pre-filter ComputePerSiteKpisAsync/ComputePerNodeKpisAsync
// already use — the global snapshot was the odd one out.
var counts = await _context.Notifications var counts = await _context.Notifications
.Where(n => n.Status == NotificationStatus.Pending
|| n.Status == NotificationStatus.Retrying
|| n.Status == NotificationStatus.Parked
|| (n.Status == NotificationStatus.Delivered
&& n.DeliveredAt != null && n.DeliveredAt >= deliveredSince))
.GroupBy(_ => 1) .GroupBy(_ => 1)
.Select(g => new .Select(g => new
{ {
@@ -72,49 +72,36 @@ public class SiteCallAuditRepository : ISiteCallAuditRepository
var idText = siteCall.TrackedOperationId.Value.ToString("D"); var idText = siteCall.TrackedOperationId.Value.ToString("D");
var incomingRank = GetRankOrThrow(siteCall.Status); var incomingRank = GetRankOrThrow(siteCall.Status);
// Step 1: insert-if-not-exists. Like AuditLogRepository.InsertIfNotExistsAsync // ONE round trip, UPDATE-first (WP2.2). The predecessor issued an
// this is check-then-act so a duplicate-key violation may surface under // unconditional IF NOT EXISTS … INSERT and THEN a monotonic UPDATE — two
// concurrent inserts on the same id — caught + logged at Debug. // statements, two round trips, on every single packet, of which the insert
// half was wasted work for every packet after the first (the steady state:
// a cached call emits Submitted → Forwarded → Attempted → terminal, so
// three of four packets hit an existing row).
// //
// SourceNode-stamping: the column is included in the INSERT // The combined batch below runs UPDATE first and inserts only when the
// column list / VALUES so a fresh row carries the originating node // UPDATE matched nothing AND the row genuinely does not exist. The
// name (node-a/node-b for site rows). A null SourceNode (legacy hosts // NOT EXISTS re-check is load-bearing: @@ROWCOUNT = 0 is ALSO what a
// / unstamped reconciled rows) writes NULL straight through. // monotonic REJECTION looks like (a stale or regressive packet against an
try // existing row), and inserting there would resurrect a row the guard just
{ // refused. Both statements ship in one command text, so this is one
await _context.Database.ExecuteSqlInterpolatedAsync( // round trip, not two.
$@"IF NOT EXISTS (SELECT 1 FROM dbo.SiteCalls WHERE TrackedOperationId = {idText})
INSERT INTO dbo.SiteCalls
(TrackedOperationId, Channel, Target, SourceSite, SourceNode, Status, RetryCount,
LastError, HttpStatus, CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, IngestedAtUtc)
VALUES
({idText}, {siteCall.Channel}, {siteCall.Target}, {siteCall.SourceSite}, {siteCall.SourceNode}, {siteCall.Status}, {siteCall.RetryCount},
{siteCall.LastError}, {siteCall.HttpStatus}, {siteCall.CreatedAtUtc}, {siteCall.UpdatedAtUtc}, {siteCall.TerminalAtUtc}, {siteCall.IngestedAtUtc});",
ct);
}
catch (SqlException ex) when (
ex.Number == SqlErrorUniqueIndexViolation
|| ex.Number == SqlErrorPrimaryKeyViolation)
{
_logger.LogDebug(
ex,
"SiteCallAuditRepository.UpsertAsync swallowed duplicate-key violation (error {SqlErrorNumber}) for TrackedOperationId {TrackedOperationId}; falling through to monotonic update.",
ex.Number,
idText);
}
// Step 2: monotonic update with a same-rank freshness tiebreaker. The
// CASE expression maps the stored Status string to the same rank table
// the caller uses. We mutate when EITHER the incoming rank is strictly
// greater, OR the incoming rank equals the stored rank AND that rank is
// non-terminal (< TerminalRank) AND the incoming UpdatedAtUtc is strictly
// newer than the stored one — so a retrying call's Attempted-phase
// RetryCount/LastError/HttpStatus stay live instead of freezing at the
// first Attempted packet. Terminal ranks are excluded from the
// tiebreaker, so a later terminal NEVER overwrites an earlier one; equal
// stamps are inert (idempotent replay) and a lower rank is always a no-op.
// //
// SourceNode-stamping: SourceNode is updated via // Monotonic update semantics are unchanged: mutate when EITHER the
// incoming rank is strictly greater, OR the incoming rank equals the
// stored rank AND that rank is non-terminal (< TerminalRank) AND the
// incoming UpdatedAtUtc is strictly newer than the stored one — so a
// retrying call's Attempted-phase RetryCount/LastError/HttpStatus stay
// live instead of freezing at the first Attempted packet. Terminal ranks
// are excluded from the tiebreaker, so a later terminal NEVER overwrites
// an earlier one; equal stamps are inert (idempotent replay) and a lower
// rank is always a no-op.
//
// SourceNode-stamping: the column is included in the INSERT column list /
// VALUES so a fresh row carries the originating node name (node-a/node-b
// for site rows). A null SourceNode (legacy hosts / unstamped reconciled
// rows) writes NULL straight through. On the UPDATE leg SourceNode is
// written via
// COALESCE(@SourceNode, SourceNode). The operator returns @SourceNode // COALESCE(@SourceNode, SourceNode). The operator returns @SourceNode
// when it is non-null, otherwise the stored value — so the column // when it is non-null, otherwise the stored value — so the column
// behaves protectively: a later packet that carries a null // behaves protectively: a later packet that carries a null
@@ -128,8 +115,12 @@ VALUES
// lifecycle every packet should carry the same SourceNode value (one // lifecycle every packet should carry the same SourceNode value (one
// execution, one node) so the "overwrite" path is in practice // execution, one node) so the "overwrite" path is in practice
// idempotent. // idempotent.
try
{
await _context.Database.ExecuteSqlInterpolatedAsync( await _context.Database.ExecuteSqlInterpolatedAsync(
$@"UPDATE dbo.SiteCalls $@"DECLARE @updated int;
UPDATE dbo.SiteCalls
SET Status = {siteCall.Status}, SET Status = {siteCall.Status},
RetryCount = {siteCall.RetryCount}, RetryCount = {siteCall.RetryCount},
LastError = {siteCall.LastError}, LastError = {siteCall.LastError},
@@ -162,9 +153,41 @@ WHERE TrackedOperationId = {idText}
ELSE -1 ELSE -1
END) END)
AND {incomingRank} < {TerminalRank} AND {incomingRank} < {TerminalRank}
AND UpdatedAtUtc < {siteCall.UpdatedAtUtc} ) );", AND UpdatedAtUtc < {siteCall.UpdatedAtUtc} ) );
-- Captured IMMEDIATELY after the UPDATE: @@ROWCOUNT is reset by the next
-- statement, and reading it inline inside a compound IF condition alongside a
-- subquery is not safe (the subquery's own execution can clobber it).
SET @updated = @@ROWCOUNT;
IF @updated = 0 AND NOT EXISTS (SELECT 1 FROM dbo.SiteCalls WHERE TrackedOperationId = {idText})
INSERT INTO dbo.SiteCalls
(TrackedOperationId, Channel, Target, SourceSite, SourceNode, Status, RetryCount,
LastError, HttpStatus, CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, IngestedAtUtc)
VALUES
({idText}, {siteCall.Channel}, {siteCall.Target}, {siteCall.SourceSite}, {siteCall.SourceNode}, {siteCall.Status}, {siteCall.RetryCount},
{siteCall.LastError}, {siteCall.HttpStatus}, {siteCall.CreatedAtUtc}, {siteCall.UpdatedAtUtc}, {siteCall.TerminalAtUtc}, {siteCall.IngestedAtUtc});",
ct); ct);
} }
catch (SqlException ex) when (
ex.Number == SqlErrorUniqueIndexViolation
|| ex.Number == SqlErrorPrimaryKeyViolation)
{
// Two concurrent sessions both found the row absent and both raced to
// INSERT; the loser raises 2601/2627 against the TrackedOperationId
// primary key. The winner's row IS the first-write, and this packet's
// content is by construction the same lifecycle state, so the race
// outcome is semantically a no-op. Swallow at Debug — the same
// check-then-act contract the sibling AuditLog/Notification repos
// document. Note the loser's UPDATE leg already ran (against no row),
// so nothing is left half-applied.
_logger.LogDebug(
ex,
"SiteCallAuditRepository.UpsertAsync swallowed duplicate-key violation (error {SqlErrorNumber}) for TrackedOperationId {TrackedOperationId}; treating as no-op.",
ex.Number,
idText);
}
}
/// <inheritdoc /> /// <inheritdoc />
public async Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default) public async Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default)
@@ -26,6 +26,22 @@ public static class ServiceCollectionExtensions
// registers IDataProtectionProvider as a singleton; resolving it here does not recurse // registers IDataProtectionProvider as a singleton; resolving it here does not recurse
// because key-ring loading is lazy (first Protect/Unprotect), not triggered by // because key-ring loading is lazy (first Protect/Unprotect), not triggered by
// CreateProtector during model building. // CreateProtector during model building.
//
// POOLING IS DELIBERATELY NOT USED (WP2.2 — verified, not overlooked).
// AddDbContextPool requires a context with a SINGLE public constructor taking
// only DbContextOptions<TContext>; EF Core constructs pooled instances through
// its own activator and cannot supply anything else. ScadaBridgeDbContext has
// two public constructors and the runtime one takes IDataProtectionProvider,
// because the encrypting value converter for secret-bearing columns is built
// during OnModelCreating from that provider. Worse, the model itself DIFFERS
// between the two constructors (no provider ⇒ no encrypting converter), so a
// pooled activator would silently produce a context that reads secret columns
// as ciphertext. Making this poolable means moving the protector out of the
// constructor and into a DbContextOptions extension — a change to the
// secrets-at-rest path, which is not a performance refactor. The registration
// below (a scoped factory overriding AddDbContext's activator) is what makes
// the provider reach the context at all, and it also bypasses pooling by
// construction. Revisit only alongside a deliberate secrets-plumbing change.
services.AddDbContext<ScadaBridgeDbContext>((serviceProvider, options) => services.AddDbContext<ScadaBridgeDbContext>((serviceProvider, options) =>
{ {
options.UseSqlServer( options.UseSqlServer(
@@ -158,6 +158,26 @@ public class SiteCallAuditActor : ReceiveActor
/// </summary> /// </summary>
private readonly Dictionary<string, bool> _reconciliationPinned = new(); private readonly Dictionary<string, bool> _reconciliationPinned = new();
/// <summary>
/// Actor-system EventStream captured on the actor thread at handler-registration
/// time. The reconciliation pass runs off-mailbox and publishes the pinned-state
/// transition from there, so it must not reach through <c>Context</c>.
/// </summary>
private Akka.Event.EventStream _eventStream = null!;
/// <summary>
/// Single-flight guard for the off-mailbox reconciliation pass. Raised on the
/// actor thread when a tick launches a pass and lowered on the actor thread
/// when the piped <see cref="ReconciliationComplete"/> arrives, so the
/// per-site cursor/pinned dictionaries the pass mutates are only ever touched
/// by one task at a time and the mailbox supplies the memory barrier between
/// consecutive passes.
/// </summary>
private bool _reconciling;
/// <summary>Single-flight guard for the off-mailbox terminal-row purge pass.</summary>
private bool _purging;
private ICancelable? _reconciliationTimer; private ICancelable? _reconciliationTimer;
private ICancelable? _purgeTimer; private ICancelable? _purgeTimer;
@@ -338,8 +358,79 @@ public class SiteCallAuditActor : ReceiveActor
// the daily terminal-row purge. Handlers stay alive across faults via // the daily terminal-row purge. Handlers stay alive across faults via
// their own per-site / per-tick try/catch (mirroring the ingest path); // their own per-site / per-tick try/catch (mirroring the ingest path);
// the timers are only started when their collaborators are available. // the timers are only started when their collaborators are available.
ReceiveAsync<ReconciliationTick>(_ => OnReconciliationTickAsync()); //
ReceiveAsync<PurgeTick>(_ => OnPurgeTickAsync()); // OFF-MAILBOX (WP2.2). Both passes run as PipeTo-completed background
// tasks behind a single-flight guard rather than as ReceiveAsync bodies.
// A ReceiveAsync handler occupies the actor for its whole duration, and a
// reconciliation pass is unbounded work — every site, up to
// MaxReconciliationPagesPerTick network pulls each, one upsert per row.
// Post-outage catch-up therefore blocked telemetry ingest, UI queries and
// KPI Asks behind it until the drain finished, and those callers timed out
// rather than queued. NotificationOutboxActor's dispatch sweep is the
// in-repo reference for this shape.
Receive<ReconciliationTick>(_ => HandleReconciliationTick());
Receive<ReconciliationComplete>(_ => _reconciling = false);
Receive<PurgeTick>(_ => HandlePurgeTick());
Receive<PurgeComplete>(_ => _purging = false);
// Captured on the actor thread so the background passes never touch
// Context off-thread. EventStream itself is thread-safe.
_eventStream = Context.System.EventStream;
}
/// <summary>
/// Launches a reconciliation pass unless one is already in flight, dropping
/// the tick if so. Overlapping passes are not merely wasteful — they would
/// race on the per-site cursor and pinned-latch dictionaries, which the
/// single-flight guard keeps confined to one task at a time (the guard itself
/// is only ever mutated on the actor thread: raised here, lowered by the
/// piped completion message).
/// </summary>
private void HandleReconciliationTick()
{
if (_reconciling)
{
return;
}
_reconciling = true;
// OnReconciliationTickAsync swallows its own per-site errors, but the
// failure projection is kept as a belt-and-braces guard so even a faulted
// task still lowers the guard — otherwise reconciliation would wedge
// permanently after a single unexpected throw.
OnReconciliationTickAsync().PipeTo(
Self,
success: () => ReconciliationComplete.Instance,
failure: ex =>
{
_logger.LogError(ex, "SiteCallAudit reconciliation pass faulted unexpectedly.");
return ReconciliationComplete.Instance;
});
}
/// <summary>
/// Launches a purge pass unless one is already in flight. Same single-flight
/// discipline as <see cref="HandleReconciliationTick"/>: a purge that outlives
/// its interval (a large catch-up after an outage) must not stack.
/// </summary>
private void HandlePurgeTick()
{
if (_purging)
{
return;
}
_purging = true;
OnPurgeTickAsync().PipeTo(
Self,
success: () => PurgeComplete.Instance,
failure: ex =>
{
_logger.LogError(ex, "SiteCallAudit purge pass faulted unexpectedly.");
return PurgeComplete.Instance;
});
} }
/// <inheritdoc /> /// <inheritdoc />
@@ -743,7 +834,10 @@ public class SiteCallAuditActor : ReceiveActor
} }
_reconciliationPinned[siteId] = pinned; _reconciliationPinned[siteId] = pinned;
Context.System.EventStream.Publish(new SiteCallReconciliationPinnedChanged(siteId, pinned));
// _eventStream, not Context.System.EventStream: this runs on the
// off-mailbox reconciliation pass, where Context must not be touched.
_eventStream.Publish(new SiteCallReconciliationPinnedChanged(siteId, pinned));
} }
// ── Piece B: daily terminal-row purge scheduler ── // ── Piece B: daily terminal-row purge scheduler ──
@@ -1452,6 +1546,24 @@ public class SiteCallAuditActor : ReceiveActor
public static readonly PurgeTick Instance = new(); public static readonly PurgeTick Instance = new();
private PurgeTick() { } private PurgeTick() { }
} }
/// <summary>
/// Piped back to <c>Self</c> when an off-mailbox reconciliation pass ends
/// (successfully or not) so the single-flight guard is lowered on the actor
/// thread rather than from the background task.
/// </summary>
internal sealed class ReconciliationComplete
{
public static readonly ReconciliationComplete Instance = new();
private ReconciliationComplete() { }
}
/// <summary>Purge counterpart of <see cref="ReconciliationComplete"/>.</summary>
internal sealed class PurgeComplete
{
public static readonly PurgeComplete Instance = new();
private PurgeComplete() { }
}
} }
/// <summary> /// <summary>
@@ -121,6 +121,105 @@ public class AuditLogIngestActorTests : TestKit, IClassFixture<MsSqlMigrationFix
Assert.Equal(3, count); Assert.Equal(3, count);
} }
[SkippableFact]
public async Task Receive_DuplicateEventIds_WithinOnePacket_ProduceOneRow_AndAreAllAcked()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// WP2.2 writes each packet as ONE set-based statement, whose anti-semi-join
// only sees already-COMMITTED rows. Two copies of an EventId inside one
// packet would therefore both pass it and collide on the clustered PK,
// taking the whole packet down — the repository de-duplicates first.
// Every id is still acked: the site's contract is "this row is now
// present at central", which is true for both copies.
var siteId = NewSiteId();
var repeated = NewEvent(siteId);
var other = NewEvent(siteId);
var batch = new List<AuditEvent> { repeated, other, repeated, repeated };
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
var actor = CreateActor(repo);
actor.Tell(new IngestAuditEventsCommand(batch), TestActor);
var reply = ExpectMsg<IngestAuditEventsReply>(TimeSpan.FromSeconds(10));
Assert.Equal(4, reply.AcceptedEventIds.Count);
Assert.True(
new[] { repeated.EventId, other.EventId }.ToHashSet()
.SetEquals(reply.AcceptedEventIds.ToHashSet()));
await using var readContext = CreateContext();
var rows = await readContext.Set<AuditLogRow>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.Equal(2, rows.Count);
}
[SkippableFact]
public async Task Receive_SamePacketTwice_IsIdempotent_AcrossPackets()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// A site whose ack was lost re-delivers the identical packet on the next
// drain, and the reconciliation pull can re-deliver it a third time. Each
// replay must ack fully (so the site can finally flip its rows to
// Forwarded) while writing nothing new.
var siteId = NewSiteId();
var events = Enumerable.Range(0, 6).Select(_ => NewEvent(siteId)).ToList();
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
var actor = CreateActor(repo);
for (var attempt = 0; attempt < 3; attempt++)
{
actor.Tell(new IngestAuditEventsCommand(events), TestActor);
var reply = ExpectMsg<IngestAuditEventsReply>(TimeSpan.FromSeconds(10));
Assert.Equal(6, reply.AcceptedEventIds.Count);
Assert.True(
events.Select(e => e.EventId).ToHashSet()
.SetEquals(reply.AcceptedEventIds.ToHashSet()));
}
await using var readContext = CreateContext();
var rows = await readContext.Set<AuditLogRow>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.Equal(6, rows.Count);
}
[SkippableFact]
public async Task Receive_OverlappingPackets_InsertOnlyTheNewRows()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// Partially-overlapping packets are the normal reconciliation shape: the
// pull cursor re-serves a tail the push already delivered. The overlap
// must be a silent no-op and the new rows must land.
var siteId = NewSiteId();
var first = Enumerable.Range(0, 4).Select(_ => NewEvent(siteId)).ToList();
var second = first.Skip(2).Concat(
Enumerable.Range(0, 3).Select(_ => NewEvent(siteId))).ToList();
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
var actor = CreateActor(repo);
actor.Tell(new IngestAuditEventsCommand(first), TestActor);
ExpectMsg<IngestAuditEventsReply>(TimeSpan.FromSeconds(10));
actor.Tell(new IngestAuditEventsCommand(second), TestActor);
var reply = ExpectMsg<IngestAuditEventsReply>(TimeSpan.FromSeconds(10));
Assert.Equal(5, reply.AcceptedEventIds.Count);
await using var readContext = CreateContext();
var rows = await readContext.Set<AuditLogRow>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.Equal(7, rows.Count);
}
[SkippableFact] [SkippableFact]
public async Task Receive_Sets_IngestedAtUtc_Before_Insert() public async Task Receive_Sets_IngestedAtUtc_Before_Insert()
{ {
@@ -28,20 +28,19 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
/// <list type="number"> /// <list type="number">
/// <item>The oldest partition (Jan) is removed.</item> /// <item>The oldest partition (Jan) is removed.</item>
/// <item>Newer partitions (Feb + Mar) are untouched.</item> /// <item>Newer partitions (Feb + Mar) are untouched.</item>
/// <item>The <c>UX_AuditLog_EventId</c> unique index survives the /// <item>The switch leaves the aligned clustered <c>PK_AuditLog</c> intact and
/// drop-and-rebuild dance.</item> /// does NOT (re)create the non-aligned <c>UX_AuditLog_EventId</c>.</item>
/// <item><see cref="IAuditLogRepository.InsertIfNotExistsAsync"/> remains /// <item><see cref="IAuditLogRepository.InsertIfNotExistsAsync"/> remains
/// idempotent against the rebuilt index after the purge.</item> /// idempotent against the aligned key after the purge.</item>
/// </list> /// </list>
/// </summary> /// </summary>
/// <remarks> /// <remarks>
/// The brief calls out that direct INSERTs bypass the writer role's INSERT-only /// The brief calls out that direct INSERTs bypass the writer role's INSERT-only
/// grant; the fixture connects as <c>sa</c> (see /// grant; the fixture connects as <c>sa</c> (see
/// <see cref="MsSqlMigrationFixture"/>'s default admin connection string), so /// <see cref="MsSqlMigrationFixture"/>'s default admin connection string), so
/// the seed step does not need the writer role at all. The drop-and-rebuild /// the seed step does not need the writer role at all. The switch batch itself
/// dance itself runs under the same admin connection because the test owns /// runs under the same admin connection because the test owns the database —
/// the database — the role granularity is exercised in the repository tests, /// the role granularity is exercised in the repository tests, not here.
/// not here.
/// </remarks> /// </remarks>
public class PartitionPurgeTests : TestKit, IClassFixture<MsSqlMigrationFixture> public class PartitionPurgeTests : TestKit, IClassFixture<MsSqlMigrationFixture>
{ {
@@ -110,22 +109,39 @@ VALUES
} }
/// <summary> /// <summary>
/// Asserts that <c>UX_AuditLog_EventId</c> exists in /// Asserts the post-purge index state: the partition-ALIGNED clustered
/// <c>sys.indexes</c>. The drop-and-rebuild dance briefly removes the /// <c>PK_AuditLog</c> is intact and the non-aligned <c>UX_AuditLog_EventId</c>
/// index inside its transaction; this check is meant to fire AFTER the /// is absent (WP2.2 — <c>AlignAuditLogEventIdUniqueness</c>).
/// actor's purge tick has committed so the rebuilt index is observable.
/// </summary> /// </summary>
private static async Task AssertUxIndexExistsAsync(SqlConnection conn) /// <remarks>
/// The purge used to bracket its SWITCH with a DROP/CREATE of
/// <c>UX_AuditLog_EventId</c>, because a non-aligned unique index blocks
/// <c>ALTER TABLE … SWITCH PARTITION</c> — an offline whole-table index build
/// inside the switch transaction, plus a window with no idempotency index at
/// all. Uniqueness now rides the aligned clustered PK, so the switch is
/// metadata-only. Asserting the index's ABSENCE is what keeps that property:
/// anything that recreates it silently reinstates the rebuild.
/// </remarks>
private static async Task AssertAlignedUniquenessAsync(SqlConnection conn)
{ {
await using var cmd = conn.CreateCommand(); await using var cmd = conn.CreateCommand();
cmd.CommandText = @" cmd.CommandText = @"
SELECT COUNT(*) SELECT
FROM sys.indexes (SELECT COUNT(*) FROM sys.indexes
WHERE name = 'UX_AuditLog_EventId' WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog')) AS NonAligned,
AND object_id = OBJECT_ID('dbo.AuditLog');"; (SELECT COUNT(*) FROM sys.indexes
var raw = await cmd.ExecuteScalarAsync(); WHERE object_id = OBJECT_ID('dbo.AuditLog') AND is_primary_key = 1) AS ClusteredPk;";
var count = Convert.ToInt32(raw); await using var reader = await cmd.ExecuteReaderAsync();
Assert.True(count == 1, $"UX_AuditLog_EventId should be present post-purge; sys.indexes count was {count}."); Assert.True(await reader.ReadAsync());
var nonAligned = reader.GetInt32(0);
var clusteredPk = reader.GetInt32(1);
Assert.True(
nonAligned == 0,
$"UX_AuditLog_EventId must NOT exist post-purge (it blocks SWITCH PARTITION); sys.indexes count was {nonAligned}.");
Assert.True(
clusteredPk == 1,
$"The aligned clustered PK_AuditLog must survive the purge; sys.indexes primary-key count was {clusteredPk}.");
} }
private IActorRef CreateActor( private IActorRef CreateActor(
@@ -255,20 +271,19 @@ WHERE name = 'UX_AuditLog_EventId'
} }
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
// 2. EndToEnd_UxIndexRebuilt_AfterPurge // 2. EndToEnd_AlignedUniqueness_AfterPurge
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
[SkippableFact] [SkippableFact]
public async Task EndToEnd_UxIndexRebuilt_AfterPurge() public async Task EndToEnd_AlignedUniqueness_AfterPurge()
{ {
Skip.IfNot(_fixture.Available, _fixture.SkipReason); Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// Same shape as test 1 — purge the Jan-2026 partition and then assert the // Same shape as test 1 — purge the Jan-2026 partition and then assert the
// UX_AuditLog_EventId index is still present. RetentionDays is computed // index state. RetentionDays is computed dynamically so the threshold always
// dynamically so the threshold always lands near 2026-01-20 (see SeedOccurredAt()). // lands near 2026-01-20 (see SeedOccurredAt()). Since WP2.2 the switch touches
// The drop-and-rebuild dance briefly removes the index inside its transaction // no index at all: uniqueness rides the aligned clustered PK, so there is
// (the SWITCH PARTITION step requires the non-aligned unique index to be absent), // nothing to drop before the SWITCH and nothing to rebuild after it.
// but step 5 rebuilds it before committing.
var siteId = "purge-uxidx-" + Guid.NewGuid().ToString("N").Substring(0, 8); var siteId = "purge-uxidx-" + Guid.NewGuid().ToString("N").Substring(0, 8);
var oldEventId = Guid.NewGuid(); var oldEventId = Guid.NewGuid();
var (oldOccurred, _, _, retentionDays) = SeedOccurredAt(); var (oldOccurred, _, _, retentionDays) = SeedOccurredAt();
@@ -305,7 +320,7 @@ WHERE name = 'UX_AuditLog_EventId'
// Open a fresh connection (the actor's pool is owned by EF) and // Open a fresh connection (the actor's pool is owned by EF) and
// assert the index is present post-purge. // assert the index is present post-purge.
await using var check = _fixture.OpenConnection(); await using var check = _fixture.OpenConnection();
await AssertUxIndexExistsAsync(check); await AssertAlignedUniquenessAsync(check);
} }
// --------------------------------------------------------------------- // ---------------------------------------------------------------------
@@ -320,7 +335,7 @@ WHERE name = 'UX_AuditLog_EventId'
// Seed + purge the Jan-2026 row, THEN exercise InsertIfNotExistsAsync twice for // Seed + purge the Jan-2026 row, THEN exercise InsertIfNotExistsAsync twice for
// a fresh recent EventId. The second call must be a no-op (duplicate-key collision // a fresh recent EventId. The second call must be a no-op (duplicate-key collision
// swallowed by the repository, per M2 Bundle A's race-fix) — which means the // swallowed by the repository, per M2 Bundle A's race-fix) — which means the
// rebuilt UX_AuditLog_EventId unique index is functioning as intended. // aligned clustered PK is still enforcing uniqueness as intended.
// RetentionDays is computed dynamically so the threshold always lands near // RetentionDays is computed dynamically so the threshold always lands near
// 2026-01-20 (see SeedOccurredAt()). // 2026-01-20 (see SeedOccurredAt()).
var siteId = "purge-idem-" + Guid.NewGuid().ToString("N").Substring(0, 8); var siteId = "purge-idem-" + Guid.NewGuid().ToString("N").Substring(0, 8);
@@ -357,11 +372,11 @@ WHERE name = 'UX_AuditLog_EventId'
max: TimeSpan.FromSeconds(30)); max: TimeSpan.FromSeconds(30));
// Settle then exercise InsertIfNotExistsAsync twice for the same // Settle then exercise InsertIfNotExistsAsync twice for the same
// EventId. The repository's idempotency relies on // EventId. The repository's idempotency relies on the aligned clustered
// UX_AuditLog_EventId being present so the IF NOT EXISTS … INSERT // PK (EventId, OccurredAtUtc) being intact so the IF NOT EXISTS … INSERT
// race window resolves to a duplicate-key violation the repo // race window resolves to a duplicate-key violation the repo swallows.
// swallows. If the index were missing here, two rows would land // If the switch had disturbed that key, two rows would land and the second
// and the second InsertIfNotExistsAsync would silently double-insert. // InsertIfNotExistsAsync would silently double-insert.
await Task.Delay(TimeSpan.FromMilliseconds(500)); await Task.Delay(TimeSpan.FromMilliseconds(500));
var freshEventId = Guid.NewGuid(); var freshEventId = Guid.NewGuid();
@@ -482,7 +497,7 @@ WHERE name = 'UX_AuditLog_EventId'
/// <summary> /// <summary>
/// Task 4 (arch-review 04, S2): proves the explicit-<see cref="TimeSpan"/> maintenance-timeout /// Task 4 (arch-review 04, S2): proves the explicit-<see cref="TimeSpan"/> maintenance-timeout
/// overload of <see cref="IAuditLogRepository.SwitchOutPartitionAsync"/> runs the real /// overload of <see cref="IAuditLogRepository.SwitchOutPartitionAsync"/> runs the real
/// drop-and-rebuild dance to completion against SQL Server — the old row is purged, the kept row /// staging/switch batch to completion against SQL Server — the old row is purged, the kept row
/// survives, and the returned sampled row-count reflects the switched partition. Exercising the /// survives, and the returned sampled row-count reflects the switched partition. Exercising the
/// timeout path end-to-end guards against a regression that only sets the timeout on the sample /// timeout path end-to-end guards against a regression that only sets the timeout on the sample
/// command and forgets the DDL batch (or vice versa). /// command and forgets the DDL batch (or vice versa).
@@ -520,8 +535,8 @@ WHERE name = 'UX_AuditLog_EventId'
Assert.DoesNotContain(rows, r => r.EventId == oldEventId); Assert.DoesNotContain(rows, r => r.EventId == oldEventId);
Assert.Contains(rows, r => r.EventId == keptEventId); Assert.Contains(rows, r => r.EventId == keptEventId);
// The dance must leave the idempotency-supporting unique index rebuilt. // The switch must leave uniqueness enforcement exactly as it found it.
await using var check = _fixture.OpenConnection(); await using var check = _fixture.OpenConnection();
await AssertUxIndexExistsAsync(check); await AssertAlignedUniquenessAsync(check);
} }
} }
@@ -52,22 +52,31 @@ public class AuditLogEntityTypeConfigurationTests : IDisposable
} }
[Fact] [Fact]
public void Configure_DeclaresUniqueIndex_OnEventIdAlone_ForIdempotencyLookups() public void Configure_DeclaresNoNonAlignedEventIdIndex_UniquenessRidesTheAlignedClusteredKey()
{ {
// EventId remains globally unique (the idempotency key for // WP2.2 (AlignAuditLogEventIdUniqueness): the standalone non-aligned
// InsertIfNotExistsAsync) via a dedicated unique index independent of the // UX_AuditLog_EventId is gone. EventId uniqueness now rides the clustered
// composite PK. // PK (EventId, OccurredAtUtc), which is partition-aligned on
// ps_AuditLog_Month — so ALTER TABLE ... SWITCH PARTITION no longer needs
// an offline index drop/rebuild around every retention purge. EventId is a
// GUID minted once at the site alongside OccurredAtUtc, so pair-uniqueness
// is EventId-uniqueness in practice, and the idempotency probe
// (WHERE EventId = @id) still seeks the clustered key's leading column.
//
// Re-declaring a single-column unique index here would silently reinstate
// the SWITCH incompatibility, so this test pins its absence.
var entity = _context.Model.FindEntityType(typeof(AuditLogRow)); var entity = _context.Model.FindEntityType(typeof(AuditLogRow));
Assert.NotNull(entity); Assert.NotNull(entity);
var eventIdIndex = entity!.GetIndexes() Assert.DoesNotContain(
.SingleOrDefault(i => i.GetDatabaseName() == "UX_AuditLog_EventId"); entity!.GetIndexes(),
i => i.GetDatabaseName() == "UX_AuditLog_EventId");
Assert.NotNull(eventIdIndex); var pk = entity.FindPrimaryKey();
Assert.True(eventIdIndex!.IsUnique); Assert.NotNull(pk);
Assert.Equal(
var indexedProperty = Assert.Single(eventIdIndex.Properties); new[] { nameof(AuditLogRow.EventId), nameof(AuditLogRow.OccurredAtUtc) },
Assert.Equal(nameof(AuditLogRow.EventId), indexedProperty.Name); pk!.Properties.Select(p => p.Name).ToArray());
} }
[Fact] [Fact]
@@ -164,7 +173,8 @@ public class AuditLogEntityTypeConfigurationTests : IDisposable
"IX_AuditLog_ParentExecution", "IX_AuditLog_ParentExecution",
"IX_AuditLog_Site_Occurred", "IX_AuditLog_Site_Occurred",
"IX_AuditLog_Target_Occurred", "IX_AuditLog_Target_Occurred",
"UX_AuditLog_EventId", // UX_AuditLog_EventId is intentionally absent — dropped by
// AlignAuditLogEventIdUniqueness; the aligned clustered PK carries it.
}; };
Assert.Equal(expected, indexNames); Assert.Equal(expected, indexNames);
@@ -85,22 +85,27 @@ public class AddAuditLogTableMigrationTests : IClassFixture<MsSqlMigrationFixtur
} }
/// <summary> /// <summary>
/// Verifies all nine named non-clustered indexes exist on the final /// Verifies all eight named non-clustered indexes exist on the final
/// <c>dbo.AuditLog</c> table after all migrations have been applied. /// <c>dbo.AuditLog</c> table after all migrations have been applied, and that
/// the non-aligned <c>UX_AuditLog_EventId</c> does NOT.
/// The original five indexes were created by <c>AddAuditLogTable</c>; /// The original five indexes were created by <c>AddAuditLogTable</c>;
/// the <c>CollapseAuditLogToCanonical</c> (C5, Task 2.5) migration rebuilt /// the <c>CollapseAuditLogToCanonical</c> (C5, Task 2.5) migration rebuilt
/// the table and added <c>IX_AuditLog_Execution</c>, /// the table and added <c>IX_AuditLog_Execution</c>,
/// <c>IX_AuditLog_ParentExecution</c>, <c>IX_AuditLog_Node_Occurred</c>, /// <c>IX_AuditLog_ParentExecution</c> and <c>IX_AuditLog_Node_Occurred</c>.
/// and <c>UX_AuditLog_EventId</c> — nine in total. /// <c>UX_AuditLog_EventId</c> was created alongside them but dropped again by
/// <c>AlignAuditLogEventIdUniqueness</c> (WP2.2) — its non-alignment forced an
/// offline drop/rebuild around every partition-switch purge, and the clustered
/// <c>PK_AuditLog (EventId, OccurredAtUtc)</c> already enforces the same
/// uniqueness partition-aligned.
/// </summary> /// </summary>
[SkippableFact] [SkippableFact]
public async Task AppliesMigration_CreatesNineNamedIndexes() public async Task AppliesMigration_CreatesEightNamedIndexes_AndNoNonAlignedUniqueIndex()
{ {
Skip.IfNot(_fixture.Available, _fixture.SkipReason); Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// All nine named non-clustered indexes present on dbo.AuditLog after // All eight named non-clustered indexes present on dbo.AuditLog after
// the full migration history is applied (AddAuditLogTable through // the full migration history is applied (AddAuditLogTable through
// CollapseAuditLogToCanonical). // AlignAuditLogEventIdUniqueness).
var expected = new[] var expected = new[]
{ {
// Original five (AddAuditLogTable / AddAuditLogSourceNode): // Original five (AddAuditLogTable / AddAuditLogSourceNode):
@@ -113,7 +118,6 @@ public class AddAuditLogTableMigrationTests : IClassFixture<MsSqlMigrationFixtur
"IX_AuditLog_Execution", "IX_AuditLog_Execution",
"IX_AuditLog_ParentExecution", "IX_AuditLog_ParentExecution",
"IX_AuditLog_Node_Occurred", "IX_AuditLog_Node_Occurred",
"UX_AuditLog_EventId",
}; };
foreach (var indexName in expected) foreach (var indexName in expected)
@@ -124,6 +128,16 @@ public class AddAuditLogTableMigrationTests : IClassFixture<MsSqlMigrationFixtur
$"WHERE o.name = 'AuditLog' AND i.name = '{indexName}';"); $"WHERE o.name = 'AuditLog' AND i.name = '{indexName}';");
Assert.True(count == 1, $"Expected index '{indexName}' to exist on AuditLog; found {count}."); Assert.True(count == 1, $"Expected index '{indexName}' to exist on AuditLog; found {count}.");
} }
var nonAligned = await ScalarAsync<int>(
"SELECT COUNT(*) FROM sys.indexes i " +
"INNER JOIN sys.objects o ON i.object_id = o.object_id " +
"WHERE o.name = 'AuditLog' AND i.name = 'UX_AuditLog_EventId';");
Assert.True(
nonAligned == 0,
"UX_AuditLog_EventId must NOT exist after AlignAuditLogEventIdUniqueness — "
+ "a non-aligned index blocks ALTER TABLE ... SWITCH PARTITION and reinstates "
+ $"the offline drop/rebuild the purge path was freed from; found {nonAligned}.");
} }
[SkippableFact] [SkippableFact]
@@ -50,6 +50,16 @@ public class NotificationOutboxRepositoryKpiQueryShapeTests
$"ComputeKpisAsync issued {counter.Count} queries against Notifications; expected <= 2"); $"ComputeKpisAsync issued {counter.Count} queries against Notifications; expected <= 2");
// The oldest lookup must be bounded (LIMIT/TOP), never a full non-terminal SELECT. // The oldest lookup must be bounded (LIMIT/TOP), never a full non-terminal SELECT.
Assert.Contains(counter.Commands, sql => sql.Contains("LIMIT", StringComparison.OrdinalIgnoreCase)); Assert.Contains(counter.Commands, sql => sql.Contains("LIMIT", StringComparison.OrdinalIgnoreCase));
// WP2.2: the aggregation must be PREDICATE-RESTRICTED, not an unrestricted
// scan. Every KPI here concerns the live queue, the parked backlog or the
// last delivery interval; historical Delivered/Discarded rows — which are
// the overwhelming bulk of a retained table — contribute to none of them.
// Without the WHERE the query's cost grows with retention rather than with
// the working set. (An index-restricted aggregation was already the shape
// of the per-site/per-node snapshots; the global one was the odd one out.)
Assert.All(
counter.Commands.Where(sql => sql.Contains("COUNT", StringComparison.OrdinalIgnoreCase)),
sql => Assert.Contains("WHERE", sql, StringComparison.OrdinalIgnoreCase));
Assert.Equal(3, global.QueueDepth); Assert.Equal(3, global.QueueDepth);
Assert.NotNull(global.OldestPendingAge); Assert.NotNull(global.OldestPendingAge);
@@ -560,6 +560,164 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
Assert.Equal(1, count); Assert.Equal(1, count);
} }
// ------------------------------------------------------------------------
// WP2.2: set-based InsertManyIfNotExistsAsync
// ------------------------------------------------------------------------
[SkippableFact]
public async Task InsertManyIfNotExistsAsync_WritesEveryDistinctEvent()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
var baseTime = new DateTime(2026, 5, 21, 8, 0, 0, DateTimeKind.Utc);
var events = Enumerable.Range(0, 12)
.Select(i => NewEvent(siteId, occurredAtUtc: baseTime.AddSeconds(i)))
.ToList();
var inserted = await repo.InsertManyIfNotExistsAsync(events);
Assert.Equal(12, inserted);
await using var readContext = CreateContext();
var rows = await readContext.Set<AuditLogRow>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.Equal(12, rows.Count);
Assert.Equal(
events.Select(e => e.EventId).OrderBy(g => g).ToArray(),
rows.Select(r => r.EventId).OrderBy(g => g).ToArray());
}
[SkippableFact]
public async Task InsertManyIfNotExistsAsync_DuplicateEventIdsWithinOnePacket_CollapseToOneRow()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
// A set-based INSERT … SELECT … WHERE NOT EXISTS only tests rows that are
// already COMMITTED, so two copies of one EventId inside a single VALUES
// constructor would both pass the anti-semi-join and collide on the
// clustered PK — taking the whole statement down with them. The repository
// de-duplicates the packet first (first-write-wins, matching the
// single-row contract), which this pins.
var occurred = new DateTime(2026, 5, 21, 9, 0, 0, DateTimeKind.Utc);
var first = NewEvent(siteId, occurredAtUtc: occurred);
var duplicate = ScadaBridgeAuditEventFactory.Create(
channel: AuditChannel.ApiOutbound,
kind: AuditKind.ApiCall,
status: AuditStatus.Delivered,
eventId: first.EventId,
occurredAtUtc: occurred,
sourceSiteId: siteId,
errorMessage: "duplicate-within-packet-should-be-ignored");
var other = NewEvent(siteId, occurredAtUtc: occurred.AddSeconds(1));
var inserted = await repo.InsertManyIfNotExistsAsync(
new[] { first, duplicate, other, duplicate });
Assert.Equal(2, inserted);
await using var readContext = CreateContext();
var rows = await readContext.Set<AuditLogRow>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.Equal(2, rows.Count);
// First-write-wins: the surviving row is the FIRST occurrence, so the
// duplicate's ErrorMessage never lands.
var stored = Assert.Single(rows, r => r.EventId == first.EventId);
Assert.Null(AuditDetailsCodec.Deserialize(stored.DetailsJson).ErrorMessage);
}
[SkippableFact]
public async Task InsertManyIfNotExistsAsync_DuplicateEventIdsAcrossPackets_AreIdempotent()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
var baseTime = new DateTime(2026, 5, 21, 10, 0, 0, DateTimeKind.Utc);
var packet = Enumerable.Range(0, 5)
.Select(i => NewEvent(siteId, occurredAtUtc: baseTime.AddSeconds(i)))
.ToList();
// A site retry / reconciliation pull re-delivers a packet that overlaps
// an already-ingested one. The second call must insert only the genuinely
// new rows and silently skip the rest (first-write-wins across packets).
var firstInserted = await repo.InsertManyIfNotExistsAsync(packet);
var extra = NewEvent(siteId, occurredAtUtc: baseTime.AddSeconds(99));
var secondInserted = await repo.InsertManyIfNotExistsAsync(
packet.Concat(new[] { extra }).ToList());
// A third, wholly-redundant replay writes nothing at all.
var thirdInserted = await repo.InsertManyIfNotExistsAsync(packet);
Assert.Equal(5, firstInserted);
Assert.Equal(1, secondInserted);
Assert.Equal(0, thirdInserted);
await using var readContext = CreateContext();
var rows = await readContext.Set<AuditLogRow>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.Equal(6, rows.Count);
}
[SkippableFact]
public async Task InsertManyIfNotExistsAsync_ChunksBeyondTheParameterCeiling()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
// 250 rows × 10 bound parameters = 2,500 — past SQL Server's 2,100
// parameter ceiling for a single statement, so this only succeeds if the
// repository chunks. Nulls in the optional columns (Target, SourceNode,
// CorrelationId, Actor) are left null on purpose: the VALUES constructor
// derives its column types from the parameters, so an untyped null would
// surface here as a conversion failure rather than silently.
var baseTime = new DateTime(2026, 5, 21, 11, 0, 0, DateTimeKind.Utc);
var events = Enumerable.Range(0, 250)
.Select(i => NewEvent(siteId, occurredAtUtc: baseTime.AddSeconds(i)))
.ToList();
var inserted = await repo.InsertManyIfNotExistsAsync(events);
Assert.Equal(250, inserted);
await using var readContext = CreateContext();
var count = await readContext.Set<AuditLogRow>()
.Where(e => e.SourceSiteId == siteId)
.CountAsync();
Assert.Equal(250, count);
}
[SkippableFact]
public async Task InsertManyIfNotExistsAsync_EmptyBatch_IsANoOp()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
Assert.Equal(0, await repo.InsertManyIfNotExistsAsync(Array.Empty<AuditEvent>()));
}
[SkippableFact] [SkippableFact]
public async Task QueryAsync_Keyset_SameOccurredAtUtc_TiebreaksOnEventId() public async Task QueryAsync_Keyset_SameOccurredAtUtc_TiebreaksOnEventId()
{ {
@@ -622,16 +780,17 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
// //
// The partition-switch path replaces M1's NotSupportedException stub with // The partition-switch path replaces M1's NotSupportedException stub with
// the production drop-DROP-INDEX → CREATE-staging → SWITCH PARTITION → // the production CREATE-staging → SWITCH PARTITION → DROP-staging batch
// DROP-staging → CREATE-INDEX dance documented in alog.md §4. These tests // documented in alog.md §4. WP2.2 removed the index drop/rebuild that used to
// bracket it: uniqueness rides the partition-ALIGNED clustered PK
// (EventId, OccurredAtUtc), so SWITCH has nothing to object to. These tests
// verify the side effects an outsider can observe: // verify the side effects an outsider can observe:
// * rows in the targeted month are removed // * rows in the targeted month are removed
// * rows in OTHER months are NOT touched // * rows in OTHER months are NOT touched
// * UX_AuditLog_EventId still exists after a successful switch // * no non-aligned UX_AuditLog_EventId is (re)created by a switch
// * InsertIfNotExistsAsync's first-write-wins idempotency still holds // * InsertIfNotExistsAsync's first-write-wins idempotency still holds
// after a switch (the rebuilt index is real) // after a switch (the aligned key really does enforce it)
// * a thrown SqlException leaves UX_AuditLog_EventId rebuilt (the CATCH // * a thrown SqlException leaves the table intact with no orphaned staging
// branch's recovery path runs)
[SkippableFact] [SkippableFact]
public async Task SwitchOutPartitionAsync_OldPartition_RemovesRows_NewPartitionsKept() public async Task SwitchOutPartitionAsync_OldPartition_RemovesRows_NewPartitionsKept()
@@ -669,7 +828,7 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
} }
[SkippableFact] [SkippableFact]
public async Task SwitchOutPartitionAsync_RebuildsUxIndex_AfterSwitch() public async Task SwitchOutPartitionAsync_LeavesNoNonAlignedUniqueIndex_AfterSwitch()
{ {
Skip.IfNot(_fixture.Available, _fixture.SkipReason); Skip.IfNot(_fixture.Available, _fixture.SkipReason);
@@ -680,12 +839,23 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
// the fixture's MSSQL database) don't tread on each other. // the fixture's MSSQL database) don't tread on each other.
await repo.SwitchOutPartitionAsync(new DateTime(2026, 4, 1, 0, 0, 0, DateTimeKind.Utc)); await repo.SwitchOutPartitionAsync(new DateTime(2026, 4, 1, 0, 0, 0, DateTimeKind.Utc));
// WP2.2: the switch must NOT recreate UX_AuditLog_EventId. Re-adding it
// would reinstate the offline drop/rebuild the purge was freed from and
// block the next SWITCH until something dropped it again.
await using var verifyContext = CreateContext(); await using var verifyContext = CreateContext();
var indexExists = await ScalarAsync<int>( var indexExists = await ScalarAsync<int>(
verifyContext, verifyContext,
"SELECT COUNT(*) FROM sys.indexes " + "SELECT COUNT(*) FROM sys.indexes " +
"WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog');"); "WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog');");
Assert.Equal(1, indexExists); Assert.Equal(0, indexExists);
// The aligned clustered PK is what enforces uniqueness now, and the
// switch must leave it untouched.
var pkExists = await ScalarAsync<int>(
verifyContext,
"SELECT COUNT(*) FROM sys.indexes " +
"WHERE name = 'PK_AuditLog' AND object_id = OBJECT_ID('dbo.AuditLog') AND is_primary_key = 1;");
Assert.Equal(1, pkExists);
} }
[SkippableFact] [SkippableFact]
@@ -705,10 +875,11 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
// Switch out the June 2026 partition (different month, empty). // Switch out the June 2026 partition (different month, empty).
await repo.SwitchOutPartitionAsync(new DateTime(2026, 6, 1, 0, 0, 0, DateTimeKind.Utc)); await repo.SwitchOutPartitionAsync(new DateTime(2026, 6, 1, 0, 0, 0, DateTimeKind.Utc));
// Re-attempting the same EventId after the switch must STILL be a no-op // Re-attempting the same EventId after the switch must STILL be a no-op.
// (UX_AuditLog_EventId is the index that enables idempotency; if the // The idempotency probe now seeks the aligned clustered PK's leading
// rebuild left it broken, this insert would silently produce a duplicate // column instead of a dedicated unique index; if the switch had disturbed
// row and the count assertion below would catch it). // that key, this insert would silently produce a duplicate row and the
// count assertion below would catch it.
// C3 (Task 2.5): rebuild a sibling row with the same EventId via the factory // C3 (Task 2.5): rebuild a sibling row with the same EventId via the factory
// (ErrorMessage rides in DetailsJson, so a top-level `with` no longer applies). // (ErrorMessage rides in DetailsJson, so a top-level `with` no longer applies).
var dup = ScadaBridgeAuditEventFactory.Create( var dup = ScadaBridgeAuditEventFactory.Create(
@@ -734,7 +905,7 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
} }
[SkippableFact] [SkippableFact]
public async Task SwitchOutPartitionAsync_PartialFailure_RebuildsUxIndex_RaisesException() public async Task SwitchOutPartitionAsync_PartialFailure_RaisesException_LeavesNoOrphanedStaging()
{ {
Skip.IfNot(_fixture.Available, _fixture.SkipReason); Skip.IfNot(_fixture.Available, _fixture.SkipReason);
@@ -745,8 +916,8 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
// ALTER TABLE … SWITCH refuses to move rows out of a partition that's // ALTER TABLE … SWITCH refuses to move rows out of a partition that's
// referenced by an FK from another table, raising msg 4928 // referenced by an FK from another table, raising msg 4928
// ("ALTER TABLE SWITCH statement failed because target table … has a // ("ALTER TABLE SWITCH statement failed because target table … has a
// foreign key …"). The CATCH branch then rolls back and rebuilds the // foreign key …"). The CATCH branch then rolls back and drops the staging
// unique index — which the assertion below verifies. // table — which the assertion below verifies.
// //
// The probe table is uniquely named with a guid suffix so reruns of // The probe table is uniquely named with a guid suffix so reruns of
// this test inside the same fixture DB never collide. We clean it up // this test inside the same fixture DB never collide. We clean it up
@@ -786,14 +957,23 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
await cmd.ExecuteNonQueryAsync(); await cmd.ExecuteNonQueryAsync();
} }
// The CATCH block in the production SQL guarantees UX_AuditLog_EventId // The CATCH block drops the GUID-suffixed staging table regardless of
// is rebuilt regardless of which step failed inside the TRY. // which step failed inside the TRY, so a failed purge leaves no orphaned
// AuditLog_Staging_* object behind for the next tick to trip over.
await using var verifyContext = CreateContext(); await using var verifyContext = CreateContext();
var orphanedStaging = await ScalarAsync<int>(
verifyContext,
"SELECT COUNT(*) FROM sys.tables WHERE name LIKE 'AuditLog\\_Staging\\_%' ESCAPE '\\';");
Assert.Equal(0, orphanedStaging);
// And it must NOT have (re)created the non-aligned unique index: WP2.2
// deleted that rebuild entirely, so a failed switch cannot reintroduce
// the object that blocks the NEXT switch.
var indexExists = await ScalarAsync<int>( var indexExists = await ScalarAsync<int>(
verifyContext, verifyContext,
"SELECT COUNT(*) FROM sys.indexes " + "SELECT COUNT(*) FROM sys.indexes " +
"WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog');"); "WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog');");
Assert.Equal(1, indexExists); Assert.Equal(0, indexExists);
} }
// ------------------------------------------------------------------------ // ------------------------------------------------------------------------
@@ -325,6 +325,57 @@ public class KpiHistoryRepositoryTests
Assert.Equal(2, row.SampleCount); Assert.Equal(2, row.SampleCount);
} }
[Fact]
public async Task FoldHourlyRollupsAsync_PreloadsExistingRollups_WithoutAPerSeriesHourProbe()
{
// WP2.2: the fold used to issue one FirstOrDefaultAsync existence probe per
// (series, hour) group before writing anything — an N+1 that scaled with
// the metric catalogue times the lookback window. It now preloads the whole
// window in a single query and resolves each group from a dictionary.
//
// Six distinct series across two hours = twelve groups, so the old shape
// issued twelve SELECTs against KpiRollupHourly. The new shape issues one
// (plus the KpiSample read). This asserts the count stays small and
// constant rather than tracking the group count.
var counter = new RollupSelectCountingInterceptor();
await using var ctx = SqliteTestHelper.CreateInMemoryContext(counter);
var repo = new KpiHistoryRepository(ctx);
var samples = new List<KpiSample>();
for (var series = 0; series < 6; series++)
{
foreach (var hourOffset in new[] { 0, 1 })
{
samples.Add(Sample(
"NotificationOutbox",
"metric" + series,
"Global",
null,
value: series + hourOffset,
capturedAtUtc: Base.AddHours(hourOffset).AddMinutes(10)));
}
}
await repo.RecordSamplesAsync(samples);
// Seed the rollups so the SECOND fold takes the re-fold (update) path —
// the branch that most needed the per-group probe.
await repo.FoldHourlyRollupsAsync(Base, Base.AddHours(2));
counter.Reset();
await repo.FoldHourlyRollupsAsync(Base, Base.AddHours(2));
Assert.True(
counter.RollupSelectCount <= 1,
$"expected the fold to preload existing rollups in a single query; it issued {counter.RollupSelectCount} SELECTs against KpiRollupHourly");
// And the fold is still correct: twelve series-hours, values unchanged by
// the re-fold.
var rollups = await ctx.KpiRollupHourly.AsNoTracking().ToListAsync();
Assert.Equal(12, rollups.Count);
Assert.All(rollups, r => Assert.Equal(1, r.SampleCount));
}
[Fact] [Fact]
public async Task GetHourlySeriesAsync_ReturnsAscending_AndHonorsNullVsSiteScopeKey() public async Task GetHourlySeriesAsync_ReturnsAscending_AndHonorsNullVsSiteScopeKey()
{ {
@@ -540,6 +591,42 @@ public class KpiHistoryRepositoryTests
/// async non-query entry points (<c>ExecuteDeleteAsync</c> routes through the /// async non-query entry points (<c>ExecuteDeleteAsync</c> routes through the
/// async path). /// async path).
/// </summary> /// </summary>
/// <summary>
/// Counts reader commands that SELECT from <c>KpiRollupHourly</c>, so a test
/// can prove the hourly fold resolves existing rows from a single preload
/// rather than one probe per (series, hour) group.
/// </summary>
private sealed class RollupSelectCountingInterceptor : DbCommandInterceptor
{
public int RollupSelectCount { get; private set; }
public void Reset() => RollupSelectCount = 0;
private void CountIfRollupSelect(DbCommand command)
{
if (command.CommandText.Contains("KpiRollupHourly", StringComparison.OrdinalIgnoreCase)
&& command.CommandText.Contains("SELECT", StringComparison.OrdinalIgnoreCase))
{
RollupSelectCount++;
}
}
public override InterceptionResult<DbDataReader> ReaderExecuting(
DbCommand command, CommandEventData eventData, InterceptionResult<DbDataReader> result)
{
CountIfRollupSelect(command);
return base.ReaderExecuting(command, eventData, result);
}
public override ValueTask<InterceptionResult<DbDataReader>> ReaderExecutingAsync(
DbCommand command, CommandEventData eventData, InterceptionResult<DbDataReader> result,
CancellationToken cancellationToken = default)
{
CountIfRollupSelect(command);
return base.ReaderExecutingAsync(command, eventData, result, cancellationToken);
}
}
private sealed class DeleteCountingInterceptor : DbCommandInterceptor private sealed class DeleteCountingInterceptor : DbCommandInterceptor
{ {
public int DeleteCount { get; private set; } public int DeleteCount { get; private set; }
@@ -96,6 +96,47 @@ public class SiteCallAuditRepositoryTests : IClassFixture<MsSqlMigrationFixture>
Assert.Equal(attemptedSnapshot!.UpdatedAtUtc, afterStale.UpdatedAtUtc); Assert.Equal(attemptedSnapshot!.UpdatedAtUtc, afterStale.UpdatedAtUtc);
} }
[SkippableFact]
public async Task UpsertAsync_RejectedMonotonicUpdate_DoesNotFallThroughToInsert()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// WP2.2 collapsed the two-round-trip "insert-if-absent then monotonic
// update" into ONE batch that updates first and inserts only when nothing
// matched. That makes @@ROWCOUNT = 0 ambiguous: it means "no such row"
// AND "the monotonic guard rejected this packet". Guarding the insert on
// @@ROWCOUNT alone would therefore let every stale/regressive packet
// append a SECOND row for an id that already exists — silently forking
// the mirror. The re-check for the row's existence is what prevents that,
// and this pins it: after a rejected regressive upsert there is still
// exactly ONE row, still carrying the advanced state.
var id = TrackedOperationId.New();
await using var context = CreateContext();
var repo = new SiteCallAuditRepository(context);
await repo.UpsertAsync(NewRow(id, status: "Delivered", retryCount: 3, lastError: null));
// Every flavour of rejection: lower rank, equal terminal rank, and an
// equal non-terminal rank with a stale timestamp.
await repo.UpsertAsync(NewRow(id, status: "Submitted", retryCount: 0));
await repo.UpsertAsync(NewRow(id, status: "Parked", retryCount: 9, lastError: "should-not-apply"));
await repo.UpsertAsync(NewRow(
id,
status: "Delivered",
retryCount: 99,
updatedAtUtc: new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc)));
await using var readContext = CreateContext();
var loaded = await readContext.Set<SiteCall>()
.Where(s => s.TrackedOperationId == id)
.ToListAsync();
Assert.Single(loaded);
Assert.Equal("Delivered", loaded[0].Status);
Assert.Equal(3, loaded[0].RetryCount);
Assert.Null(loaded[0].LastError);
}
[SkippableFact] [SkippableFact]
public async Task UpsertAsync_SameStatus_EqualUpdatedAt_IsNoOp() public async Task UpsertAsync_SameStatus_EqualUpdatedAt_IsNoOp()
{ {
@@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central; using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit; using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
using ZB.MOM.WW.ScadaBridge.Commons.Types; using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
@@ -582,4 +583,122 @@ public class SiteCallAuditReconciliationTests : TestKit
Assert.Equal(siteId, evt.SiteId); Assert.Equal(siteId, evt.SiteId);
Assert.True(evt.Pinned, "a legacy site that ignores after_id must publish Pinned=true"); Assert.True(evt.Pinned, "a legacy site that ignores after_id must publish Pinned=true");
} }
// ---------------------------------------------------------------------
// 9. WP2.2: the reconciliation drain runs OFF the mailbox, so ingest,
// query and KPI messages are answered while a long post-outage
// catch-up is still pulling.
// ---------------------------------------------------------------------
/// <summary>
/// Pull client whose first call blocks until released, simulating a
/// post-outage catch-up that takes far longer than the caller's Ask timeout.
/// </summary>
private sealed class BlockingPullClient : IPullSiteCallsClient
{
private readonly TaskCompletionSource _release =
new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _entered =
new(TaskCreationOptions.RunContinuationsAsynchronously);
/// <summary>Completes once the drain has actually started pulling.</summary>
public Task Entered => _entered.Task;
public void Release() => _release.TrySetResult();
public async Task<PullSiteCallsResponse> PullAsync(
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
{
_entered.TrySetResult();
await _release.Task.ConfigureAwait(false);
return new PullSiteCallsResponse(Array.Empty<SiteCall>(), MoreAvailable: false);
}
}
[Fact]
public async Task ReconciliationDrain_InFlight_DoesNotBlockIngestUpsert()
{
// The drain used to run inside ReceiveAsync, which occupies the actor for
// its whole duration. A post-outage catch-up (every site, many paged
// network pulls, one upsert per row) therefore parked telemetry ingest,
// UI queries and KPI Asks behind it — and those callers timed out rather
// than queued, so a slow site could make central look dead. This pins the
// fix: with the drain off-mailbox behind a single-flight guard, an ingest
// Ask completes promptly while the pull is still blocked.
var siteId = "siteSlow";
var sites = new StaticEnumerator(new SiteEntry(siteId, "http://siteSlow:8083"));
var client = new BlockingPullClient();
var repo = new RecordingRepo();
var actor = CreateActor(sites, client, repo, FastTickOptions());
// Wait until the drain is genuinely in flight and blocked inside PullAsync.
await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
// The mailbox must still be serving. A generous-but-finite budget: this
// fails at the pre-fix behaviour (the reply only arrives once the pull
// unblocks, which never happens until Release below).
var id = TrackedOperationId.New();
var reply = await actor.Ask<UpsertSiteCallReply>(
new UpsertSiteCallCommand(NewRow(id, sourceSite: siteId)),
TimeSpan.FromSeconds(3));
Assert.True(reply.Accepted);
Assert.Equal(id, reply.TrackedOperationId);
// Let the drain finish so the actor shuts down cleanly.
client.Release();
}
[Fact]
public async Task ReconciliationTicks_DoNotOverlap_WhileADrainIsInFlight()
{
// Single-flight guard: with a 100 ms tick and a drain blocked for far
// longer, every subsequent tick must be dropped rather than starting a
// second concurrent pass — overlapping passes would race on the per-site
// cursor and pinned-latch dictionaries the drain mutates.
var siteId = "siteSlow";
var sites = new StaticEnumerator(new SiteEntry(siteId, "http://siteSlow:8083"));
var client = new CountingBlockingPullClient();
var repo = new RecordingRepo();
CreateActor(sites, client, repo, FastTickOptions());
await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
// Several tick intervals elapse while the first pass is still blocked.
await Task.Delay(TimeSpan.FromMilliseconds(600));
Assert.Equal(1, client.CallCount);
client.Release();
}
/// <summary>
/// <see cref="BlockingPullClient"/> that also counts invocations, so a test
/// can prove no second pass started while the first was blocked.
/// </summary>
private sealed class CountingBlockingPullClient : IPullSiteCallsClient
{
private readonly TaskCompletionSource _release =
new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _entered =
new(TaskCreationOptions.RunContinuationsAsynchronously);
private int _callCount;
public Task Entered => _entered.Task;
public int CallCount => Volatile.Read(ref _callCount);
public void Release() => _release.TrySetResult();
public async Task<PullSiteCallsResponse> PullAsync(
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
{
Interlocked.Increment(ref _callCount);
_entered.TrySetResult();
await _release.Task.ConfigureAwait(false);
return new PullSiteCallsResponse(Array.Empty<SiteCall>(), MoreAvailable: false);
}
}
} }