Merge branch 'worktree-agent-a3b474b485c0288de' into arch-review-remediation

This commit is contained in:
Joseph Doherty
2026-08-14 21:15:31 -04:00
29 changed files with 3790 additions and 266 deletions
+10 -6
View File
@@ -78,13 +78,15 @@ builder.Property(e => e.ExecutionId)
"CAST(JSON_VALUE(DetailsJson,'$.executionId') AS uniqueidentifier)", stored: true)
.ValueGeneratedOnAddOrUpdate();
// Composite PK includes OccurredAtUtc for partition alignment
// Composite PK includes OccurredAtUtc for partition alignment — and is the ONLY
// EventId uniqueness enforcement (WP2.2 / AlignAuditLogEventIdUniqueness). EventId
// is a GUID minted once at the site alongside OccurredAtUtc, so a given EventId
// always lands in one partition and pair-uniqueness is EventId-uniqueness.
builder.HasKey(e => new { e.EventId, e.OccurredAtUtc });
builder.HasIndex(e => e.EventId).IsUnique()
.HasDatabaseName("UX_AuditLog_EventId");
```
The predecessor `UX_AuditLog_EventId` — a single-column unique index on `[PRIMARY]`, deliberately NOT partition-aligned — was dropped: a non-aligned index blocks `ALTER TABLE … SWITCH PARTITION`, so the retention purge had to drop it, switch, and rebuild it offline inside the switch transaction on every run.
**`TemplateConfiguration`** (representative of the domain-area configs) sets up the self-referencing parent FK, folder FK, cascade-delete relationships to attributes/alarms/scripts/compositions/native alarm sources, and the filtered unique index that enforces name uniqueness only on non-derived (base) templates.
**`SiteCallEntityTypeConfiguration`** maps `SiteCall` to `dbo.SiteCalls` with a `TrackedOperationId` PK stored as `varchar(36)` (GUID in `"D"` format) so the column shape matches the wire format and the site SQLite store — one consistent format for operational debugging.
@@ -108,11 +110,13 @@ VALUES
`FormattableString` interpolation parameterises every value so there is no injection surface. SQL error numbers `2601` and `2627` (unique-index violation) are swallowed as no-ops because the IF NOT EXISTS check has a race window; both the check-loser and the retrying telemetry path are semantically correct duplicates.
`InsertManyIfNotExistsAsync` is the set-based form the ingest hot path uses: one `INSERT … SELECT … FROM (VALUES …) WHERE NOT EXISTS` per chunk of 100 rows (ten bound parameters per row against SQL Server's 2,100-parameter ceiling), so a telemetry packet costs one round trip instead of one per event. Because the anti-semi-join only sees committed rows, the packet is de-duplicated by `EventId` in C# first (first-write-wins, matching the single-row contract); a duplicate-key fault from a concurrent writer falls back to the per-row path so the batch is never a correctness dependency. It uses raw ADO.NET with explicitly typed `SqlParameter`s — the `VALUES` constructor derives its column types from the first row's parameters, so an untyped null would give the derived column the wrong type — and enlists in the DbContext's ambient transaction when one is open (the cached-telemetry dual-write).
`QueryAsync` builds LINQ predicates over `AuditLogRow` using `AsNoTracking()`, translating filter dimensions (`Channels`, `Kinds`, `Statuses`, `SourceSiteIds`, `SourceNodes`, `ExecutionId`, `ParentExecutionId`, time range) to server-side SQL IN/equality predicates and using keyset pagination on `(OccurredAtUtc DESC, EventId DESC)`.
`GetExecutionTreeAsync` walks the `ParentExecutionId` graph in two phases: a loop climbs to the root (bounded at 32 levels), then a recursive CTE descends the full tree and LEFT JOINs back to `AuditLog` so stub nodes (purged or row-less executions) still appear with `RowCount = 0`.
`SwitchOutPartitionAsync` executes a drop-and-rebuild dance — dropping `UX_AuditLog_EventId`, creating a byte-identical staging table (including the computed-column definitions), switching the target partition to staging, dropping staging, and rebuilding the unique index — all inside a single `BEGIN TRY / BEGIN CATCH` block that guarantees the index is present whether the switch succeeds or rolls back.
`SwitchOutPartitionAsync` creates a byte-identical staging table (including the computed-column definitions), switches the target partition to staging, and drops staging, inside a single `BEGIN TRY / BEGIN CATCH` block whose CATCH cleans up the staging table on any failure. No index is dropped or rebuilt: uniqueness rides the partition-aligned clustered PK, so the switch is metadata-only. A guarded defensive `DROP INDEX UX_AuditLog_EventId` remains at the head of the batch purely so a database restored from a pre-alignment backup still purges.
### IAuditService — config-change audit
@@ -241,7 +245,7 @@ The host is running in production mode and `GetPendingMigrationsAsync` found una
### AuditLog partition switch fails mid-operation
`SwitchOutPartitionAsync` wraps the drop-and-rebuild dance in `BEGIN TRY / BEGIN CATCH`. On failure the CATCH block drops the staging table if it exists and rebuilds `UX_AuditLog_EventId` if it was dropped before the failure. The original exception is re-thrown so the Audit Log purge actor logs it and retries on the next daily tick. Verify that the `scadabridge_audit_purger` role still holds `ALTER ON SCHEMA::dbo` if the operation fails with a permissions error.
`SwitchOutPartitionAsync` wraps the staging/switch batch in `BEGIN TRY / BEGIN CATCH`. On failure the CATCH block drops the staging table if it exists, so no orphaned `AuditLog_Staging_*` object is left behind. There is no index to repair — uniqueness lives on the clustered PK, which the switch never touches. The original exception is re-thrown so the Audit Log purge actor logs it and retries on the next daily tick. Verify that the `scadabridge_audit_purger` role still holds `ALTER ON SCHEMA::dbo` if the operation fails with a permissions error.
### Design-time `dotnet ef` tooling cannot find a connection string
@@ -0,0 +1,24 @@
BEGIN TRANSACTION;
IF NOT EXISTS (
SELECT * FROM [__EFMigrationsHistory]
WHERE [MigrationId] = N'20260815004957_AlignAuditLogEventIdUniqueness'
)
BEGIN
IF EXISTS (SELECT 1 FROM sys.indexes
WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
DROP INDEX UX_AuditLog_EventId ON dbo.AuditLog;
END;
IF NOT EXISTS (
SELECT * FROM [__EFMigrationsHistory]
WHERE [MigrationId] = N'20260815004957_AlignAuditLogEventIdUniqueness'
)
BEGIN
INSERT INTO [__EFMigrationsHistory] ([MigrationId], [ProductVersion])
VALUES (N'20260815004957_AlignAuditLogEventIdUniqueness', N'10.0.7');
END;
COMMIT;
GO
+32 -4
View File
@@ -291,6 +291,27 @@ stay `Pending` for the next sweep. Cadence is short (default 5 s) when
non-empty, longer (default 30 s) when idle; telemetry runs on a dedicated
dispatcher.
**Central-side ingest is set-based.** `AuditLogIngestActor` writes a whole
packet with ONE `InsertManyIfNotExistsAsync` statement rather than one
`IF NOT EXISTS … INSERT` round trip per event; the cached-telemetry dual-write
similarly runs the whole packet in ONE transaction (set-based audit insert plus
one single-statement `SiteCalls` upsert per entry) instead of a transaction per
entry. Idempotency is unchanged: duplicates that repeat *within* a packet
collapse first-write-wins before the statement is built, duplicates *across*
packets are eliminated by the anti-semi-join, and any failure falls back to the
per-row / per-entry path — so the documented invariant that one bad row cannot
sink the rest of the batch still holds, it is simply no longer paid for on the
healthy path.
**Timeout ladder.** The ingest budget is deliberately the smallest on the path:
the site's Ask and the central gRPC handler's Ask are both 30 s, the actor's
own database budget is 20 s and the per-statement SQL timeout is 15 s. Before
this the three were identical, so they expired at the same instant and the
caller learned nothing but "it took 30 s" — no partial ack, no way to tell a
slow database from a wedged singleton. With the inner budgets strictly smaller,
a slow batch is abandoned by the actor first and the accepted-so-far ids are
still replied while the outer Asks are still waiting.
### Reconciliation pull (self-healing for missed telemetry)
A central `SiteAuditReconciliationActor` periodically (default 5 min per site)
@@ -490,12 +511,19 @@ MS SQL for direct-write events). Unredacted secrets never persist.
silently failing retention job would otherwise be invisible until the table grew
unbounded. Per-boundary/per-channel error isolation is unchanged — a single failure
still never abandons the rest of the tick.
- **Maintenance command timeout:** the switch-out drop-and-rebuild dance and each
- **Maintenance command timeout:** the switch-out staging batch and each
per-channel `DELETE TOP` batch run with an explicit command timeout
(`AuditLog:Purge:MaintenanceCommandTimeoutMinutes`, default 30, floor 1 min) rather
than the ~30 s ADO.NET default, which could abort the metadata-only SWITCH mid-dance
on a large or contended partition and leave the live table without
`UX_AuditLog_EventId` until a later tick's CATCH branch rebuilt it.
than the ~30 s ADO.NET default, which could abort the metadata-only SWITCH mid-batch
on a large or contended partition and leave an orphaned staging table for the next
tick's CATCH branch to clean up.
- **Partition-aligned uniqueness:** the switch no longer drops and rebuilds an index.
`EventId` uniqueness rides the clustered `PK_AuditLog (EventId, OccurredAtUtc)`,
which is aligned on `ps_AuditLog_Month(OccurredAtUtc)`, so `SWITCH PARTITION` has no
non-aligned unique index to object to. The predecessor `UX_AuditLog_EventId` forced
an offline whole-table index build inside the switch transaction — blocking every
audit writer for its duration — and left a window in which the idempotency-supporting
index did not exist at all. See migration `AlignAuditLogEventIdUniqueness`.
- **Per-channel retention overrides (M5.5 T3):** `AuditLog:PerChannelRetentionDays`
is a dictionary keyed by canonical channel name (`ApiOutbound`, `DbOutbound`,
`Notification`, `ApiInbound`, `SecuredWrite` — all five `AuditChannel` values are
@@ -95,9 +95,11 @@ The configuration database stores all central system data, organized by domain a
- `ParentExecutionId``CAST(JSON_VALUE(DetailsJson,'$.parentExecutionId') AS uniqueidentifier)` PERSISTED — spawner's `ExecutionId`; null for top-level runs.
- `IngestedAtUtc``CAST(SWITCHOFFSET(CAST(JSON_VALUE(DetailsJson,'$.ingestedAtUtc') AS datetimeoffset), 0) AS datetime2(7))` — central ingest timestamp; **not** persisted (SQL Server rejects PERSISTED on the non-deterministic `SWITCHOFFSET` expression).
*Clustered primary key:* `(EventId, OccurredAtUtc)` — composite so the key is partition-aligned. `UX_AuditLog_EventId` (unique, non-aligned on `[PRIMARY]`) enforces global `EventId` uniqueness for `InsertIfNotExistsAsync` idempotency.
*Clustered primary key:* `(EventId, OccurredAtUtc)` — composite so the key is partition-aligned, and the **sole** `EventId` uniqueness enforcement backing `InsertIfNotExistsAsync` idempotency. `EventId` is a GUID minted once at the emitting site in the same operation that stamps `OccurredAtUtc`, and neither field is ever re-stamped downstream, so a given `EventId` always arrives with the same `OccurredAtUtc` and can only map to one partition — pair-uniqueness is `EventId`-uniqueness for every row the system produces. The idempotency probe (`WHERE EventId = @id`) still seeks the clustered key's leading column, at the cost of one seek per partition rather than one seek overall.
*Indexes* (all non-clustered, partition-aligned on `ps_AuditLog_Month(OccurredAtUtc)` except `UX_AuditLog_EventId`):
A non-aligned single-column `UX_AuditLog_EventId` on `[PRIMARY]` used to carry that uniqueness; it was dropped by the `AlignAuditLogEventIdUniqueness` migration because a non-aligned index blocks `ALTER TABLE … SWITCH PARTITION`, forcing the retention purge to drop it, switch, and rebuild it **offline inside the switch transaction** on every run.
*Indexes* (all non-clustered, partition-aligned on `ps_AuditLog_Month(OccurredAtUtc)`):
- `IX_AuditLog_OccurredAtUtc` (primary time-range index for global scans)
- `IX_AuditLog_Site_Occurred (SourceSiteId, OccurredAtUtc)` (per-site filters)
- `IX_AuditLog_CorrelationId (CorrelationId) WHERE CorrelationId IS NOT NULL` (drilldown from a single operation)
@@ -135,6 +135,28 @@ Pinned)` on the EventStream (transition-only, mirroring
health-observable condition rather than a silent log line, and the latch clears
with `Pinned=false` once a later tick makes progress.
**The drain runs off the mailbox.** A reconciliation pass is unbounded work —
every site, up to a page ceiling of network pulls each, one upsert per row — so
it runs as a background task with a `PipeTo`-delivered completion message and a
single-flight guard, not as an actor message handler. A handler occupies the
actor for its whole duration, so a post-outage catch-up used to park telemetry
ingest, UI queries and KPI Asks behind it; those callers time out rather than
queue, which made a slow site look like a dead central. The single-flight guard
is raised and lowered ON the actor thread, so the per-site cursor and pinned-latch
dictionaries the pass mutates are still only ever touched by one task at a time,
and the mailbox supplies the memory barrier between consecutive passes. The
daily terminal-row purge uses the same shape. (`NotificationOutboxActor`'s
dispatch sweep is the in-repo reference.)
**The central upsert is one statement.** `SiteCallAuditRepository.UpsertAsync`
issues a single batch that runs the monotonic UPDATE first and INSERTs only when
nothing matched *and* the row genuinely does not exist — instead of an
unconditional insert-if-absent followed by the update, which cost two round trips
on every packet and wasted the insert half for every packet after the first. The
existence re-check is load-bearing: a zero row count also means "the monotonic
guard rejected this packet", and inserting there would fork the mirror with a
second row for an id that already exists.
## Retry / Discard Relay
Parked cached calls live in the owning site's S&F buffer. Operator Retry/Discard
@@ -18,8 +18,9 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// ingest timestamp into DetailsJson (there is no promoted IngestedAtUtc
/// column — the value is a DetailsJson field set via
/// <see cref="AuditRowProjection.WithIngestedAtUtc"/>) and inserted idempotently
/// via <see cref="IAuditLogRepository.InsertIfNotExistsAsync"/> — duplicates are
/// silently swallowed (first-write-wins).
/// via <see cref="IAuditLogRepository.InsertManyIfNotExistsAsync"/> — duplicates
/// are silently swallowed (first-write-wins), whether they repeat inside one
/// packet or across packets.
/// </summary>
/// <remarks>
/// <para>
@@ -28,9 +29,11 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// consistent and the site is free to flip its local row to <c>Forwarded</c>.
/// </para>
/// <para>
/// Audit-write failures must NEVER abort the user-facing action. The actor
/// wraps each repository call in its own try/catch so a single bad row cannot
/// cause the rest of the batch to be lost, and it guards scope/repository
/// Audit-write failures must NEVER abort the user-facing action. Each message
/// is written as ONE set-based statement (or one transaction, for the cached
/// dual-write); if that fails the actor retries the same work row-by-row inside
/// per-row try/catch, so a single bad row still cannot cause the rest of the
/// batch to be lost. It also guards scope/repository
/// resolution so a transient DI fault cannot restart the singleton — those
/// catches are what keep this actor alive across handler throws, not the
/// supervisor strategy. The <see cref="SupervisorStrategy"/> override returns
@@ -50,6 +53,31 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// </remarks>
public class AuditLogIngestActor : ReceiveActor
{
/// <summary>
/// Overall budget for one ingest message's database work, deliberately
/// SHORTER than the gRPC Ask that wraps it.
/// </summary>
/// <remarks>
/// The path used to stack three identical 30 s budgets — the site's Ask
/// (<c>CommunicationOptions.NotificationForwardTimeout</c>), the central gRPC
/// handler's Ask (<c>SiteStreamGrpcServer.AuditIngestAskTimeout</c>) and the
/// ADO.NET command default — so they all expired at the same instant. The
/// caller therefore learned nothing except "it took 30 s": no partial ack, no
/// distinction between a slow database and a wedged singleton. Making the
/// innermost budget strictly smallest means a slow batch is abandoned by the
/// actor FIRST, with the accepted-so-far ids still replied, while the outer
/// Asks are still waiting.
/// </remarks>
internal static readonly TimeSpan IngestBudget = TimeSpan.FromSeconds(20);
/// <summary>
/// Per-statement SQL timeout for the ingest write, strictly inside
/// <see cref="IngestBudget"/> so a single wedged statement surfaces as a SQL
/// timeout the per-batch catch can log, rather than consuming the whole
/// actor-level budget and starving the rest of the batch.
/// </summary>
internal static readonly TimeSpan IngestSqlCommandTimeout = TimeSpan.FromSeconds(15);
private readonly IServiceProvider? _serviceProvider;
private readonly IAuditLogRepository? _injectedRepository;
private readonly ILogger<AuditLogIngestActor> _logger;
@@ -185,39 +213,75 @@ public class AuditLogIngestActor : ReceiveActor
DateTime nowUtc,
List<Guid> accepted)
{
// Stamp the ingest timestamp here, not at the site. Redact BEFORE the
// IngestedAtUtc stamp so the redacted copy carries the central-side
// ingest timestamp. The redactor is contract-bound to never throw; a null
// redactor (test composition root, no IAuditRedactor registered) falls
// back to the SafeDefault rather than pass-through, so HTTP header
// redaction always runs. IngestedAtUtc is a DetailsJson field on the
// canonical record, so stamp it via the projection helper.
var safeRedactor = redactor ?? SafeDefaultAuditRedactor.Instance;
var projected = new List<AuditEvent>(cmd.Events.Count);
foreach (var evt in cmd.Events)
{
var filtered = safeRedactor.Apply(evt);
projected.Add(AuditRowProjection.WithIngestedAtUtc(filtered, nowUtc));
}
// ONE set-based statement for the whole packet (WP2.2) instead of one
// IF NOT EXISTS … INSERT round trip per event. A telemetry packet of 200
// rows used to cost 200 sequential round trips on the central singleton's
// dispatcher; it now costs two. Idempotency is unchanged — duplicates
// within the packet collapse first-write-wins, duplicates against
// committed rows are eliminated by the anti-semi-join, and a concurrent
// writer degrades to the repository's per-row fallback.
using var budget = new CancellationTokenSource(IngestBudget);
try
{
// Stamp the ingest timestamp here, not at the site. The
// repository's duplicate-key hardening already swallows
// duplicate-key races, so the same id arriving twice (site
// retry, reconciliation) is a silent no-op.
// Redact BEFORE the IngestedAtUtc stamp so the redacted
// copy carries the central-side ingest timestamp. The redactor
// is contract-bound to never throw. A null
// redactor (test composition root, no IAuditRedactor
// registered) now falls back to the SafeDefault rather than
// pass-through, so HTTP header redaction always runs.
// IngestedAtUtc is a DetailsJson field on
// the canonical record, so stamp it via the projection helper.
var safeRedactor = redactor ?? SafeDefaultAuditRedactor.Instance;
var filtered = safeRedactor.Apply(evt);
var ingested = AuditRowProjection.WithIngestedAtUtc(filtered, nowUtc);
await repository.InsertIfNotExistsAsync(ingested).ConfigureAwait(false);
await repository
.InsertManyIfNotExistsAsync(projected, IngestSqlCommandTimeout, budget.Token)
.ConfigureAwait(false);
// A batch that returned without throwing means every row is now
// present — inserted here or already committed by an earlier delivery.
// Both count as accepted: the storage state is consistent and the site
// is free to flip its local rows to Forwarded.
foreach (var evt in cmd.Events)
{
accepted.Add(evt.EventId);
}
}
catch (Exception ex)
{
// The batch is a throughput optimisation, NOT a change to the
// failure grain. The documented invariant — "a single bad row cannot
// cause the rest of the batch to be lost" — is preserved by falling
// back to the per-row path here, so a poison row still costs only
// itself. Re-running rows the failed batch may already have committed
// is safe: every insert is idempotent on EventId.
_logger.LogWarning(ex,
"Set-based ingest of {Count} audit event(s) failed; falling back to per-row inserts so one bad row does not sink the batch.",
cmd.Events.Count);
for (var i = 0; i < projected.Count; i++)
{
try
{
await repository.InsertIfNotExistsAsync(projected[i], budget.Token).ConfigureAwait(false);
accepted.Add(cmd.Events[i].EventId);
}
catch (Exception rowEx)
{
// Per-row catch — one bad row never sinks the whole batch.
// The row stays Pending at the site; the next drain retries.
// Bump the central health counter so a
// sustained insert-throw failure surfaces on the dashboard.
// Bump the central health counter so a sustained insert-throw
// failure surfaces on the dashboard.
try { failureCounter?.Increment(); }
catch { /* counter must never throw — defence in depth */ }
_logger.LogError(ex,
_logger.LogError(rowEx,
"Failed to persist audit event {EventId} during batch ingest; row will be retried by the site.",
evt.EventId);
cmd.Events[i].EventId);
}
}
}
}
@@ -262,6 +326,24 @@ public class AuditLogIngestActor : ReceiveActor
var strategy = dbContext.Database.CreateExecutionStrategy();
// Fast path (WP2.2): the WHOLE packet in ONE transaction — one
// set-based audit insert plus one single-statement upsert per entry.
// The predecessor opened a transaction PER entry and issued three
// statements inside it (IF NOT EXISTS insert, then the two-statement
// SiteCalls upsert), so a 50-entry packet cost ~200 round trips and 50
// commits. If anything faults, the per-entry loop below re-runs the
// packet with its original per-entry isolation, so the documented
// "one entry's failure does not abort the others" invariant survives —
// it is simply no longer paid for on the healthy path.
using var budget = new CancellationTokenSource(IngestBudget);
if (await TryIngestCachedBatchAsync(
strategy, dbContext, auditRepo, siteCallRepo, redactor, cmd, accepted, budget.Token)
.ConfigureAwait(false))
{
replyTo.Tell(new IngestCachedTelemetryReply(accepted));
return;
}
foreach (var entry in cmd.Entries)
{
try
@@ -330,6 +412,90 @@ public class AuditLogIngestActor : ReceiveActor
replyTo.Tell(new IngestCachedTelemetryReply(accepted));
}
/// <summary>
/// Attempts the whole cached-telemetry packet as ONE transaction: a single
/// set-based audit insert followed by one monotonic <c>SiteCalls</c> upsert
/// per entry. Returns <see langword="true"/> when it committed (and only then
/// appends to <paramref name="accepted"/>), <see langword="false"/> when the
/// caller should fall back to the per-entry transaction loop.
/// </summary>
/// <remarks>
/// The batch is all-or-nothing by construction — it is a single transaction —
/// which is why a failure MUST fall back rather than report partial success:
/// the per-entry loop is what turns one poison entry into one lost entry
/// instead of a lost packet. Nothing is appended to
/// <paramref name="accepted"/> until the commit returns, so a failed attempt
/// leaves the caller's list untouched and the retry starts from a clean slate.
/// </remarks>
private async Task<bool> TryIngestCachedBatchAsync(
Microsoft.EntityFrameworkCore.Storage.IExecutionStrategy strategy,
ScadaBridgeDbContext dbContext,
IAuditLogRepository auditRepo,
ISiteCallAuditRepository siteCallRepo,
IAuditRedactor? redactor,
IngestCachedTelemetryCommand cmd,
List<Guid> accepted,
CancellationToken ct)
{
try
{
await strategy.ExecuteAsync(async () =>
{
await using var tx = await dbContext.Database
.BeginTransactionAsync(ct)
.ConfigureAwait(false);
// One central-side instant for the whole packet so a join across
// the two tables sees matching timestamps (debugging convenience,
// not a correctness invariant).
var ingestedAt = DateTime.UtcNow;
var safeRedactor = redactor ?? SafeDefaultAuditRedactor.Instance;
var auditRows = new List<AuditEvent>(cmd.Entries.Count);
foreach (var entry in cmd.Entries)
{
// Only the AuditLog row's payload columns are redactable;
// SiteCalls carries operational state only (status, retry
// count) and is left untouched.
var filteredAudit = safeRedactor.Apply(entry.Audit);
auditRows.Add(AuditRowProjection.WithIngestedAtUtc(filteredAudit, ingestedAt));
}
await auditRepo
.InsertManyIfNotExistsAsync(auditRows, IngestSqlCommandTimeout, ct)
.ConfigureAwait(false);
foreach (var entry in cmd.Entries)
{
await siteCallRepo
.UpsertAsync(entry.SiteCall with { IngestedAtUtc = ingestedAt }, ct)
.ConfigureAwait(false);
}
await tx.CommitAsync(ct).ConfigureAwait(false);
}).ConfigureAwait(false);
}
catch (Exception ex)
{
// No health-counter bump here — the per-entry retry is the authority
// on whether this packet genuinely failed, and double-counting a
// batch that the fallback then writes successfully would make the
// dashboard read as a sustained fault during normal contention.
_logger.LogWarning(
ex,
"Batched cached-telemetry dual-write of {Count} entr(ies) failed; falling back to per-entry transactions.",
cmd.Entries.Count);
return false;
}
foreach (var entry in cmd.Entries)
{
accepted.Add(entry.Audit.EventId);
}
return true;
}
/// <summary>
/// Fallback handler installed on the single-repository test ctor — that
/// ctor has no DbContext and no <see cref="ISiteCallAuditRepository"/>, so
@@ -17,8 +17,9 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// for monthly boundaries whose latest <c>OccurredAtUtc</c> is older
/// than <c>DateTime.UtcNow - RetentionDays</c>.</item>
/// <item>For each eligible boundary, calls
/// <see cref="IAuditLogRepository.SwitchOutPartitionAsync"/> which runs
/// the drop-and-rebuild dance around <c>UX_AuditLog_EventId</c>.</item>
/// <see cref="IAuditLogRepository.SwitchOutPartitionAsync"/>, a
/// metadata-only staging-table switch (WP2.2 removed the index
/// drop/rebuild that used to bracket it).</item>
/// <item>Publishes <see cref="AuditLogPurgedEvent"/> on the actor-system
/// EventStream so the central health collector + ops surfaces
/// can subscribe without coupling to this actor.</item>
@@ -26,11 +27,10 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// </summary>
/// <remarks>
/// <para>
/// <b>Daily cadence.</b> Partition switch is metadata-only but the
/// drop-and-rebuild dance briefly removes <c>UX_AuditLog_EventId</c>; running
/// more often than necessary trades unique-index rebuild outages for
/// negligible freshness wins. The default 24-hour interval matches
/// alog.md §10's retention policy.
/// <b>Daily cadence.</b> The partition switch is metadata-only, but it still
/// takes schema-modification locks on a table the ingest path is writing to;
/// running more often than necessary trades contention for negligible freshness
/// wins. The default 24-hour interval matches alog.md §10's retention policy.
/// </para>
/// <para>
/// <b>Continue-on-error.</b> A single boundary that throws (transient SQL
@@ -10,12 +10,12 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// </summary>
/// <remarks>
/// <para>
/// The purge actor is a daily-cadence singleton, not a hot-loop, because
/// partition-switch I/O is metadata-only but the drop-and-rebuild dance
/// briefly removes the <c>UX_AuditLog_EventId</c> unique index — running
/// more often than necessary trades index-rebuild outages for marginal
/// freshness gains. Lower this only when an operator can prove they need
/// sub-daily purge granularity.
/// The purge actor is a daily-cadence singleton, not a hot-loop. The
/// partition switch itself is metadata-only (since WP2.2's
/// <c>AlignAuditLogEventIdUniqueness</c> it no longer drops and rebuilds a
/// non-aligned unique index around it), but it still takes schema-modification
/// locks and competes with ingest for the same table. Lower this only when an
/// operator can prove they need sub-daily purge granularity.
/// </para>
/// <para>
/// <see cref="IntervalOverride"/> exists for tests to drop the cadence to
@@ -58,15 +58,14 @@ public sealed class AuditLogPurgeOptions
/// <summary>
/// Per-command timeout (in minutes) for the maintenance SQL the purge tick issues —
/// both the partition switch-out drop-and-rebuild dance
/// both the partition switch-out staging batch
/// (<see cref="ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories.IAuditLogRepository.SwitchOutPartitionAsync"/>)
/// and each per-channel <c>DELETE TOP</c> batch. Default 30 minutes.
/// </summary>
/// <remarks>
/// The ADO.NET default command timeout is ~30 seconds. On a large or contended partition the
/// SWITCH dance (which briefly drops <c>UX_AuditLog_EventId</c>) can exceed that and abort
/// mid-flight — leaving the live table without its idempotency-supporting unique index until a
/// later tick's CATCH branch rebuilds it (arch-review 04, S2). A generous maintenance timeout
/// SWITCH batch can exceed that and abort mid-flight, leaving an orphaned staging table for the
/// next tick's CATCH branch to clean up (arch-review 04, S2). A generous maintenance timeout
/// lets the metadata-only switch complete rather than self-locking. Resolved via
/// <see cref="ResolvedMaintenanceCommandTimeout"/>, clamped to a 1-minute floor.
/// </remarks>
@@ -34,6 +34,62 @@ public interface IAuditLogRepository
/// <returns>A task that represents the asynchronous operation.</returns>
Task InsertIfNotExistsAsync(AuditEvent evt, CancellationToken ct = default);
/// <summary>
/// Set-based form of <see cref="InsertIfNotExistsAsync"/>: inserts every
/// event in <paramref name="events"/> that does not already exist, in as few
/// round trips as the provider allows, and returns the number of rows
/// actually written. First-write-wins idempotency is unchanged — an EventId
/// already present is silently skipped.
/// </summary>
/// <remarks>
/// <para>
/// <b>Duplicate EventIds are tolerated both within and across packets.</b>
/// Duplicates inside one call collapse to the first occurrence before the
/// statement is built (an append-only row is immutable, so later copies of an
/// EventId carry the same content); duplicates against already-committed rows
/// are eliminated by the anti-semi-join. A concurrent writer that commits a
/// row mid-statement is handled by falling back to the per-row path, so the
/// batch is a throughput optimisation and never a correctness dependency.
/// </para>
/// <para>
/// <b>Default implementation.</b> The interface supplies a per-row loop so a
/// test double or an alternative store keeps working unchanged; the EF Core
/// implementation overrides it with a genuine set-based statement. Callers on
/// the ingest hot path should always prefer this method — the per-row form
/// cost one <c>IF NOT EXISTS … INSERT</c> round trip per audit event.
/// </para>
/// </remarks>
/// <param name="events">Audit events to insert; may contain duplicates.</param>
/// <param name="commandTimeout">
/// Optional per-statement timeout. The ingest actor passes a budget strictly
/// SHORTER than the gRPC Ask that wraps it, so the reply is produced before
/// the caller gives up rather than at the same instant.
/// </param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A task that resolves to the number of rows inserted (duplicates excluded).</returns>
async Task<int> InsertManyIfNotExistsAsync(
IReadOnlyList<AuditEvent> events,
TimeSpan? commandTimeout = null,
CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(events);
var inserted = 0;
var seen = new HashSet<Guid>(events.Count);
foreach (var evt in events)
{
if (evt is null || !seen.Add(evt.EventId))
{
continue;
}
await InsertIfNotExistsAsync(evt, ct).ConfigureAwait(false);
inserted++;
}
return inserted;
}
/// <summary>
/// Returns up to <see cref="AuditLogPaging.PageSize"/> rows matching
/// <paramref name="filter"/>, ordered by <c>(OccurredAtUtc DESC, EventId DESC)</c>.
@@ -60,36 +116,32 @@ public interface IAuditLogRepository
/// </summary>
/// <remarks>
/// <para>
/// <b>Drop-and-rebuild dance.</b> <c>UX_AuditLog_EventId</c> is intentionally
/// non-partition-aligned (it lives on <c>[PRIMARY]</c> so single-column
/// EventId uniqueness — required by <see cref="InsertIfNotExistsAsync"/> —
/// can be enforced cheaply). SQL Server rejects
/// <c>ALTER TABLE … SWITCH PARTITION</c> while a non-aligned unique index
/// is present, so the implementation drops the index, creates a staging
/// table with byte-identical schema, switches the partition's data into
/// staging, drops staging (discarding the rows), and rebuilds the unique
/// index. The CATCH branch guarantees the index is rebuilt even on partial
/// failure so the table never returns to live traffic without its
/// idempotency-supporting index.
/// <b>Partition-aligned uniqueness — no index drop.</b> EventId uniqueness is
/// enforced by the clustered <c>PK_AuditLog (EventId, OccurredAtUtc)</c>,
/// which is aligned on <c>ps_AuditLog_Month(OccurredAtUtc)</c>, so
/// <c>ALTER TABLE … SWITCH PARTITION</c> has no non-aligned unique index to
/// object to. The implementation creates a staging table with byte-identical
/// schema, switches the partition's data into staging, and drops staging
/// (discarding the rows). Nothing is dropped from the live table, so there is
/// no window in which the idempotency enforcement is absent and no offline
/// index rebuild inside the switch transaction.
/// </para>
/// <para>
/// <b>Outage window.</b> The dance briefly removes the unique index, so
/// concurrent <see cref="InsertIfNotExistsAsync"/> calls during the switch
/// could in principle race past the IF NOT EXISTS check without the index
/// catching the duplicate. This is acceptable for the daily purge cadence
/// — the inserts that the IF NOT EXISTS check guards are themselves rare
/// enough that a sub-second collision window is operationally negligible,
/// and the composite PK still rejects same-(EventId, OccurredAtUtc) rows.
/// The predecessor design carried a non-aligned
/// <c>UX_AuditLog_EventId</c> on <c>[PRIMARY]</c> that had to be dropped and
/// rebuilt around every switch. A defensive guarded <c>DROP INDEX</c> remains
/// in the batch so a database restored from a pre-alignment backup still
/// purges; it is a one-way cleanup, never rebuilt.
/// </para>
/// </remarks>
/// <param name="monthBoundary">Lower-bound datetime of the monthly partition to switch out.</param>
/// <param name="commandTimeout">
/// Optional per-command timeout for the maintenance SQL (the row-count sample plus the
/// drop-and-rebuild dance). When null the provider default applies. The purge actor passes
/// staging/switch batch). When null the provider default applies. The purge actor passes
/// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout"/>
/// (default 30 min) because the ~30s ADO.NET default can abort the switch mid-dance on a large
/// or contended partition, leaving the table without <c>UX_AuditLog_EventId</c> until the next
/// tick recovers (arch-review 04, S2).
/// (default 30 min) because the ~30s ADO.NET default can abort the switch mid-batch on a large
/// or contended partition, leaving an orphaned staging table for the next tick's CATCH to clean
/// up (arch-review 04, S2).
/// </param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A task that resolves to the approximate number of rows discarded by the partition switch.</returns>
@@ -46,10 +46,25 @@ public interface INotificationOutboxRepository
Task<Notification?> GetByIdAsync(string notificationId, CancellationToken cancellationToken = default);
/// <summary>
/// Marks <paramref name="n"/> modified and persists it (status transitions).
/// Commits internally — this call is its own transaction.
/// Persists <paramref name="n"/>'s <b>delivery-state</b> columns —
/// <c>Status</c>, <c>RetryCount</c>, <c>LastError</c>, <c>ResolvedTargets</c>,
/// <c>LastAttemptAt</c>, <c>NextAttemptAt</c>, <c>DeliveredAt</c>. Commits
/// internally — this call is its own transaction.
/// </summary>
/// <param name="n">The notification to update.</param>
/// <remarks>
/// <b>Scope is deliberately narrow.</b> Those seven columns are the ONLY
/// mutable state a notification has: everything else (identity, type, list,
/// subject, body, type data, source/origin attribution, enqueue and creation
/// timestamps) is written once at ingest and is immutable by contract. Every
/// caller — the dispatcher's per-attempt write and the operator retry/discard
/// one-shots — touches only this set. Implementations are therefore free to
/// issue a targeted UPDATE of these columns rather than rewriting the whole
/// row, which matters because the row carries <c>nvarchar(max)</c> body and
/// payload columns and the dispatcher writes on EVERY attempt. A future
/// caller that needs to change an immutable column must add its own method
/// rather than widening this one.
/// </remarks>
/// <param name="n">The notification whose delivery state should be persisted.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>A task that completes when the notification has been persisted.</returns>
Task UpdateAsync(Notification n, CancellationToken cancellationToken = default);
@@ -172,15 +172,18 @@ public class AuditLogEntityTypeConfiguration : IEntityTypeConfiguration<AuditLog
// ── Keys + indexes ───────────────────────────────────────────────────
// Composite PK includes OccurredAtUtc — required by the monthly partition scheme
// (ps_AuditLog_Month) so the clustered key is partition-aligned. EventId still
// needs to be globally unique for InsertIfNotExistsAsync idempotency, so a
// separate (non-aligned) unique index is declared on EventId alone.
// (ps_AuditLog_Month) so the clustered key is partition-aligned. It is ALSO the
// only uniqueness enforcement the ingest path needs: EventId is a GUID minted
// once at the emitting site and never re-stamped, so a given EventId always
// arrives with the same OccurredAtUtc and can only ever land in one partition.
// Uniqueness of the pair is therefore uniqueness of EventId in practice, and the
// idempotency probe (WHERE EventId = @id) still seeks the clustered key's leading
// column. The predecessor non-aligned UX_AuditLog_EventId on [PRIMARY] was
// dropped by AlignAuditLogEventIdUniqueness — it existed only to give
// single-column uniqueness and its non-alignment forced an offline drop/rebuild
// around every partition-switch purge.
builder.HasKey(e => new { e.EventId, e.OccurredAtUtc });
builder.HasIndex(e => e.EventId)
.IsUnique()
.HasDatabaseName("UX_AuditLog_EventId");
// Index names are locked for reconciliation/migration discoverability. The
// column SETS migrate to the canonical/computed shape (alog.md §4 semantics
// preserved): Channel→Category, Site/Node/Execution/ParentExecution now read
@@ -0,0 +1,93 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
{
/// <summary>
/// Makes <c>dbo.AuditLog</c>'s EventId uniqueness <b>partition-aligned</b> by
/// dropping the non-aligned <c>UX_AuditLog_EventId</c> and leaving the clustered
/// <c>PK_AuditLog (EventId, OccurredAtUtc)</c> — already aligned on
/// <c>ps_AuditLog_Month(OccurredAtUtc)</c> — as the sole enforcement.
/// </summary>
/// <remarks>
/// <para>
/// <b>Why.</b> <c>ALTER TABLE … SWITCH PARTITION</c> refuses to run while a
/// non-aligned index exists on the table, so the monthly retention purge
/// (<c>AuditLogRepository.SwitchOutPartitionAsync</c>) had to DROP
/// <c>UX_AuditLog_EventId</c>, switch, and then CREATE it again — an OFFLINE
/// whole-table unique-index build, inside the switch transaction, blocking every
/// audit writer for its duration. It also opened a window in which the index that
/// backs ingest idempotency did not exist at all, and a mid-dance failure could
/// leave the live table without it until a later tick's CATCH branch repaired it.
/// With alignment there is nothing to drop, so the switch is metadata-only and the
/// purge stops competing with ingest.
/// </para>
/// <para>
/// <b>Why dropping it is safe — EventId is globally unique by construction.</b>
/// The composite key enforces uniqueness of the PAIR, not of EventId alone, so in
/// principle the same EventId could now be stored twice under two different
/// <c>OccurredAtUtc</c> values (in two different partitions). That cannot happen
/// here: <c>EventId</c> is a GUID minted ONCE at the emitting site, in the same
/// operation that stamps <c>OccurredAtUtc</c>, and both travel together verbatim
/// through telemetry and reconciliation — nothing downstream re-stamps either
/// field. A given EventId therefore always arrives with the same OccurredAtUtc and
/// can only ever map to one partition, which makes pair-uniqueness equivalent to
/// EventId-uniqueness for every row this system produces. GUID collision across
/// partitions is not a real risk.
/// </para>
/// <para>
/// <b>The idempotency probe still seeks.</b> Both ingest forms test
/// <c>WHERE EventId = @id</c>, which is the LEADING column of the clustered PK, so
/// the probe remains an index seek. The cost changes shape rather than order: it
/// becomes one seek per partition (the partition column is not in the predicate, so
/// SQL Server cannot eliminate partitions) instead of a single seek on a
/// non-partitioned index. Against a monthly scheme that is a couple of dozen
/// shallow B-tree seeks — cheap, and paid on a path that now issues one statement
/// per telemetry packet rather than one per row.
/// </para>
/// <para>
/// <b>Edition note.</b> The alternative remedy — keeping the non-aligned index and
/// rebuilding it with <c>ONLINE = ON</c> outside the switch transaction — requires
/// Enterprise (or Azure SQL / Developer) edition; online index rebuild is not
/// available on Standard, which this deployment does not guarantee. Alignment
/// needs no edition-specific feature and removes the rebuild entirely, so it is
/// preferred regardless of edition.
/// </para>
/// <para>
/// <b>Down is a faithful reverse</b> and recreates the index on <c>[PRIMARY]</c>
/// exactly as <c>CollapseAuditLogToCanonical</c> created it. Reverting also
/// reinstates the SWITCH incompatibility, so the purge's guarded defensive
/// <c>DROP INDEX</c> (retained in <c>SwitchOutPartitionAsync</c> for databases
/// restored from pre-alignment backups) would remove it again on the next purge.
/// The partition function/scheme (<c>pf_AuditLog_Month</c> /
/// <c>ps_AuditLog_Month</c>) and every aligned index are untouched by both
/// directions.
/// </para>
/// </remarks>
public partial class AlignAuditLogEventIdUniqueness : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
// Raw, existence-guarded SQL rather than the scaffolded DropIndex: the
// AuditLog table is raw-SQL managed (partition scheme, persisted computed
// columns, append-only role grants), so its migrations stay explicit and
// re-runnable. The guard also lets this apply cleanly to a database whose
// index was already removed by the purge path's defensive cleanup.
migrationBuilder.Sql(@"
IF EXISTS (SELECT 1 FROM sys.indexes
WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
DROP INDEX UX_AuditLog_EventId ON dbo.AuditLog;");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.Sql(@"
IF NOT EXISTS (SELECT 1 FROM sys.indexes
WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
CREATE UNIQUE NONCLUSTERED INDEX UX_AuditLog_EventId ON dbo.AuditLog (EventId) ON [PRIMARY];");
}
}
}
@@ -1806,10 +1806,6 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
.HasDatabaseName("IX_AuditLog_CorrelationId")
.HasFilter("[CorrelationId] IS NOT NULL");
b.HasIndex("EventId")
.IsUnique()
.HasDatabaseName("UX_AuditLog_EventId");
b.HasIndex("ExecutionId")
.HasDatabaseName("IX_AuditLog_Execution");
@@ -1,5 +1,8 @@
using System.Data;
using System.Text;
using Microsoft.Data.SqlClient;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Storage;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.Audit;
@@ -17,15 +20,32 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
/// </summary>
public class AuditLogRepository : IAuditLogRepository
{
// SQL Server error numbers for duplicate-key violations on
// UX_AuditLog_EventId. 2601 is a unique-index violation; 2627 is a
// primary-key/unique-constraint violation. The IF NOT EXISTS … INSERT
// pattern has a check-then-act race window — two sessions can both pass
// the EXISTS check and then both attempt the INSERT — and the loser
// surfaces as one of these errors. Idempotency demands we swallow them.
// SQL Server error numbers for duplicate-key violations on the
// partition-aligned clustered PK_AuditLog (EventId, OccurredAtUtc).
// 2601 is a unique-index violation; 2627 is a primary-key/unique-constraint
// violation. The IF NOT EXISTS … INSERT pattern has a check-then-act race
// window — two sessions can both pass the EXISTS check and then both attempt
// the INSERT — and the loser surfaces as one of these errors. Idempotency
// demands we swallow them.
private const int SqlErrorUniqueIndexViolation = 2601;
private const int SqlErrorPrimaryKeyViolation = 2627;
// Rows per set-based ingest statement. Ten bound parameters per row against
// SQL Server's 2,100-parameter ceiling leaves ample headroom at 100 rows
// (1,000 parameters) while still collapsing a typical telemetry packet into
// a single round trip. Larger chunks buy little — the win is round-trip
// elimination, not statement size — and would push plan-cache churn up
// (one cached plan per distinct row count).
private const int IngestChunkRows = 100;
// Ordinal-stable column list shared by the single-row and set-based inserts.
// The five persisted computed columns (Kind/Status/SourceSiteId/ExecutionId/
// ParentExecutionId) plus the non-persisted IngestedAtUtc are derived
// server-side from DetailsJson and must NEVER appear here — writing a
// computed column is an error.
private const string CanonicalColumnList =
"EventId, OccurredAtUtc, Actor, Action, Outcome, Category, Target, SourceNode, CorrelationId, DetailsJson";
private readonly ScadaBridgeDbContext _context;
private readonly ILogger<AuditLogRepository> _logger;
@@ -83,7 +103,7 @@ VALUES
{
// Two concurrent sessions both passed the IF NOT EXISTS check and
// both attempted the INSERT — the loser raises 2601/2627 against
// UX_AuditLog_EventId. First-write-wins idempotency is already the
// the clustered PK. First-write-wins idempotency is already the
// documented contract for this method, so the race outcome is
// semantically a no-op. Swallow at Debug; other SqlExceptions
// bubble.
@@ -95,6 +115,196 @@ VALUES
}
}
/// <inheritdoc />
public async Task<int> InsertManyIfNotExistsAsync(
IReadOnlyList<AuditEvent> events,
TimeSpan? commandTimeout = null,
CancellationToken ct = default)
{
ArgumentNullException.ThrowIfNull(events);
if (events.Count == 0)
{
return 0;
}
// De-duplicate WITHIN the packet before the statement is built. The
// set-based INSERT … SELECT … WHERE NOT EXISTS only tests rows that are
// ALREADY committed, so two copies of one EventId inside a single VALUES
// constructor would both pass the anti-semi-join and collide on the
// clustered PK — losing the whole statement to a duplicate-key fault.
// First-write-wins matches the single-row contract exactly (a later copy
// of the same EventId is by definition the same immutable append-only
// row), so keeping the first occurrence is not merely convenient, it is
// the documented semantics.
var seen = new HashSet<Guid>(events.Count);
var distinct = new List<AuditEvent>(events.Count);
foreach (var evt in events)
{
if (evt is not null && seen.Add(evt.EventId))
{
distinct.Add(evt);
}
}
var inserted = 0;
for (var offset = 0; offset < distinct.Count; offset += IngestChunkRows)
{
var length = Math.Min(IngestChunkRows, distinct.Count - offset);
var chunk = distinct.GetRange(offset, length);
try
{
inserted += await InsertChunkAsync(chunk, commandTimeout, ct).ConfigureAwait(false);
}
catch (SqlException ex) when (
ex.Number == SqlErrorUniqueIndexViolation
|| ex.Number == SqlErrorPrimaryKeyViolation)
{
// A CONCURRENT writer committed one of this chunk's EventIds
// between the anti-semi-join and the insert (the same
// check-then-act window the single-row path documents), and the
// whole set-based statement rolled back with it. Fall back to
// the per-row path so the rows that are genuinely new still land
// — the batch is a throughput optimisation, never a correctness
// dependency. Every row is retried, including the one that
// collided, because InsertIfNotExistsAsync swallows its own
// duplicate-key fault as a no-op.
_logger.LogDebug(
ex,
"Set-based audit ingest chunk of {Count} row(s) hit a duplicate-key violation (error {SqlErrorNumber}); falling back to per-row inserts.",
chunk.Count,
ex.Number);
foreach (var evt in chunk)
{
await InsertIfNotExistsAsync(evt, ct).ConfigureAwait(false);
}
}
}
return inserted;
}
/// <summary>
/// Executes one set-based idempotent insert: a VALUES table constructor
/// anti-semi-joined against the committed rows, so a whole telemetry packet
/// costs ONE round trip instead of one <c>IF NOT EXISTS … INSERT</c> per row.
/// </summary>
/// <remarks>
/// Raw ADO.NET (rather than <c>ExecuteSqlInterpolated</c>) because the
/// statement's parameter count varies with the chunk size and every parameter
/// needs an explicit <see cref="SqlDbType"/>: the VALUES constructor's column
/// types are inferred from the first row's parameters, so leaving a null
/// <c>Target</c>/<c>SourceNode</c> untyped would give the derived column the
/// wrong type and defeat the seek on the anti-semi-join. The command enlists
/// in the DbContext's ambient transaction when one is open — the cached
/// telemetry dual-write runs the audit insert and the SiteCalls upsert inside
/// a single transaction and both must commit or roll back together.
/// </remarks>
private async Task<int> InsertChunkAsync(
IReadOnlyList<AuditEvent> chunk, TimeSpan? commandTimeout, CancellationToken ct)
{
var sql = new StringBuilder(256 + (chunk.Count * 64));
sql.Append("INSERT INTO dbo.AuditLog (").Append(CanonicalColumnList).Append(")\n");
sql.Append("SELECT v.EventId, v.OccurredAtUtc, v.Actor, v.Action, v.Outcome, v.Category, ");
sql.Append("v.Target, v.SourceNode, v.CorrelationId, v.DetailsJson\n");
sql.Append("FROM (VALUES\n");
for (var i = 0; i < chunk.Count; i++)
{
if (i > 0)
{
sql.Append(",\n");
}
sql.Append(" (@e").Append(i)
.Append(",@t").Append(i)
.Append(",@a").Append(i)
.Append(",@n").Append(i)
.Append(",@o").Append(i)
.Append(",@c").Append(i)
.Append(",@g").Append(i)
.Append(",@s").Append(i)
.Append(",@r").Append(i)
.Append(",@d").Append(i)
.Append(')');
}
sql.Append("\n) AS v (").Append(CanonicalColumnList).Append(")\n");
sql.Append("WHERE NOT EXISTS (SELECT 1 FROM dbo.AuditLog x WHERE x.EventId = v.EventId);");
var conn = _context.Database.GetDbConnection();
var openedHere = false;
if (conn.State != ConnectionState.Open)
{
await conn.OpenAsync(ct).ConfigureAwait(false);
openedHere = true;
}
try
{
await using var cmd = conn.CreateCommand();
cmd.CommandText = sql.ToString();
cmd.Transaction = _context.Database.CurrentTransaction?.GetDbTransaction();
if (commandTimeout is { } timeout)
{
cmd.CommandTimeout = (int)timeout.TotalSeconds;
}
for (var i = 0; i < chunk.Count; i++)
{
var evt = chunk[i];
// Same canonical projection as the single-row path: UTC-kind
// OccurredAtUtc, empty Actor collapses to NULL, Outcome/Category
// bound as their varchar storage form.
var occurred = DateTime.SpecifyKind(evt.OccurredAtUtc.UtcDateTime, DateTimeKind.Utc);
object actor = string.IsNullOrEmpty(evt.Actor) ? DBNull.Value : evt.Actor;
AddParameter(cmd, "@e" + i, SqlDbType.UniqueIdentifier, size: 0, evt.EventId);
AddParameter(cmd, "@t" + i, SqlDbType.DateTime2, size: 0, occurred);
AddParameter(cmd, "@a" + i, SqlDbType.NVarChar, size: 256, actor);
AddParameter(cmd, "@n" + i, SqlDbType.VarChar, size: 64, evt.Action);
AddParameter(cmd, "@o" + i, SqlDbType.VarChar, size: 16, evt.Outcome.ToString());
AddParameter(cmd, "@c" + i, SqlDbType.VarChar, size: 32, evt.Category);
AddParameter(cmd, "@g" + i, SqlDbType.NVarChar, size: 256, evt.Target);
AddParameter(cmd, "@s" + i, SqlDbType.VarChar, size: 64, evt.SourceNode);
AddParameter(cmd, "@r" + i, SqlDbType.UniqueIdentifier, size: 0, evt.CorrelationId);
AddParameter(cmd, "@d" + i, SqlDbType.NVarChar, size: -1, evt.DetailsJson);
}
return await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
}
finally
{
if (openedHere)
{
await conn.CloseAsync().ConfigureAwait(false);
}
}
}
/// <summary>
/// Binds one explicitly-typed parameter. A null CLR value binds as
/// <see cref="DBNull"/> while KEEPING its declared <see cref="SqlDbType"/>,
/// which is what makes the VALUES constructor's derived column types stable
/// regardless of which rows happen to carry nulls.
/// </summary>
private static void AddParameter(
System.Data.Common.DbCommand cmd, string name, SqlDbType type, int size, object? value)
{
var p = (SqlParameter)cmd.CreateParameter();
p.ParameterName = name;
p.SqlDbType = type;
if (size != 0)
{
p.Size = size;
}
p.Value = value ?? DBNull.Value;
cmd.Parameters.Add(p);
}
/// <inheritdoc />
public async Task<IReadOnlyList<AuditEvent>> QueryAsync(
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default)
@@ -229,13 +439,23 @@ VALUES
/// <inheritdoc />
public async Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default)
{
// The drop-and-rebuild batch below runs via
// The switch batch below runs via
// ExecuteSqlRaw with NO EF user-transaction — it carries its own server-side
// BEGIN TRANSACTION / TRY-CATCH / ROLLBACK — so the DbContext's retrying
// execution strategy (EnableRetryOnFailure) MAY auto-replay the whole batch on
// a transient fault. That replay is safe: every step is IF-EXISTS / IF-NOT-EXISTS
// guarded and the staging table is GUID-suffixed, so a re-run is idempotent.
//
// ALIGNED UNIQUENESS (WP2.2): there is no longer an index drop/rebuild around
// the SWITCH. EventId uniqueness is now enforced solely by the clustered
// PK_AuditLog (EventId, OccurredAtUtc), which is partition-aligned on
// ps_AuditLog_Month(OccurredAtUtc) — so ALTER TABLE … SWITCH PARTITION has no
// non-aligned index to object to. The former dance dropped UX_AuditLog_EventId,
// switched, then rebuilt it OFFLINE inside the same transaction: a whole-table
// unique-index build blocking every writer for the duration of the purge, and a
// window in which the idempotency-supporting index did not exist at all. Both
// are gone. See migration AlignAuditLogEventIdUniqueness for the reasoning.
//
// Maintenance timeout in whole seconds (ADO.NET CommandTimeout unit). Null leaves the
// provider default in place. See AuditLogPurgeOptions.MaintenanceCommandTimeoutMinutes /
// arch-review 04 S2 for why the ~30s default is unsafe for the switch-out dance.
@@ -270,8 +490,13 @@ VALUES
BEGIN TRY
BEGIN TRANSACTION;
-- 1. Drop the non-aligned unique index. ALTER TABLE SWITCH refuses
-- to run while it exists.
-- 1. Defensive cleanup for databases created before
-- AlignAuditLogEventIdUniqueness: the migration drops the
-- non-aligned UX_AuditLog_EventId, but a database restored from an
-- older backup could still carry it and SWITCH refuses to run while
-- a non-aligned unique index exists. Dropping it here is idempotent
-- and permanent the aligned clustered PK is the only uniqueness
-- enforcement the ingest path needs, so there is nothing to rebuild.
IF EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
DROP INDEX UX_AuditLog_EventId ON dbo.AuditLog;
@@ -313,31 +538,19 @@ VALUES
-- 4. Drop staging the rows are discarded here. This is the purge.
DROP TABLE dbo.[{stagingTableName}];
-- 5. Rebuild the non-aligned unique index. Live traffic that hit the
-- table during steps 1-4 saw composite-PK uniqueness only; from
-- here on, single-column EventId uniqueness is restored.
CREATE UNIQUE NONCLUSTERED INDEX UX_AuditLog_EventId ON dbo.AuditLog (EventId) ON [PRIMARY];
COMMIT TRANSACTION;
END TRY
BEGIN CATCH
IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION;
-- Best-effort staging cleanup. The DROP INDEX in step 1 is now
-- rolled back (so the index is back), but the staging table from
-- step 2 may or may not survive the rollback depending on the
-- failure point. Guard the DROP so a missing staging table doesn't
-- mask the original error.
-- Best-effort staging cleanup. The staging table from step 2 may or
-- may not survive the rollback depending on the failure point. Guard
-- the DROP so a missing staging table doesn't mask the original error.
-- Nothing else needs repairing: uniqueness lives on the clustered PK,
-- which the switch never touches, so a failed purge can no longer
-- leave the live table without its idempotency enforcement.
IF OBJECT_ID('dbo.[{stagingTableName}]', 'U') IS NOT NULL DROP TABLE dbo.[{stagingTableName}];
-- Idempotent index rebuild covers the niche case where ROLLBACK
-- failed to restore UX_AuditLog_EventId (or the failure happened
-- AFTER the COMMIT, which shouldn't be possible inside this TRY
-- but is cheap insurance). Without this, a failed switch could
-- leave the live table without its idempotency-supporting index.
IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
CREATE UNIQUE NONCLUSTERED INDEX UX_AuditLog_EventId ON dbo.AuditLog (EventId) ON [PRIMARY];
-- Surface the original error to the caller the purge actor logs
-- and continues with the next boundary.
THROW;
@@ -128,6 +128,37 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
var groups = samples.GroupBy(s => new SeriesHourKey(
s.Source, s.Metric, s.Scope, s.ScopeKey, TruncateToHour(s.CapturedAtUtc)));
// Preload every rollup row already covering this window in ONE query and
// index it by series+hour (WP2.2). The predecessor issued a
// FirstOrDefaultAsync existence probe PER (series, hour) group — an N+1
// that scaled with the metric catalogue times the lookback: a 3 h re-fold
// over ~40 series cost ~120 sequential round trips before a single row was
// written. The window is bounded by the caller's small trailing lookback
// and the rollup table holds exactly one row per series-hour, so the
// preload is a narrow range seek on IX_KpiRollupHourly_Series.
//
// Deliberately TRACKED (not a projection): the re-fold path mutates the
// existing entity in place and relies on the change tracker to emit the
// UPDATE. The dictionary's ScopeKey comparison is ordinal where the
// previous SQL predicate used the database collation; both sides are
// written from the same KpiSample.ScopeKey values, so they are
// byte-identical in practice and a residual mismatch degrades to the
// already-handled upsert-race path rather than a wrong aggregate.
var existingRollups = await _context.KpiRollupHourly
.Where(r => r.HourStartUtc >= from && r.HourStartUtc < to)
.ToListAsync(cancellationToken);
var existingByKey = new Dictionary<SeriesHourKey, KpiRollupHourly>(existingRollups.Count);
foreach (var row in existingRollups)
{
existingByKey[new SeriesHourKey(
row.Source,
row.Metric,
row.Scope,
row.ScopeKey,
DateTime.SpecifyKind(row.HourStartUtc, DateTimeKind.Utc))] = row;
}
foreach (var group in groups)
{
var key = group.Key;
@@ -143,18 +174,11 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
var maxValue = group.Max(s => s.Value);
var sampleCount = group.Count();
// Idempotent upsert on the unique series+hour key. The ScopeKey == key.ScopeKey
// comparison matches null against the Global-scope rows (IS NULL) exactly as the
// UNIQUE IX_KpiRollupHourly_Series index treats a null key as participating.
var existing = await _context.KpiRollupHourly.FirstOrDefaultAsync(
r => r.Source == key.Source
&& r.Metric == key.Metric
&& r.Scope == key.Scope
&& r.ScopeKey == key.ScopeKey
&& r.HourStartUtc == key.HourStartUtc,
cancellationToken);
if (existing is null)
// Idempotent upsert on the unique series+hour key, resolved against the
// preloaded dictionary. A null ScopeKey keys the Global-scope rows exactly
// as the UNIQUE IX_KpiRollupHourly_Series index treats a null key as
// participating.
if (!existingByKey.TryGetValue(key, out var existing))
{
_context.KpiRollupHourly.Add(new KpiRollupHourly
{
@@ -140,7 +140,14 @@ VALUES
public async Task<IReadOnlyList<Notification>> GetDueAsync(
DateTimeOffset now, int batchSize, CancellationToken cancellationToken = default)
{
// AsNoTracking (WP2.2): the dispatcher mutates each row's delivery state in
// memory and persists it through UpdateAsync, which is now a targeted
// server-side ExecuteUpdate and needs no change tracker. Tracking a batch
// of notifications — each carrying an nvarchar(max) Body and TypeData —
// paid for a full snapshot copy per row plus a DetectChanges scan of the
// whole batch on every save.
return await _context.Notifications
.AsNoTracking()
.Where(n => n.Status == NotificationStatus.Pending
|| (n.Status == NotificationStatus.Retrying
&& n.NextAttemptAt != null
@@ -150,22 +157,59 @@ VALUES
.ToListAsync(cancellationToken);
}
/// <inheritdoc />
public async Task<Notification?> GetByIdAsync(string notificationId, CancellationToken cancellationToken = default)
=> await _context.Notifications.FindAsync(new object[] { notificationId }, cancellationToken);
/// <inheritdoc />
public async Task UpdateAsync(Notification n, CancellationToken cancellationToken = default)
{
_context.Notifications.Update(n);
await _context.SaveChangesAsync(cancellationToken);
ArgumentNullException.ThrowIfNull(n);
// Targeted server-side UPDATE of the seven mutable delivery-state columns
// (WP2.2). The predecessor called DbSet.Update(n) + SaveChanges, which
// marks EVERY property modified and rewrites all 21 columns — including
// the immutable nvarchar(max) Body/TypeData payloads — on every single
// delivery attempt. ExecuteUpdate also bypasses the change tracker
// entirely, so it composes with the untracked GetDueAsync read.
//
// Immutable-by-contract columns (NotificationId, Type, ListName, Subject,
// Body, TypeData, Source*, Origin*, SiteEnqueuedAt, CreatedAt) are
// deliberately absent — see the interface contract: nothing in the
// notification lifecycle ever changes them, and omitting them is what
// makes the write narrow.
var status = n.Status;
var retryCount = n.RetryCount;
var lastError = n.LastError;
var resolvedTargets = n.ResolvedTargets;
var lastAttemptAt = n.LastAttemptAt;
var nextAttemptAt = n.NextAttemptAt;
var deliveredAt = n.DeliveredAt;
var notificationId = n.NotificationId;
await _context.Notifications
.Where(row => row.NotificationId == notificationId)
.ExecuteUpdateAsync(
setters => setters
.SetProperty(row => row.Status, status)
.SetProperty(row => row.RetryCount, retryCount)
.SetProperty(row => row.LastError, lastError)
.SetProperty(row => row.ResolvedTargets, resolvedTargets)
.SetProperty(row => row.LastAttemptAt, lastAttemptAt)
.SetProperty(row => row.NextAttemptAt, nextAttemptAt)
.SetProperty(row => row.DeliveredAt, deliveredAt),
cancellationToken);
}
/// <inheritdoc />
public async Task<Notification?> GetByIdAsync(string notificationId, CancellationToken cancellationToken = default)
=> await _context.Notifications.FindAsync(new object[] { notificationId }, cancellationToken);
/// <inheritdoc />
public async Task<(IReadOnlyList<Notification> Rows, int TotalCount)> QueryAsync(
NotificationOutboxFilter filter, int pageNumber, int pageSize, CancellationToken cancellationToken = default)
{
var query = _context.Notifications.AsQueryable();
// AsNoTracking (WP2.2): this is the Central UI's read-only list page. The
// rows are projected to the wire and never saved, so tracking them cost a
// snapshot copy of every nvarchar(max) Body in the page for nothing.
var query = _context.Notifications.AsNoTracking().AsQueryable();
if (filter.Status is { } status)
{
@@ -218,8 +262,14 @@ VALUES
var totalCount = await query.CountAsync(cancellationToken);
// NotificationId breaks CreatedAt ties so the OFFSET window is deterministic —
// without it two rows sharing a CreatedAt could appear on both pages or on
// neither. (This page keeps OFFSET paging rather than the sibling repos'
// keyset cursor because its contract surfaces a page number and a total
// count, neither of which a keyset cursor can express.)
var rows = await query
.OrderByDescending(n => n.CreatedAt)
.ThenByDescending(n => n.NotificationId)
.Skip((pageNumber - 1) * pageSize)
.Take(pageSize)
.ToListAsync(cancellationToken);
@@ -265,7 +315,24 @@ VALUES
// One conditional-aggregation pass replaces four sequential COUNT round trips:
// each metric is a COUNT(CASE WHEN <predicate> THEN 1 END) over the same scan
// (arch-review 04). GroupBy(_ => 1) yields a single group (no rows → no group).
//
// WP2.2 — the aggregation is now PREDICATE-RESTRICTED instead of scanning the
// whole table. Every KPI here is about the live queue (Pending/Retrying), the
// parked backlog, or the last delivery interval; the overwhelming bulk of the
// Notifications table is historical Delivered and Discarded rows that
// contribute to NONE of them. The pre-filter below is the union of the
// metric-contributing predicates, which lets the optimizer seek the status
// index for the live/parked legs and the filtered
// (DeliveredAt) WHERE Status='Delivered' index (WP1.4) for the interval leg,
// instead of paying a full scan whose cost grows with retained history.
// Mirrors the pre-filter ComputePerSiteKpisAsync/ComputePerNodeKpisAsync
// already use — the global snapshot was the odd one out.
var counts = await _context.Notifications
.Where(n => n.Status == NotificationStatus.Pending
|| n.Status == NotificationStatus.Retrying
|| n.Status == NotificationStatus.Parked
|| (n.Status == NotificationStatus.Delivered
&& n.DeliveredAt != null && n.DeliveredAt >= deliveredSince))
.GroupBy(_ => 1)
.Select(g => new
{
@@ -72,49 +72,36 @@ public class SiteCallAuditRepository : ISiteCallAuditRepository
var idText = siteCall.TrackedOperationId.Value.ToString("D");
var incomingRank = GetRankOrThrow(siteCall.Status);
// Step 1: insert-if-not-exists. Like AuditLogRepository.InsertIfNotExistsAsync
// this is check-then-act so a duplicate-key violation may surface under
// concurrent inserts on the same id — caught + logged at Debug.
// ONE round trip, UPDATE-first (WP2.2). The predecessor issued an
// unconditional IF NOT EXISTS … INSERT and THEN a monotonic UPDATE — two
// statements, two round trips, on every single packet, of which the insert
// half was wasted work for every packet after the first (the steady state:
// a cached call emits Submitted → Forwarded → Attempted → terminal, so
// three of four packets hit an existing row).
//
// SourceNode-stamping: the column is included in the INSERT
// column list / VALUES so a fresh row carries the originating node
// name (node-a/node-b for site rows). A null SourceNode (legacy hosts
// / unstamped reconciled rows) writes NULL straight through.
try
{
await _context.Database.ExecuteSqlInterpolatedAsync(
$@"IF NOT EXISTS (SELECT 1 FROM dbo.SiteCalls WHERE TrackedOperationId = {idText})
INSERT INTO dbo.SiteCalls
(TrackedOperationId, Channel, Target, SourceSite, SourceNode, Status, RetryCount,
LastError, HttpStatus, CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, IngestedAtUtc)
VALUES
({idText}, {siteCall.Channel}, {siteCall.Target}, {siteCall.SourceSite}, {siteCall.SourceNode}, {siteCall.Status}, {siteCall.RetryCount},
{siteCall.LastError}, {siteCall.HttpStatus}, {siteCall.CreatedAtUtc}, {siteCall.UpdatedAtUtc}, {siteCall.TerminalAtUtc}, {siteCall.IngestedAtUtc});",
ct);
}
catch (SqlException ex) when (
ex.Number == SqlErrorUniqueIndexViolation
|| ex.Number == SqlErrorPrimaryKeyViolation)
{
_logger.LogDebug(
ex,
"SiteCallAuditRepository.UpsertAsync swallowed duplicate-key violation (error {SqlErrorNumber}) for TrackedOperationId {TrackedOperationId}; falling through to monotonic update.",
ex.Number,
idText);
}
// Step 2: monotonic update with a same-rank freshness tiebreaker. The
// CASE expression maps the stored Status string to the same rank table
// the caller uses. We mutate when EITHER the incoming rank is strictly
// greater, OR the incoming rank equals the stored rank AND that rank is
// non-terminal (< TerminalRank) AND the incoming UpdatedAtUtc is strictly
// newer than the stored one — so a retrying call's Attempted-phase
// RetryCount/LastError/HttpStatus stay live instead of freezing at the
// first Attempted packet. Terminal ranks are excluded from the
// tiebreaker, so a later terminal NEVER overwrites an earlier one; equal
// stamps are inert (idempotent replay) and a lower rank is always a no-op.
// The combined batch below runs UPDATE first and inserts only when the
// UPDATE matched nothing AND the row genuinely does not exist. The
// NOT EXISTS re-check is load-bearing: @@ROWCOUNT = 0 is ALSO what a
// monotonic REJECTION looks like (a stale or regressive packet against an
// existing row), and inserting there would resurrect a row the guard just
// refused. Both statements ship in one command text, so this is one
// round trip, not two.
//
// SourceNode-stamping: SourceNode is updated via
// Monotonic update semantics are unchanged: mutate when EITHER the
// incoming rank is strictly greater, OR the incoming rank equals the
// stored rank AND that rank is non-terminal (< TerminalRank) AND the
// incoming UpdatedAtUtc is strictly newer than the stored one — so a
// retrying call's Attempted-phase RetryCount/LastError/HttpStatus stay
// live instead of freezing at the first Attempted packet. Terminal ranks
// are excluded from the tiebreaker, so a later terminal NEVER overwrites
// an earlier one; equal stamps are inert (idempotent replay) and a lower
// rank is always a no-op.
//
// SourceNode-stamping: the column is included in the INSERT column list /
// VALUES so a fresh row carries the originating node name (node-a/node-b
// for site rows). A null SourceNode (legacy hosts / unstamped reconciled
// rows) writes NULL straight through. On the UPDATE leg SourceNode is
// written via
// COALESCE(@SourceNode, SourceNode). The operator returns @SourceNode
// when it is non-null, otherwise the stored value — so the column
// behaves protectively: a later packet that carries a null
@@ -128,8 +115,12 @@ VALUES
// lifecycle every packet should carry the same SourceNode value (one
// execution, one node) so the "overwrite" path is in practice
// idempotent.
try
{
await _context.Database.ExecuteSqlInterpolatedAsync(
$@"UPDATE dbo.SiteCalls
$@"DECLARE @updated int;
UPDATE dbo.SiteCalls
SET Status = {siteCall.Status},
RetryCount = {siteCall.RetryCount},
LastError = {siteCall.LastError},
@@ -162,9 +153,41 @@ WHERE TrackedOperationId = {idText}
ELSE -1
END)
AND {incomingRank} < {TerminalRank}
AND UpdatedAtUtc < {siteCall.UpdatedAtUtc} ) );",
AND UpdatedAtUtc < {siteCall.UpdatedAtUtc} ) );
-- Captured IMMEDIATELY after the UPDATE: @@ROWCOUNT is reset by the next
-- statement, and reading it inline inside a compound IF condition alongside a
-- subquery is not safe (the subquery's own execution can clobber it).
SET @updated = @@ROWCOUNT;
IF @updated = 0 AND NOT EXISTS (SELECT 1 FROM dbo.SiteCalls WHERE TrackedOperationId = {idText})
INSERT INTO dbo.SiteCalls
(TrackedOperationId, Channel, Target, SourceSite, SourceNode, Status, RetryCount,
LastError, HttpStatus, CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, IngestedAtUtc)
VALUES
({idText}, {siteCall.Channel}, {siteCall.Target}, {siteCall.SourceSite}, {siteCall.SourceNode}, {siteCall.Status}, {siteCall.RetryCount},
{siteCall.LastError}, {siteCall.HttpStatus}, {siteCall.CreatedAtUtc}, {siteCall.UpdatedAtUtc}, {siteCall.TerminalAtUtc}, {siteCall.IngestedAtUtc});",
ct);
}
catch (SqlException ex) when (
ex.Number == SqlErrorUniqueIndexViolation
|| ex.Number == SqlErrorPrimaryKeyViolation)
{
// Two concurrent sessions both found the row absent and both raced to
// INSERT; the loser raises 2601/2627 against the TrackedOperationId
// primary key. The winner's row IS the first-write, and this packet's
// content is by construction the same lifecycle state, so the race
// outcome is semantically a no-op. Swallow at Debug — the same
// check-then-act contract the sibling AuditLog/Notification repos
// document. Note the loser's UPDATE leg already ran (against no row),
// so nothing is left half-applied.
_logger.LogDebug(
ex,
"SiteCallAuditRepository.UpsertAsync swallowed duplicate-key violation (error {SqlErrorNumber}) for TrackedOperationId {TrackedOperationId}; treating as no-op.",
ex.Number,
idText);
}
}
/// <inheritdoc />
public async Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default)
@@ -26,6 +26,22 @@ public static class ServiceCollectionExtensions
// registers IDataProtectionProvider as a singleton; resolving it here does not recurse
// because key-ring loading is lazy (first Protect/Unprotect), not triggered by
// CreateProtector during model building.
//
// POOLING IS DELIBERATELY NOT USED (WP2.2 — verified, not overlooked).
// AddDbContextPool requires a context with a SINGLE public constructor taking
// only DbContextOptions<TContext>; EF Core constructs pooled instances through
// its own activator and cannot supply anything else. ScadaBridgeDbContext has
// two public constructors and the runtime one takes IDataProtectionProvider,
// because the encrypting value converter for secret-bearing columns is built
// during OnModelCreating from that provider. Worse, the model itself DIFFERS
// between the two constructors (no provider ⇒ no encrypting converter), so a
// pooled activator would silently produce a context that reads secret columns
// as ciphertext. Making this poolable means moving the protector out of the
// constructor and into a DbContextOptions extension — a change to the
// secrets-at-rest path, which is not a performance refactor. The registration
// below (a scoped factory overriding AddDbContext's activator) is what makes
// the provider reach the context at all, and it also bypasses pooling by
// construction. Revisit only alongside a deliberate secrets-plumbing change.
services.AddDbContext<ScadaBridgeDbContext>((serviceProvider, options) =>
{
options.UseSqlServer(
@@ -158,6 +158,26 @@ public class SiteCallAuditActor : ReceiveActor
/// </summary>
private readonly Dictionary<string, bool> _reconciliationPinned = new();
/// <summary>
/// Actor-system EventStream captured on the actor thread at handler-registration
/// time. The reconciliation pass runs off-mailbox and publishes the pinned-state
/// transition from there, so it must not reach through <c>Context</c>.
/// </summary>
private Akka.Event.EventStream _eventStream = null!;
/// <summary>
/// Single-flight guard for the off-mailbox reconciliation pass. Raised on the
/// actor thread when a tick launches a pass and lowered on the actor thread
/// when the piped <see cref="ReconciliationComplete"/> arrives, so the
/// per-site cursor/pinned dictionaries the pass mutates are only ever touched
/// by one task at a time and the mailbox supplies the memory barrier between
/// consecutive passes.
/// </summary>
private bool _reconciling;
/// <summary>Single-flight guard for the off-mailbox terminal-row purge pass.</summary>
private bool _purging;
private ICancelable? _reconciliationTimer;
private ICancelable? _purgeTimer;
@@ -338,8 +358,79 @@ public class SiteCallAuditActor : ReceiveActor
// the daily terminal-row purge. Handlers stay alive across faults via
// their own per-site / per-tick try/catch (mirroring the ingest path);
// the timers are only started when their collaborators are available.
ReceiveAsync<ReconciliationTick>(_ => OnReconciliationTickAsync());
ReceiveAsync<PurgeTick>(_ => OnPurgeTickAsync());
//
// OFF-MAILBOX (WP2.2). Both passes run as PipeTo-completed background
// tasks behind a single-flight guard rather than as ReceiveAsync bodies.
// A ReceiveAsync handler occupies the actor for its whole duration, and a
// reconciliation pass is unbounded work — every site, up to
// MaxReconciliationPagesPerTick network pulls each, one upsert per row.
// Post-outage catch-up therefore blocked telemetry ingest, UI queries and
// KPI Asks behind it until the drain finished, and those callers timed out
// rather than queued. NotificationOutboxActor's dispatch sweep is the
// in-repo reference for this shape.
Receive<ReconciliationTick>(_ => HandleReconciliationTick());
Receive<ReconciliationComplete>(_ => _reconciling = false);
Receive<PurgeTick>(_ => HandlePurgeTick());
Receive<PurgeComplete>(_ => _purging = false);
// Captured on the actor thread so the background passes never touch
// Context off-thread. EventStream itself is thread-safe.
_eventStream = Context.System.EventStream;
}
/// <summary>
/// Launches a reconciliation pass unless one is already in flight, dropping
/// the tick if so. Overlapping passes are not merely wasteful — they would
/// race on the per-site cursor and pinned-latch dictionaries, which the
/// single-flight guard keeps confined to one task at a time (the guard itself
/// is only ever mutated on the actor thread: raised here, lowered by the
/// piped completion message).
/// </summary>
private void HandleReconciliationTick()
{
if (_reconciling)
{
return;
}
_reconciling = true;
// OnReconciliationTickAsync swallows its own per-site errors, but the
// failure projection is kept as a belt-and-braces guard so even a faulted
// task still lowers the guard — otherwise reconciliation would wedge
// permanently after a single unexpected throw.
OnReconciliationTickAsync().PipeTo(
Self,
success: () => ReconciliationComplete.Instance,
failure: ex =>
{
_logger.LogError(ex, "SiteCallAudit reconciliation pass faulted unexpectedly.");
return ReconciliationComplete.Instance;
});
}
/// <summary>
/// Launches a purge pass unless one is already in flight. Same single-flight
/// discipline as <see cref="HandleReconciliationTick"/>: a purge that outlives
/// its interval (a large catch-up after an outage) must not stack.
/// </summary>
private void HandlePurgeTick()
{
if (_purging)
{
return;
}
_purging = true;
OnPurgeTickAsync().PipeTo(
Self,
success: () => PurgeComplete.Instance,
failure: ex =>
{
_logger.LogError(ex, "SiteCallAudit purge pass faulted unexpectedly.");
return PurgeComplete.Instance;
});
}
/// <inheritdoc />
@@ -743,7 +834,10 @@ public class SiteCallAuditActor : ReceiveActor
}
_reconciliationPinned[siteId] = pinned;
Context.System.EventStream.Publish(new SiteCallReconciliationPinnedChanged(siteId, pinned));
// _eventStream, not Context.System.EventStream: this runs on the
// off-mailbox reconciliation pass, where Context must not be touched.
_eventStream.Publish(new SiteCallReconciliationPinnedChanged(siteId, pinned));
}
// ── Piece B: daily terminal-row purge scheduler ──
@@ -1452,6 +1546,24 @@ public class SiteCallAuditActor : ReceiveActor
public static readonly PurgeTick Instance = new();
private PurgeTick() { }
}
/// <summary>
/// Piped back to <c>Self</c> when an off-mailbox reconciliation pass ends
/// (successfully or not) so the single-flight guard is lowered on the actor
/// thread rather than from the background task.
/// </summary>
internal sealed class ReconciliationComplete
{
public static readonly ReconciliationComplete Instance = new();
private ReconciliationComplete() { }
}
/// <summary>Purge counterpart of <see cref="ReconciliationComplete"/>.</summary>
internal sealed class PurgeComplete
{
public static readonly PurgeComplete Instance = new();
private PurgeComplete() { }
}
}
/// <summary>
@@ -121,6 +121,105 @@ public class AuditLogIngestActorTests : TestKit, IClassFixture<MsSqlMigrationFix
Assert.Equal(3, count);
}
[SkippableFact]
public async Task Receive_DuplicateEventIds_WithinOnePacket_ProduceOneRow_AndAreAllAcked()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// WP2.2 writes each packet as ONE set-based statement, whose anti-semi-join
// only sees already-COMMITTED rows. Two copies of an EventId inside one
// packet would therefore both pass it and collide on the clustered PK,
// taking the whole packet down — the repository de-duplicates first.
// Every id is still acked: the site's contract is "this row is now
// present at central", which is true for both copies.
var siteId = NewSiteId();
var repeated = NewEvent(siteId);
var other = NewEvent(siteId);
var batch = new List<AuditEvent> { repeated, other, repeated, repeated };
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
var actor = CreateActor(repo);
actor.Tell(new IngestAuditEventsCommand(batch), TestActor);
var reply = ExpectMsg<IngestAuditEventsReply>(TimeSpan.FromSeconds(10));
Assert.Equal(4, reply.AcceptedEventIds.Count);
Assert.True(
new[] { repeated.EventId, other.EventId }.ToHashSet()
.SetEquals(reply.AcceptedEventIds.ToHashSet()));
await using var readContext = CreateContext();
var rows = await readContext.Set<AuditLogRow>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.Equal(2, rows.Count);
}
[SkippableFact]
public async Task Receive_SamePacketTwice_IsIdempotent_AcrossPackets()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// A site whose ack was lost re-delivers the identical packet on the next
// drain, and the reconciliation pull can re-deliver it a third time. Each
// replay must ack fully (so the site can finally flip its rows to
// Forwarded) while writing nothing new.
var siteId = NewSiteId();
var events = Enumerable.Range(0, 6).Select(_ => NewEvent(siteId)).ToList();
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
var actor = CreateActor(repo);
for (var attempt = 0; attempt < 3; attempt++)
{
actor.Tell(new IngestAuditEventsCommand(events), TestActor);
var reply = ExpectMsg<IngestAuditEventsReply>(TimeSpan.FromSeconds(10));
Assert.Equal(6, reply.AcceptedEventIds.Count);
Assert.True(
events.Select(e => e.EventId).ToHashSet()
.SetEquals(reply.AcceptedEventIds.ToHashSet()));
}
await using var readContext = CreateContext();
var rows = await readContext.Set<AuditLogRow>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.Equal(6, rows.Count);
}
[SkippableFact]
public async Task Receive_OverlappingPackets_InsertOnlyTheNewRows()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// Partially-overlapping packets are the normal reconciliation shape: the
// pull cursor re-serves a tail the push already delivered. The overlap
// must be a silent no-op and the new rows must land.
var siteId = NewSiteId();
var first = Enumerable.Range(0, 4).Select(_ => NewEvent(siteId)).ToList();
var second = first.Skip(2).Concat(
Enumerable.Range(0, 3).Select(_ => NewEvent(siteId))).ToList();
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
var actor = CreateActor(repo);
actor.Tell(new IngestAuditEventsCommand(first), TestActor);
ExpectMsg<IngestAuditEventsReply>(TimeSpan.FromSeconds(10));
actor.Tell(new IngestAuditEventsCommand(second), TestActor);
var reply = ExpectMsg<IngestAuditEventsReply>(TimeSpan.FromSeconds(10));
Assert.Equal(5, reply.AcceptedEventIds.Count);
await using var readContext = CreateContext();
var rows = await readContext.Set<AuditLogRow>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.Equal(7, rows.Count);
}
[SkippableFact]
public async Task Receive_Sets_IngestedAtUtc_Before_Insert()
{
@@ -28,20 +28,19 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
/// <list type="number">
/// <item>The oldest partition (Jan) is removed.</item>
/// <item>Newer partitions (Feb + Mar) are untouched.</item>
/// <item>The <c>UX_AuditLog_EventId</c> unique index survives the
/// drop-and-rebuild dance.</item>
/// <item>The switch leaves the aligned clustered <c>PK_AuditLog</c> intact and
/// does NOT (re)create the non-aligned <c>UX_AuditLog_EventId</c>.</item>
/// <item><see cref="IAuditLogRepository.InsertIfNotExistsAsync"/> remains
/// idempotent against the rebuilt index after the purge.</item>
/// idempotent against the aligned key after the purge.</item>
/// </list>
/// </summary>
/// <remarks>
/// The brief calls out that direct INSERTs bypass the writer role's INSERT-only
/// grant; the fixture connects as <c>sa</c> (see
/// <see cref="MsSqlMigrationFixture"/>'s default admin connection string), so
/// the seed step does not need the writer role at all. The drop-and-rebuild
/// dance itself runs under the same admin connection because the test owns
/// the database — the role granularity is exercised in the repository tests,
/// not here.
/// the seed step does not need the writer role at all. The switch batch itself
/// runs under the same admin connection because the test owns the database —
/// the role granularity is exercised in the repository tests, not here.
/// </remarks>
public class PartitionPurgeTests : TestKit, IClassFixture<MsSqlMigrationFixture>
{
@@ -110,22 +109,39 @@ VALUES
}
/// <summary>
/// Asserts that <c>UX_AuditLog_EventId</c> exists in
/// <c>sys.indexes</c>. The drop-and-rebuild dance briefly removes the
/// index inside its transaction; this check is meant to fire AFTER the
/// actor's purge tick has committed so the rebuilt index is observable.
/// Asserts the post-purge index state: the partition-ALIGNED clustered
/// <c>PK_AuditLog</c> is intact and the non-aligned <c>UX_AuditLog_EventId</c>
/// is absent (WP2.2 — <c>AlignAuditLogEventIdUniqueness</c>).
/// </summary>
private static async Task AssertUxIndexExistsAsync(SqlConnection conn)
/// <remarks>
/// The purge used to bracket its SWITCH with a DROP/CREATE of
/// <c>UX_AuditLog_EventId</c>, because a non-aligned unique index blocks
/// <c>ALTER TABLE … SWITCH PARTITION</c> — an offline whole-table index build
/// inside the switch transaction, plus a window with no idempotency index at
/// all. Uniqueness now rides the aligned clustered PK, so the switch is
/// metadata-only. Asserting the index's ABSENCE is what keeps that property:
/// anything that recreates it silently reinstates the rebuild.
/// </remarks>
private static async Task AssertAlignedUniquenessAsync(SqlConnection conn)
{
await using var cmd = conn.CreateCommand();
cmd.CommandText = @"
SELECT COUNT(*)
FROM sys.indexes
WHERE name = 'UX_AuditLog_EventId'
AND object_id = OBJECT_ID('dbo.AuditLog');";
var raw = await cmd.ExecuteScalarAsync();
var count = Convert.ToInt32(raw);
Assert.True(count == 1, $"UX_AuditLog_EventId should be present post-purge; sys.indexes count was {count}.");
SELECT
(SELECT COUNT(*) FROM sys.indexes
WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog')) AS NonAligned,
(SELECT COUNT(*) FROM sys.indexes
WHERE object_id = OBJECT_ID('dbo.AuditLog') AND is_primary_key = 1) AS ClusteredPk;";
await using var reader = await cmd.ExecuteReaderAsync();
Assert.True(await reader.ReadAsync());
var nonAligned = reader.GetInt32(0);
var clusteredPk = reader.GetInt32(1);
Assert.True(
nonAligned == 0,
$"UX_AuditLog_EventId must NOT exist post-purge (it blocks SWITCH PARTITION); sys.indexes count was {nonAligned}.");
Assert.True(
clusteredPk == 1,
$"The aligned clustered PK_AuditLog must survive the purge; sys.indexes primary-key count was {clusteredPk}.");
}
private IActorRef CreateActor(
@@ -255,20 +271,19 @@ WHERE name = 'UX_AuditLog_EventId'
}
// ---------------------------------------------------------------------
// 2. EndToEnd_UxIndexRebuilt_AfterPurge
// 2. EndToEnd_AlignedUniqueness_AfterPurge
// ---------------------------------------------------------------------
[SkippableFact]
public async Task EndToEnd_UxIndexRebuilt_AfterPurge()
public async Task EndToEnd_AlignedUniqueness_AfterPurge()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// Same shape as test 1 — purge the Jan-2026 partition and then assert the
// UX_AuditLog_EventId index is still present. RetentionDays is computed
// dynamically so the threshold always lands near 2026-01-20 (see SeedOccurredAt()).
// The drop-and-rebuild dance briefly removes the index inside its transaction
// (the SWITCH PARTITION step requires the non-aligned unique index to be absent),
// but step 5 rebuilds it before committing.
// index state. RetentionDays is computed dynamically so the threshold always
// lands near 2026-01-20 (see SeedOccurredAt()). Since WP2.2 the switch touches
// no index at all: uniqueness rides the aligned clustered PK, so there is
// nothing to drop before the SWITCH and nothing to rebuild after it.
var siteId = "purge-uxidx-" + Guid.NewGuid().ToString("N").Substring(0, 8);
var oldEventId = Guid.NewGuid();
var (oldOccurred, _, _, retentionDays) = SeedOccurredAt();
@@ -305,7 +320,7 @@ WHERE name = 'UX_AuditLog_EventId'
// Open a fresh connection (the actor's pool is owned by EF) and
// assert the index is present post-purge.
await using var check = _fixture.OpenConnection();
await AssertUxIndexExistsAsync(check);
await AssertAlignedUniquenessAsync(check);
}
// ---------------------------------------------------------------------
@@ -320,7 +335,7 @@ WHERE name = 'UX_AuditLog_EventId'
// Seed + purge the Jan-2026 row, THEN exercise InsertIfNotExistsAsync twice for
// a fresh recent EventId. The second call must be a no-op (duplicate-key collision
// swallowed by the repository, per M2 Bundle A's race-fix) — which means the
// rebuilt UX_AuditLog_EventId unique index is functioning as intended.
// aligned clustered PK is still enforcing uniqueness as intended.
// RetentionDays is computed dynamically so the threshold always lands near
// 2026-01-20 (see SeedOccurredAt()).
var siteId = "purge-idem-" + Guid.NewGuid().ToString("N").Substring(0, 8);
@@ -357,11 +372,11 @@ WHERE name = 'UX_AuditLog_EventId'
max: TimeSpan.FromSeconds(30));
// Settle then exercise InsertIfNotExistsAsync twice for the same
// EventId. The repository's idempotency relies on
// UX_AuditLog_EventId being present so the IF NOT EXISTS … INSERT
// race window resolves to a duplicate-key violation the repo
// swallows. If the index were missing here, two rows would land
// and the second InsertIfNotExistsAsync would silently double-insert.
// EventId. The repository's idempotency relies on the aligned clustered
// PK (EventId, OccurredAtUtc) being intact so the IF NOT EXISTS … INSERT
// race window resolves to a duplicate-key violation the repo swallows.
// If the switch had disturbed that key, two rows would land and the second
// InsertIfNotExistsAsync would silently double-insert.
await Task.Delay(TimeSpan.FromMilliseconds(500));
var freshEventId = Guid.NewGuid();
@@ -482,7 +497,7 @@ WHERE name = 'UX_AuditLog_EventId'
/// <summary>
/// Task 4 (arch-review 04, S2): proves the explicit-<see cref="TimeSpan"/> maintenance-timeout
/// overload of <see cref="IAuditLogRepository.SwitchOutPartitionAsync"/> runs the real
/// drop-and-rebuild dance to completion against SQL Server — the old row is purged, the kept row
/// staging/switch batch to completion against SQL Server — the old row is purged, the kept row
/// survives, and the returned sampled row-count reflects the switched partition. Exercising the
/// timeout path end-to-end guards against a regression that only sets the timeout on the sample
/// command and forgets the DDL batch (or vice versa).
@@ -520,8 +535,8 @@ WHERE name = 'UX_AuditLog_EventId'
Assert.DoesNotContain(rows, r => r.EventId == oldEventId);
Assert.Contains(rows, r => r.EventId == keptEventId);
// The dance must leave the idempotency-supporting unique index rebuilt.
// The switch must leave uniqueness enforcement exactly as it found it.
await using var check = _fixture.OpenConnection();
await AssertUxIndexExistsAsync(check);
await AssertAlignedUniquenessAsync(check);
}
}
@@ -52,22 +52,31 @@ public class AuditLogEntityTypeConfigurationTests : IDisposable
}
[Fact]
public void Configure_DeclaresUniqueIndex_OnEventIdAlone_ForIdempotencyLookups()
public void Configure_DeclaresNoNonAlignedEventIdIndex_UniquenessRidesTheAlignedClusteredKey()
{
// EventId remains globally unique (the idempotency key for
// InsertIfNotExistsAsync) via a dedicated unique index independent of the
// composite PK.
// WP2.2 (AlignAuditLogEventIdUniqueness): the standalone non-aligned
// UX_AuditLog_EventId is gone. EventId uniqueness now rides the clustered
// PK (EventId, OccurredAtUtc), which is partition-aligned on
// ps_AuditLog_Month — so ALTER TABLE ... SWITCH PARTITION no longer needs
// an offline index drop/rebuild around every retention purge. EventId is a
// GUID minted once at the site alongside OccurredAtUtc, so pair-uniqueness
// is EventId-uniqueness in practice, and the idempotency probe
// (WHERE EventId = @id) still seeks the clustered key's leading column.
//
// Re-declaring a single-column unique index here would silently reinstate
// the SWITCH incompatibility, so this test pins its absence.
var entity = _context.Model.FindEntityType(typeof(AuditLogRow));
Assert.NotNull(entity);
var eventIdIndex = entity!.GetIndexes()
.SingleOrDefault(i => i.GetDatabaseName() == "UX_AuditLog_EventId");
Assert.DoesNotContain(
entity!.GetIndexes(),
i => i.GetDatabaseName() == "UX_AuditLog_EventId");
Assert.NotNull(eventIdIndex);
Assert.True(eventIdIndex!.IsUnique);
var indexedProperty = Assert.Single(eventIdIndex.Properties);
Assert.Equal(nameof(AuditLogRow.EventId), indexedProperty.Name);
var pk = entity.FindPrimaryKey();
Assert.NotNull(pk);
Assert.Equal(
new[] { nameof(AuditLogRow.EventId), nameof(AuditLogRow.OccurredAtUtc) },
pk!.Properties.Select(p => p.Name).ToArray());
}
[Fact]
@@ -164,7 +173,8 @@ public class AuditLogEntityTypeConfigurationTests : IDisposable
"IX_AuditLog_ParentExecution",
"IX_AuditLog_Site_Occurred",
"IX_AuditLog_Target_Occurred",
"UX_AuditLog_EventId",
// UX_AuditLog_EventId is intentionally absent — dropped by
// AlignAuditLogEventIdUniqueness; the aligned clustered PK carries it.
};
Assert.Equal(expected, indexNames);
@@ -85,22 +85,27 @@ public class AddAuditLogTableMigrationTests : IClassFixture<MsSqlMigrationFixtur
}
/// <summary>
/// Verifies all nine named non-clustered indexes exist on the final
/// <c>dbo.AuditLog</c> table after all migrations have been applied.
/// Verifies all eight named non-clustered indexes exist on the final
/// <c>dbo.AuditLog</c> table after all migrations have been applied, and that
/// the non-aligned <c>UX_AuditLog_EventId</c> does NOT.
/// The original five indexes were created by <c>AddAuditLogTable</c>;
/// the <c>CollapseAuditLogToCanonical</c> (C5, Task 2.5) migration rebuilt
/// the table and added <c>IX_AuditLog_Execution</c>,
/// <c>IX_AuditLog_ParentExecution</c>, <c>IX_AuditLog_Node_Occurred</c>,
/// and <c>UX_AuditLog_EventId</c> — nine in total.
/// <c>IX_AuditLog_ParentExecution</c> and <c>IX_AuditLog_Node_Occurred</c>.
/// <c>UX_AuditLog_EventId</c> was created alongside them but dropped again by
/// <c>AlignAuditLogEventIdUniqueness</c> (WP2.2) — its non-alignment forced an
/// offline drop/rebuild around every partition-switch purge, and the clustered
/// <c>PK_AuditLog (EventId, OccurredAtUtc)</c> already enforces the same
/// uniqueness partition-aligned.
/// </summary>
[SkippableFact]
public async Task AppliesMigration_CreatesNineNamedIndexes()
public async Task AppliesMigration_CreatesEightNamedIndexes_AndNoNonAlignedUniqueIndex()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// All nine named non-clustered indexes present on dbo.AuditLog after
// All eight named non-clustered indexes present on dbo.AuditLog after
// the full migration history is applied (AddAuditLogTable through
// CollapseAuditLogToCanonical).
// AlignAuditLogEventIdUniqueness).
var expected = new[]
{
// Original five (AddAuditLogTable / AddAuditLogSourceNode):
@@ -113,7 +118,6 @@ public class AddAuditLogTableMigrationTests : IClassFixture<MsSqlMigrationFixtur
"IX_AuditLog_Execution",
"IX_AuditLog_ParentExecution",
"IX_AuditLog_Node_Occurred",
"UX_AuditLog_EventId",
};
foreach (var indexName in expected)
@@ -124,6 +128,16 @@ public class AddAuditLogTableMigrationTests : IClassFixture<MsSqlMigrationFixtur
$"WHERE o.name = 'AuditLog' AND i.name = '{indexName}';");
Assert.True(count == 1, $"Expected index '{indexName}' to exist on AuditLog; found {count}.");
}
var nonAligned = await ScalarAsync<int>(
"SELECT COUNT(*) FROM sys.indexes i " +
"INNER JOIN sys.objects o ON i.object_id = o.object_id " +
"WHERE o.name = 'AuditLog' AND i.name = 'UX_AuditLog_EventId';");
Assert.True(
nonAligned == 0,
"UX_AuditLog_EventId must NOT exist after AlignAuditLogEventIdUniqueness — "
+ "a non-aligned index blocks ALTER TABLE ... SWITCH PARTITION and reinstates "
+ $"the offline drop/rebuild the purge path was freed from; found {nonAligned}.");
}
[SkippableFact]
@@ -50,6 +50,16 @@ public class NotificationOutboxRepositoryKpiQueryShapeTests
$"ComputeKpisAsync issued {counter.Count} queries against Notifications; expected <= 2");
// The oldest lookup must be bounded (LIMIT/TOP), never a full non-terminal SELECT.
Assert.Contains(counter.Commands, sql => sql.Contains("LIMIT", StringComparison.OrdinalIgnoreCase));
// WP2.2: the aggregation must be PREDICATE-RESTRICTED, not an unrestricted
// scan. Every KPI here concerns the live queue, the parked backlog or the
// last delivery interval; historical Delivered/Discarded rows — which are
// the overwhelming bulk of a retained table — contribute to none of them.
// Without the WHERE the query's cost grows with retention rather than with
// the working set. (An index-restricted aggregation was already the shape
// of the per-site/per-node snapshots; the global one was the odd one out.)
Assert.All(
counter.Commands.Where(sql => sql.Contains("COUNT", StringComparison.OrdinalIgnoreCase)),
sql => Assert.Contains("WHERE", sql, StringComparison.OrdinalIgnoreCase));
Assert.Equal(3, global.QueueDepth);
Assert.NotNull(global.OldestPendingAge);
@@ -560,6 +560,164 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
Assert.Equal(1, count);
}
// ------------------------------------------------------------------------
// WP2.2: set-based InsertManyIfNotExistsAsync
// ------------------------------------------------------------------------
[SkippableFact]
public async Task InsertManyIfNotExistsAsync_WritesEveryDistinctEvent()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
var baseTime = new DateTime(2026, 5, 21, 8, 0, 0, DateTimeKind.Utc);
var events = Enumerable.Range(0, 12)
.Select(i => NewEvent(siteId, occurredAtUtc: baseTime.AddSeconds(i)))
.ToList();
var inserted = await repo.InsertManyIfNotExistsAsync(events);
Assert.Equal(12, inserted);
await using var readContext = CreateContext();
var rows = await readContext.Set<AuditLogRow>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.Equal(12, rows.Count);
Assert.Equal(
events.Select(e => e.EventId).OrderBy(g => g).ToArray(),
rows.Select(r => r.EventId).OrderBy(g => g).ToArray());
}
[SkippableFact]
public async Task InsertManyIfNotExistsAsync_DuplicateEventIdsWithinOnePacket_CollapseToOneRow()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
// A set-based INSERT … SELECT … WHERE NOT EXISTS only tests rows that are
// already COMMITTED, so two copies of one EventId inside a single VALUES
// constructor would both pass the anti-semi-join and collide on the
// clustered PK — taking the whole statement down with them. The repository
// de-duplicates the packet first (first-write-wins, matching the
// single-row contract), which this pins.
var occurred = new DateTime(2026, 5, 21, 9, 0, 0, DateTimeKind.Utc);
var first = NewEvent(siteId, occurredAtUtc: occurred);
var duplicate = ScadaBridgeAuditEventFactory.Create(
channel: AuditChannel.ApiOutbound,
kind: AuditKind.ApiCall,
status: AuditStatus.Delivered,
eventId: first.EventId,
occurredAtUtc: occurred,
sourceSiteId: siteId,
errorMessage: "duplicate-within-packet-should-be-ignored");
var other = NewEvent(siteId, occurredAtUtc: occurred.AddSeconds(1));
var inserted = await repo.InsertManyIfNotExistsAsync(
new[] { first, duplicate, other, duplicate });
Assert.Equal(2, inserted);
await using var readContext = CreateContext();
var rows = await readContext.Set<AuditLogRow>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.Equal(2, rows.Count);
// First-write-wins: the surviving row is the FIRST occurrence, so the
// duplicate's ErrorMessage never lands.
var stored = Assert.Single(rows, r => r.EventId == first.EventId);
Assert.Null(AuditDetailsCodec.Deserialize(stored.DetailsJson).ErrorMessage);
}
[SkippableFact]
public async Task InsertManyIfNotExistsAsync_DuplicateEventIdsAcrossPackets_AreIdempotent()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
var baseTime = new DateTime(2026, 5, 21, 10, 0, 0, DateTimeKind.Utc);
var packet = Enumerable.Range(0, 5)
.Select(i => NewEvent(siteId, occurredAtUtc: baseTime.AddSeconds(i)))
.ToList();
// A site retry / reconciliation pull re-delivers a packet that overlaps
// an already-ingested one. The second call must insert only the genuinely
// new rows and silently skip the rest (first-write-wins across packets).
var firstInserted = await repo.InsertManyIfNotExistsAsync(packet);
var extra = NewEvent(siteId, occurredAtUtc: baseTime.AddSeconds(99));
var secondInserted = await repo.InsertManyIfNotExistsAsync(
packet.Concat(new[] { extra }).ToList());
// A third, wholly-redundant replay writes nothing at all.
var thirdInserted = await repo.InsertManyIfNotExistsAsync(packet);
Assert.Equal(5, firstInserted);
Assert.Equal(1, secondInserted);
Assert.Equal(0, thirdInserted);
await using var readContext = CreateContext();
var rows = await readContext.Set<AuditLogRow>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.Equal(6, rows.Count);
}
[SkippableFact]
public async Task InsertManyIfNotExistsAsync_ChunksBeyondTheParameterCeiling()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = NewSiteId();
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
// 250 rows × 10 bound parameters = 2,500 — past SQL Server's 2,100
// parameter ceiling for a single statement, so this only succeeds if the
// repository chunks. Nulls in the optional columns (Target, SourceNode,
// CorrelationId, Actor) are left null on purpose: the VALUES constructor
// derives its column types from the parameters, so an untyped null would
// surface here as a conversion failure rather than silently.
var baseTime = new DateTime(2026, 5, 21, 11, 0, 0, DateTimeKind.Utc);
var events = Enumerable.Range(0, 250)
.Select(i => NewEvent(siteId, occurredAtUtc: baseTime.AddSeconds(i)))
.ToList();
var inserted = await repo.InsertManyIfNotExistsAsync(events);
Assert.Equal(250, inserted);
await using var readContext = CreateContext();
var count = await readContext.Set<AuditLogRow>()
.Where(e => e.SourceSiteId == siteId)
.CountAsync();
Assert.Equal(250, count);
}
[SkippableFact]
public async Task InsertManyIfNotExistsAsync_EmptyBatch_IsANoOp()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
Assert.Equal(0, await repo.InsertManyIfNotExistsAsync(Array.Empty<AuditEvent>()));
}
[SkippableFact]
public async Task QueryAsync_Keyset_SameOccurredAtUtc_TiebreaksOnEventId()
{
@@ -622,16 +780,17 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
// ------------------------------------------------------------------------
//
// The partition-switch path replaces M1's NotSupportedException stub with
// the production drop-DROP-INDEX → CREATE-staging → SWITCH PARTITION →
// DROP-staging → CREATE-INDEX dance documented in alog.md §4. These tests
// the production CREATE-staging → SWITCH PARTITION → DROP-staging batch
// documented in alog.md §4. WP2.2 removed the index drop/rebuild that used to
// bracket it: uniqueness rides the partition-ALIGNED clustered PK
// (EventId, OccurredAtUtc), so SWITCH has nothing to object to. These tests
// verify the side effects an outsider can observe:
// * rows in the targeted month are removed
// * rows in OTHER months are NOT touched
// * UX_AuditLog_EventId still exists after a successful switch
// * no non-aligned UX_AuditLog_EventId is (re)created by a switch
// * InsertIfNotExistsAsync's first-write-wins idempotency still holds
// after a switch (the rebuilt index is real)
// * a thrown SqlException leaves UX_AuditLog_EventId rebuilt (the CATCH
// branch's recovery path runs)
// after a switch (the aligned key really does enforce it)
// * a thrown SqlException leaves the table intact with no orphaned staging
[SkippableFact]
public async Task SwitchOutPartitionAsync_OldPartition_RemovesRows_NewPartitionsKept()
@@ -669,7 +828,7 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
}
[SkippableFact]
public async Task SwitchOutPartitionAsync_RebuildsUxIndex_AfterSwitch()
public async Task SwitchOutPartitionAsync_LeavesNoNonAlignedUniqueIndex_AfterSwitch()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
@@ -680,12 +839,23 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
// the fixture's MSSQL database) don't tread on each other.
await repo.SwitchOutPartitionAsync(new DateTime(2026, 4, 1, 0, 0, 0, DateTimeKind.Utc));
// WP2.2: the switch must NOT recreate UX_AuditLog_EventId. Re-adding it
// would reinstate the offline drop/rebuild the purge was freed from and
// block the next SWITCH until something dropped it again.
await using var verifyContext = CreateContext();
var indexExists = await ScalarAsync<int>(
verifyContext,
"SELECT COUNT(*) FROM sys.indexes " +
"WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog');");
Assert.Equal(1, indexExists);
Assert.Equal(0, indexExists);
// The aligned clustered PK is what enforces uniqueness now, and the
// switch must leave it untouched.
var pkExists = await ScalarAsync<int>(
verifyContext,
"SELECT COUNT(*) FROM sys.indexes " +
"WHERE name = 'PK_AuditLog' AND object_id = OBJECT_ID('dbo.AuditLog') AND is_primary_key = 1;");
Assert.Equal(1, pkExists);
}
[SkippableFact]
@@ -705,10 +875,11 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
// Switch out the June 2026 partition (different month, empty).
await repo.SwitchOutPartitionAsync(new DateTime(2026, 6, 1, 0, 0, 0, DateTimeKind.Utc));
// Re-attempting the same EventId after the switch must STILL be a no-op
// (UX_AuditLog_EventId is the index that enables idempotency; if the
// rebuild left it broken, this insert would silently produce a duplicate
// row and the count assertion below would catch it).
// Re-attempting the same EventId after the switch must STILL be a no-op.
// The idempotency probe now seeks the aligned clustered PK's leading
// column instead of a dedicated unique index; if the switch had disturbed
// that key, this insert would silently produce a duplicate row and the
// count assertion below would catch it.
// C3 (Task 2.5): rebuild a sibling row with the same EventId via the factory
// (ErrorMessage rides in DetailsJson, so a top-level `with` no longer applies).
var dup = ScadaBridgeAuditEventFactory.Create(
@@ -734,7 +905,7 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
}
[SkippableFact]
public async Task SwitchOutPartitionAsync_PartialFailure_RebuildsUxIndex_RaisesException()
public async Task SwitchOutPartitionAsync_PartialFailure_RaisesException_LeavesNoOrphanedStaging()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
@@ -745,8 +916,8 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
// ALTER TABLE … SWITCH refuses to move rows out of a partition that's
// referenced by an FK from another table, raising msg 4928
// ("ALTER TABLE SWITCH statement failed because target table … has a
// foreign key …"). The CATCH branch then rolls back and rebuilds the
// unique index — which the assertion below verifies.
// foreign key …"). The CATCH branch then rolls back and drops the staging
// table — which the assertion below verifies.
//
// The probe table is uniquely named with a guid suffix so reruns of
// this test inside the same fixture DB never collide. We clean it up
@@ -786,14 +957,23 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
await cmd.ExecuteNonQueryAsync();
}
// The CATCH block in the production SQL guarantees UX_AuditLog_EventId
// is rebuilt regardless of which step failed inside the TRY.
// The CATCH block drops the GUID-suffixed staging table regardless of
// which step failed inside the TRY, so a failed purge leaves no orphaned
// AuditLog_Staging_* object behind for the next tick to trip over.
await using var verifyContext = CreateContext();
var orphanedStaging = await ScalarAsync<int>(
verifyContext,
"SELECT COUNT(*) FROM sys.tables WHERE name LIKE 'AuditLog\\_Staging\\_%' ESCAPE '\\';");
Assert.Equal(0, orphanedStaging);
// And it must NOT have (re)created the non-aligned unique index: WP2.2
// deleted that rebuild entirely, so a failed switch cannot reintroduce
// the object that blocks the NEXT switch.
var indexExists = await ScalarAsync<int>(
verifyContext,
"SELECT COUNT(*) FROM sys.indexes " +
"WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog');");
Assert.Equal(1, indexExists);
Assert.Equal(0, indexExists);
}
// ------------------------------------------------------------------------
@@ -325,6 +325,57 @@ public class KpiHistoryRepositoryTests
Assert.Equal(2, row.SampleCount);
}
[Fact]
public async Task FoldHourlyRollupsAsync_PreloadsExistingRollups_WithoutAPerSeriesHourProbe()
{
// WP2.2: the fold used to issue one FirstOrDefaultAsync existence probe per
// (series, hour) group before writing anything — an N+1 that scaled with
// the metric catalogue times the lookback window. It now preloads the whole
// window in a single query and resolves each group from a dictionary.
//
// Six distinct series across two hours = twelve groups, so the old shape
// issued twelve SELECTs against KpiRollupHourly. The new shape issues one
// (plus the KpiSample read). This asserts the count stays small and
// constant rather than tracking the group count.
var counter = new RollupSelectCountingInterceptor();
await using var ctx = SqliteTestHelper.CreateInMemoryContext(counter);
var repo = new KpiHistoryRepository(ctx);
var samples = new List<KpiSample>();
for (var series = 0; series < 6; series++)
{
foreach (var hourOffset in new[] { 0, 1 })
{
samples.Add(Sample(
"NotificationOutbox",
"metric" + series,
"Global",
null,
value: series + hourOffset,
capturedAtUtc: Base.AddHours(hourOffset).AddMinutes(10)));
}
}
await repo.RecordSamplesAsync(samples);
// Seed the rollups so the SECOND fold takes the re-fold (update) path —
// the branch that most needed the per-group probe.
await repo.FoldHourlyRollupsAsync(Base, Base.AddHours(2));
counter.Reset();
await repo.FoldHourlyRollupsAsync(Base, Base.AddHours(2));
Assert.True(
counter.RollupSelectCount <= 1,
$"expected the fold to preload existing rollups in a single query; it issued {counter.RollupSelectCount} SELECTs against KpiRollupHourly");
// And the fold is still correct: twelve series-hours, values unchanged by
// the re-fold.
var rollups = await ctx.KpiRollupHourly.AsNoTracking().ToListAsync();
Assert.Equal(12, rollups.Count);
Assert.All(rollups, r => Assert.Equal(1, r.SampleCount));
}
[Fact]
public async Task GetHourlySeriesAsync_ReturnsAscending_AndHonorsNullVsSiteScopeKey()
{
@@ -540,6 +591,42 @@ public class KpiHistoryRepositoryTests
/// async non-query entry points (<c>ExecuteDeleteAsync</c> routes through the
/// async path).
/// </summary>
/// <summary>
/// Counts reader commands that SELECT from <c>KpiRollupHourly</c>, so a test
/// can prove the hourly fold resolves existing rows from a single preload
/// rather than one probe per (series, hour) group.
/// </summary>
private sealed class RollupSelectCountingInterceptor : DbCommandInterceptor
{
public int RollupSelectCount { get; private set; }
public void Reset() => RollupSelectCount = 0;
private void CountIfRollupSelect(DbCommand command)
{
if (command.CommandText.Contains("KpiRollupHourly", StringComparison.OrdinalIgnoreCase)
&& command.CommandText.Contains("SELECT", StringComparison.OrdinalIgnoreCase))
{
RollupSelectCount++;
}
}
public override InterceptionResult<DbDataReader> ReaderExecuting(
DbCommand command, CommandEventData eventData, InterceptionResult<DbDataReader> result)
{
CountIfRollupSelect(command);
return base.ReaderExecuting(command, eventData, result);
}
public override ValueTask<InterceptionResult<DbDataReader>> ReaderExecutingAsync(
DbCommand command, CommandEventData eventData, InterceptionResult<DbDataReader> result,
CancellationToken cancellationToken = default)
{
CountIfRollupSelect(command);
return base.ReaderExecutingAsync(command, eventData, result, cancellationToken);
}
}
private sealed class DeleteCountingInterceptor : DbCommandInterceptor
{
public int DeleteCount { get; private set; }
@@ -96,6 +96,47 @@ public class SiteCallAuditRepositoryTests : IClassFixture<MsSqlMigrationFixture>
Assert.Equal(attemptedSnapshot!.UpdatedAtUtc, afterStale.UpdatedAtUtc);
}
[SkippableFact]
public async Task UpsertAsync_RejectedMonotonicUpdate_DoesNotFallThroughToInsert()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// WP2.2 collapsed the two-round-trip "insert-if-absent then monotonic
// update" into ONE batch that updates first and inserts only when nothing
// matched. That makes @@ROWCOUNT = 0 ambiguous: it means "no such row"
// AND "the monotonic guard rejected this packet". Guarding the insert on
// @@ROWCOUNT alone would therefore let every stale/regressive packet
// append a SECOND row for an id that already exists — silently forking
// the mirror. The re-check for the row's existence is what prevents that,
// and this pins it: after a rejected regressive upsert there is still
// exactly ONE row, still carrying the advanced state.
var id = TrackedOperationId.New();
await using var context = CreateContext();
var repo = new SiteCallAuditRepository(context);
await repo.UpsertAsync(NewRow(id, status: "Delivered", retryCount: 3, lastError: null));
// Every flavour of rejection: lower rank, equal terminal rank, and an
// equal non-terminal rank with a stale timestamp.
await repo.UpsertAsync(NewRow(id, status: "Submitted", retryCount: 0));
await repo.UpsertAsync(NewRow(id, status: "Parked", retryCount: 9, lastError: "should-not-apply"));
await repo.UpsertAsync(NewRow(
id,
status: "Delivered",
retryCount: 99,
updatedAtUtc: new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc)));
await using var readContext = CreateContext();
var loaded = await readContext.Set<SiteCall>()
.Where(s => s.TrackedOperationId == id)
.ToListAsync();
Assert.Single(loaded);
Assert.Equal("Delivered", loaded[0].Status);
Assert.Equal(3, loaded[0].RetryCount);
Assert.Null(loaded[0].LastError);
}
[SkippableFact]
public async Task UpsertAsync_SameStatus_EqualUpdatedAt_IsNoOp()
{
@@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging.Abstractions;
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
@@ -582,4 +583,122 @@ public class SiteCallAuditReconciliationTests : TestKit
Assert.Equal(siteId, evt.SiteId);
Assert.True(evt.Pinned, "a legacy site that ignores after_id must publish Pinned=true");
}
// ---------------------------------------------------------------------
// 9. WP2.2: the reconciliation drain runs OFF the mailbox, so ingest,
// query and KPI messages are answered while a long post-outage
// catch-up is still pulling.
// ---------------------------------------------------------------------
/// <summary>
/// Pull client whose first call blocks until released, simulating a
/// post-outage catch-up that takes far longer than the caller's Ask timeout.
/// </summary>
private sealed class BlockingPullClient : IPullSiteCallsClient
{
private readonly TaskCompletionSource _release =
new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _entered =
new(TaskCreationOptions.RunContinuationsAsynchronously);
/// <summary>Completes once the drain has actually started pulling.</summary>
public Task Entered => _entered.Task;
public void Release() => _release.TrySetResult();
public async Task<PullSiteCallsResponse> PullAsync(
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
{
_entered.TrySetResult();
await _release.Task.ConfigureAwait(false);
return new PullSiteCallsResponse(Array.Empty<SiteCall>(), MoreAvailable: false);
}
}
[Fact]
public async Task ReconciliationDrain_InFlight_DoesNotBlockIngestUpsert()
{
// The drain used to run inside ReceiveAsync, which occupies the actor for
// its whole duration. A post-outage catch-up (every site, many paged
// network pulls, one upsert per row) therefore parked telemetry ingest,
// UI queries and KPI Asks behind it — and those callers timed out rather
// than queued, so a slow site could make central look dead. This pins the
// fix: with the drain off-mailbox behind a single-flight guard, an ingest
// Ask completes promptly while the pull is still blocked.
var siteId = "siteSlow";
var sites = new StaticEnumerator(new SiteEntry(siteId, "http://siteSlow:8083"));
var client = new BlockingPullClient();
var repo = new RecordingRepo();
var actor = CreateActor(sites, client, repo, FastTickOptions());
// Wait until the drain is genuinely in flight and blocked inside PullAsync.
await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
// The mailbox must still be serving. A generous-but-finite budget: this
// fails at the pre-fix behaviour (the reply only arrives once the pull
// unblocks, which never happens until Release below).
var id = TrackedOperationId.New();
var reply = await actor.Ask<UpsertSiteCallReply>(
new UpsertSiteCallCommand(NewRow(id, sourceSite: siteId)),
TimeSpan.FromSeconds(3));
Assert.True(reply.Accepted);
Assert.Equal(id, reply.TrackedOperationId);
// Let the drain finish so the actor shuts down cleanly.
client.Release();
}
[Fact]
public async Task ReconciliationTicks_DoNotOverlap_WhileADrainIsInFlight()
{
// Single-flight guard: with a 100 ms tick and a drain blocked for far
// longer, every subsequent tick must be dropped rather than starting a
// second concurrent pass — overlapping passes would race on the per-site
// cursor and pinned-latch dictionaries the drain mutates.
var siteId = "siteSlow";
var sites = new StaticEnumerator(new SiteEntry(siteId, "http://siteSlow:8083"));
var client = new CountingBlockingPullClient();
var repo = new RecordingRepo();
CreateActor(sites, client, repo, FastTickOptions());
await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
// Several tick intervals elapse while the first pass is still blocked.
await Task.Delay(TimeSpan.FromMilliseconds(600));
Assert.Equal(1, client.CallCount);
client.Release();
}
/// <summary>
/// <see cref="BlockingPullClient"/> that also counts invocations, so a test
/// can prove no second pass started while the first was blocked.
/// </summary>
private sealed class CountingBlockingPullClient : IPullSiteCallsClient
{
private readonly TaskCompletionSource _release =
new(TaskCreationOptions.RunContinuationsAsynchronously);
private readonly TaskCompletionSource _entered =
new(TaskCreationOptions.RunContinuationsAsynchronously);
private int _callCount;
public Task Entered => _entered.Task;
public int CallCount => Volatile.Read(ref _callCount);
public void Release() => _release.TrySetResult();
public async Task<PullSiteCallsResponse> PullAsync(
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
{
Interlocked.Increment(ref _callCount);
_entered.TrySetResult();
await _release.Task.ConfigureAwait(false);
return new PullSiteCallsResponse(Array.Empty<SiteCall>(), MoreAvailable: false);
}
}
}