Merge branch 'worktree-agent-a3b474b485c0288de' into arch-review-remediation
This commit is contained in:
@@ -78,13 +78,15 @@ builder.Property(e => e.ExecutionId)
|
|||||||
"CAST(JSON_VALUE(DetailsJson,'$.executionId') AS uniqueidentifier)", stored: true)
|
"CAST(JSON_VALUE(DetailsJson,'$.executionId') AS uniqueidentifier)", stored: true)
|
||||||
.ValueGeneratedOnAddOrUpdate();
|
.ValueGeneratedOnAddOrUpdate();
|
||||||
|
|
||||||
// Composite PK includes OccurredAtUtc for partition alignment
|
// Composite PK includes OccurredAtUtc for partition alignment — and is the ONLY
|
||||||
|
// EventId uniqueness enforcement (WP2.2 / AlignAuditLogEventIdUniqueness). EventId
|
||||||
|
// is a GUID minted once at the site alongside OccurredAtUtc, so a given EventId
|
||||||
|
// always lands in one partition and pair-uniqueness is EventId-uniqueness.
|
||||||
builder.HasKey(e => new { e.EventId, e.OccurredAtUtc });
|
builder.HasKey(e => new { e.EventId, e.OccurredAtUtc });
|
||||||
|
|
||||||
builder.HasIndex(e => e.EventId).IsUnique()
|
|
||||||
.HasDatabaseName("UX_AuditLog_EventId");
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The predecessor `UX_AuditLog_EventId` — a single-column unique index on `[PRIMARY]`, deliberately NOT partition-aligned — was dropped: a non-aligned index blocks `ALTER TABLE … SWITCH PARTITION`, so the retention purge had to drop it, switch, and rebuild it offline inside the switch transaction on every run.
|
||||||
|
|
||||||
**`TemplateConfiguration`** (representative of the domain-area configs) sets up the self-referencing parent FK, folder FK, cascade-delete relationships to attributes/alarms/scripts/compositions/native alarm sources, and the filtered unique index that enforces name uniqueness only on non-derived (base) templates.
|
**`TemplateConfiguration`** (representative of the domain-area configs) sets up the self-referencing parent FK, folder FK, cascade-delete relationships to attributes/alarms/scripts/compositions/native alarm sources, and the filtered unique index that enforces name uniqueness only on non-derived (base) templates.
|
||||||
|
|
||||||
**`SiteCallEntityTypeConfiguration`** maps `SiteCall` to `dbo.SiteCalls` with a `TrackedOperationId` PK stored as `varchar(36)` (GUID in `"D"` format) so the column shape matches the wire format and the site SQLite store — one consistent format for operational debugging.
|
**`SiteCallEntityTypeConfiguration`** maps `SiteCall` to `dbo.SiteCalls` with a `TrackedOperationId` PK stored as `varchar(36)` (GUID in `"D"` format) so the column shape matches the wire format and the site SQLite store — one consistent format for operational debugging.
|
||||||
@@ -108,11 +110,13 @@ VALUES
|
|||||||
|
|
||||||
`FormattableString` interpolation parameterises every value so there is no injection surface. SQL error numbers `2601` and `2627` (unique-index violation) are swallowed as no-ops because the IF NOT EXISTS check has a race window; both the check-loser and the retrying telemetry path are semantically correct duplicates.
|
`FormattableString` interpolation parameterises every value so there is no injection surface. SQL error numbers `2601` and `2627` (unique-index violation) are swallowed as no-ops because the IF NOT EXISTS check has a race window; both the check-loser and the retrying telemetry path are semantically correct duplicates.
|
||||||
|
|
||||||
|
`InsertManyIfNotExistsAsync` is the set-based form the ingest hot path uses: one `INSERT … SELECT … FROM (VALUES …) WHERE NOT EXISTS` per chunk of 100 rows (ten bound parameters per row against SQL Server's 2,100-parameter ceiling), so a telemetry packet costs one round trip instead of one per event. Because the anti-semi-join only sees committed rows, the packet is de-duplicated by `EventId` in C# first (first-write-wins, matching the single-row contract); a duplicate-key fault from a concurrent writer falls back to the per-row path so the batch is never a correctness dependency. It uses raw ADO.NET with explicitly typed `SqlParameter`s — the `VALUES` constructor derives its column types from the first row's parameters, so an untyped null would give the derived column the wrong type — and enlists in the DbContext's ambient transaction when one is open (the cached-telemetry dual-write).
|
||||||
|
|
||||||
`QueryAsync` builds LINQ predicates over `AuditLogRow` using `AsNoTracking()`, translating filter dimensions (`Channels`, `Kinds`, `Statuses`, `SourceSiteIds`, `SourceNodes`, `ExecutionId`, `ParentExecutionId`, time range) to server-side SQL IN/equality predicates and using keyset pagination on `(OccurredAtUtc DESC, EventId DESC)`.
|
`QueryAsync` builds LINQ predicates over `AuditLogRow` using `AsNoTracking()`, translating filter dimensions (`Channels`, `Kinds`, `Statuses`, `SourceSiteIds`, `SourceNodes`, `ExecutionId`, `ParentExecutionId`, time range) to server-side SQL IN/equality predicates and using keyset pagination on `(OccurredAtUtc DESC, EventId DESC)`.
|
||||||
|
|
||||||
`GetExecutionTreeAsync` walks the `ParentExecutionId` graph in two phases: a loop climbs to the root (bounded at 32 levels), then a recursive CTE descends the full tree and LEFT JOINs back to `AuditLog` so stub nodes (purged or row-less executions) still appear with `RowCount = 0`.
|
`GetExecutionTreeAsync` walks the `ParentExecutionId` graph in two phases: a loop climbs to the root (bounded at 32 levels), then a recursive CTE descends the full tree and LEFT JOINs back to `AuditLog` so stub nodes (purged or row-less executions) still appear with `RowCount = 0`.
|
||||||
|
|
||||||
`SwitchOutPartitionAsync` executes a drop-and-rebuild dance — dropping `UX_AuditLog_EventId`, creating a byte-identical staging table (including the computed-column definitions), switching the target partition to staging, dropping staging, and rebuilding the unique index — all inside a single `BEGIN TRY / BEGIN CATCH` block that guarantees the index is present whether the switch succeeds or rolls back.
|
`SwitchOutPartitionAsync` creates a byte-identical staging table (including the computed-column definitions), switches the target partition to staging, and drops staging, inside a single `BEGIN TRY / BEGIN CATCH` block whose CATCH cleans up the staging table on any failure. No index is dropped or rebuilt: uniqueness rides the partition-aligned clustered PK, so the switch is metadata-only. A guarded defensive `DROP INDEX UX_AuditLog_EventId` remains at the head of the batch purely so a database restored from a pre-alignment backup still purges.
|
||||||
|
|
||||||
### IAuditService — config-change audit
|
### IAuditService — config-change audit
|
||||||
|
|
||||||
@@ -241,7 +245,7 @@ The host is running in production mode and `GetPendingMigrationsAsync` found una
|
|||||||
|
|
||||||
### AuditLog partition switch fails mid-operation
|
### AuditLog partition switch fails mid-operation
|
||||||
|
|
||||||
`SwitchOutPartitionAsync` wraps the drop-and-rebuild dance in `BEGIN TRY / BEGIN CATCH`. On failure the CATCH block drops the staging table if it exists and rebuilds `UX_AuditLog_EventId` if it was dropped before the failure. The original exception is re-thrown so the Audit Log purge actor logs it and retries on the next daily tick. Verify that the `scadabridge_audit_purger` role still holds `ALTER ON SCHEMA::dbo` if the operation fails with a permissions error.
|
`SwitchOutPartitionAsync` wraps the staging/switch batch in `BEGIN TRY / BEGIN CATCH`. On failure the CATCH block drops the staging table if it exists, so no orphaned `AuditLog_Staging_*` object is left behind. There is no index to repair — uniqueness lives on the clustered PK, which the switch never touches. The original exception is re-thrown so the Audit Log purge actor logs it and retries on the next daily tick. Verify that the `scadabridge_audit_purger` role still holds `ALTER ON SCHEMA::dbo` if the operation fails with a permissions error.
|
||||||
|
|
||||||
### Design-time `dotnet ef` tooling cannot find a connection string
|
### Design-time `dotnet ef` tooling cannot find a connection string
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
BEGIN TRANSACTION;
|
||||||
|
IF NOT EXISTS (
|
||||||
|
SELECT * FROM [__EFMigrationsHistory]
|
||||||
|
WHERE [MigrationId] = N'20260815004957_AlignAuditLogEventIdUniqueness'
|
||||||
|
)
|
||||||
|
BEGIN
|
||||||
|
|
||||||
|
IF EXISTS (SELECT 1 FROM sys.indexes
|
||||||
|
WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
|
||||||
|
DROP INDEX UX_AuditLog_EventId ON dbo.AuditLog;
|
||||||
|
END;
|
||||||
|
|
||||||
|
IF NOT EXISTS (
|
||||||
|
SELECT * FROM [__EFMigrationsHistory]
|
||||||
|
WHERE [MigrationId] = N'20260815004957_AlignAuditLogEventIdUniqueness'
|
||||||
|
)
|
||||||
|
BEGIN
|
||||||
|
INSERT INTO [__EFMigrationsHistory] ([MigrationId], [ProductVersion])
|
||||||
|
VALUES (N'20260815004957_AlignAuditLogEventIdUniqueness', N'10.0.7');
|
||||||
|
END;
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
|
GO
|
||||||
|
|
||||||
@@ -291,6 +291,27 @@ stay `Pending` for the next sweep. Cadence is short (default 5 s) when
|
|||||||
non-empty, longer (default 30 s) when idle; telemetry runs on a dedicated
|
non-empty, longer (default 30 s) when idle; telemetry runs on a dedicated
|
||||||
dispatcher.
|
dispatcher.
|
||||||
|
|
||||||
|
**Central-side ingest is set-based.** `AuditLogIngestActor` writes a whole
|
||||||
|
packet with ONE `InsertManyIfNotExistsAsync` statement rather than one
|
||||||
|
`IF NOT EXISTS … INSERT` round trip per event; the cached-telemetry dual-write
|
||||||
|
similarly runs the whole packet in ONE transaction (set-based audit insert plus
|
||||||
|
one single-statement `SiteCalls` upsert per entry) instead of a transaction per
|
||||||
|
entry. Idempotency is unchanged: duplicates that repeat *within* a packet
|
||||||
|
collapse first-write-wins before the statement is built, duplicates *across*
|
||||||
|
packets are eliminated by the anti-semi-join, and any failure falls back to the
|
||||||
|
per-row / per-entry path — so the documented invariant that one bad row cannot
|
||||||
|
sink the rest of the batch still holds, it is simply no longer paid for on the
|
||||||
|
healthy path.
|
||||||
|
|
||||||
|
**Timeout ladder.** The ingest budget is deliberately the smallest on the path:
|
||||||
|
the site's Ask and the central gRPC handler's Ask are both 30 s, the actor's
|
||||||
|
own database budget is 20 s and the per-statement SQL timeout is 15 s. Before
|
||||||
|
this the three were identical, so they expired at the same instant and the
|
||||||
|
caller learned nothing but "it took 30 s" — no partial ack, no way to tell a
|
||||||
|
slow database from a wedged singleton. With the inner budgets strictly smaller,
|
||||||
|
a slow batch is abandoned by the actor first and the accepted-so-far ids are
|
||||||
|
still replied while the outer Asks are still waiting.
|
||||||
|
|
||||||
### Reconciliation pull (self-healing for missed telemetry)
|
### Reconciliation pull (self-healing for missed telemetry)
|
||||||
|
|
||||||
A central `SiteAuditReconciliationActor` periodically (default 5 min per site)
|
A central `SiteAuditReconciliationActor` periodically (default 5 min per site)
|
||||||
@@ -490,12 +511,19 @@ MS SQL for direct-write events). Unredacted secrets never persist.
|
|||||||
silently failing retention job would otherwise be invisible until the table grew
|
silently failing retention job would otherwise be invisible until the table grew
|
||||||
unbounded. Per-boundary/per-channel error isolation is unchanged — a single failure
|
unbounded. Per-boundary/per-channel error isolation is unchanged — a single failure
|
||||||
still never abandons the rest of the tick.
|
still never abandons the rest of the tick.
|
||||||
- **Maintenance command timeout:** the switch-out drop-and-rebuild dance and each
|
- **Maintenance command timeout:** the switch-out staging batch and each
|
||||||
per-channel `DELETE TOP` batch run with an explicit command timeout
|
per-channel `DELETE TOP` batch run with an explicit command timeout
|
||||||
(`AuditLog:Purge:MaintenanceCommandTimeoutMinutes`, default 30, floor 1 min) rather
|
(`AuditLog:Purge:MaintenanceCommandTimeoutMinutes`, default 30, floor 1 min) rather
|
||||||
than the ~30 s ADO.NET default, which could abort the metadata-only SWITCH mid-dance
|
than the ~30 s ADO.NET default, which could abort the metadata-only SWITCH mid-batch
|
||||||
on a large or contended partition and leave the live table without
|
on a large or contended partition and leave an orphaned staging table for the next
|
||||||
`UX_AuditLog_EventId` until a later tick's CATCH branch rebuilt it.
|
tick's CATCH branch to clean up.
|
||||||
|
- **Partition-aligned uniqueness:** the switch no longer drops and rebuilds an index.
|
||||||
|
`EventId` uniqueness rides the clustered `PK_AuditLog (EventId, OccurredAtUtc)`,
|
||||||
|
which is aligned on `ps_AuditLog_Month(OccurredAtUtc)`, so `SWITCH PARTITION` has no
|
||||||
|
non-aligned unique index to object to. The predecessor `UX_AuditLog_EventId` forced
|
||||||
|
an offline whole-table index build inside the switch transaction — blocking every
|
||||||
|
audit writer for its duration — and left a window in which the idempotency-supporting
|
||||||
|
index did not exist at all. See migration `AlignAuditLogEventIdUniqueness`.
|
||||||
- **Per-channel retention overrides (M5.5 T3):** `AuditLog:PerChannelRetentionDays`
|
- **Per-channel retention overrides (M5.5 T3):** `AuditLog:PerChannelRetentionDays`
|
||||||
is a dictionary keyed by canonical channel name (`ApiOutbound`, `DbOutbound`,
|
is a dictionary keyed by canonical channel name (`ApiOutbound`, `DbOutbound`,
|
||||||
`Notification`, `ApiInbound`, `SecuredWrite` — all five `AuditChannel` values are
|
`Notification`, `ApiInbound`, `SecuredWrite` — all five `AuditChannel` values are
|
||||||
|
|||||||
@@ -95,9 +95,11 @@ The configuration database stores all central system data, organized by domain a
|
|||||||
- `ParentExecutionId` — `CAST(JSON_VALUE(DetailsJson,'$.parentExecutionId') AS uniqueidentifier)` PERSISTED — spawner's `ExecutionId`; null for top-level runs.
|
- `ParentExecutionId` — `CAST(JSON_VALUE(DetailsJson,'$.parentExecutionId') AS uniqueidentifier)` PERSISTED — spawner's `ExecutionId`; null for top-level runs.
|
||||||
- `IngestedAtUtc` — `CAST(SWITCHOFFSET(CAST(JSON_VALUE(DetailsJson,'$.ingestedAtUtc') AS datetimeoffset), 0) AS datetime2(7))` — central ingest timestamp; **not** persisted (SQL Server rejects PERSISTED on the non-deterministic `SWITCHOFFSET` expression).
|
- `IngestedAtUtc` — `CAST(SWITCHOFFSET(CAST(JSON_VALUE(DetailsJson,'$.ingestedAtUtc') AS datetimeoffset), 0) AS datetime2(7))` — central ingest timestamp; **not** persisted (SQL Server rejects PERSISTED on the non-deterministic `SWITCHOFFSET` expression).
|
||||||
|
|
||||||
*Clustered primary key:* `(EventId, OccurredAtUtc)` — composite so the key is partition-aligned. `UX_AuditLog_EventId` (unique, non-aligned on `[PRIMARY]`) enforces global `EventId` uniqueness for `InsertIfNotExistsAsync` idempotency.
|
*Clustered primary key:* `(EventId, OccurredAtUtc)` — composite so the key is partition-aligned, and the **sole** `EventId` uniqueness enforcement backing `InsertIfNotExistsAsync` idempotency. `EventId` is a GUID minted once at the emitting site in the same operation that stamps `OccurredAtUtc`, and neither field is ever re-stamped downstream, so a given `EventId` always arrives with the same `OccurredAtUtc` and can only map to one partition — pair-uniqueness is `EventId`-uniqueness for every row the system produces. The idempotency probe (`WHERE EventId = @id`) still seeks the clustered key's leading column, at the cost of one seek per partition rather than one seek overall.
|
||||||
|
|
||||||
*Indexes* (all non-clustered, partition-aligned on `ps_AuditLog_Month(OccurredAtUtc)` except `UX_AuditLog_EventId`):
|
A non-aligned single-column `UX_AuditLog_EventId` on `[PRIMARY]` used to carry that uniqueness; it was dropped by the `AlignAuditLogEventIdUniqueness` migration because a non-aligned index blocks `ALTER TABLE … SWITCH PARTITION`, forcing the retention purge to drop it, switch, and rebuild it **offline inside the switch transaction** on every run.
|
||||||
|
|
||||||
|
*Indexes* (all non-clustered, partition-aligned on `ps_AuditLog_Month(OccurredAtUtc)`):
|
||||||
- `IX_AuditLog_OccurredAtUtc` (primary time-range index for global scans)
|
- `IX_AuditLog_OccurredAtUtc` (primary time-range index for global scans)
|
||||||
- `IX_AuditLog_Site_Occurred (SourceSiteId, OccurredAtUtc)` (per-site filters)
|
- `IX_AuditLog_Site_Occurred (SourceSiteId, OccurredAtUtc)` (per-site filters)
|
||||||
- `IX_AuditLog_CorrelationId (CorrelationId) WHERE CorrelationId IS NOT NULL` (drilldown from a single operation)
|
- `IX_AuditLog_CorrelationId (CorrelationId) WHERE CorrelationId IS NOT NULL` (drilldown from a single operation)
|
||||||
|
|||||||
@@ -135,6 +135,28 @@ Pinned)` on the EventStream (transition-only, mirroring
|
|||||||
health-observable condition rather than a silent log line, and the latch clears
|
health-observable condition rather than a silent log line, and the latch clears
|
||||||
with `Pinned=false` once a later tick makes progress.
|
with `Pinned=false` once a later tick makes progress.
|
||||||
|
|
||||||
|
**The drain runs off the mailbox.** A reconciliation pass is unbounded work —
|
||||||
|
every site, up to a page ceiling of network pulls each, one upsert per row — so
|
||||||
|
it runs as a background task with a `PipeTo`-delivered completion message and a
|
||||||
|
single-flight guard, not as an actor message handler. A handler occupies the
|
||||||
|
actor for its whole duration, so a post-outage catch-up used to park telemetry
|
||||||
|
ingest, UI queries and KPI Asks behind it; those callers time out rather than
|
||||||
|
queue, which made a slow site look like a dead central. The single-flight guard
|
||||||
|
is raised and lowered ON the actor thread, so the per-site cursor and pinned-latch
|
||||||
|
dictionaries the pass mutates are still only ever touched by one task at a time,
|
||||||
|
and the mailbox supplies the memory barrier between consecutive passes. The
|
||||||
|
daily terminal-row purge uses the same shape. (`NotificationOutboxActor`'s
|
||||||
|
dispatch sweep is the in-repo reference.)
|
||||||
|
|
||||||
|
**The central upsert is one statement.** `SiteCallAuditRepository.UpsertAsync`
|
||||||
|
issues a single batch that runs the monotonic UPDATE first and INSERTs only when
|
||||||
|
nothing matched *and* the row genuinely does not exist — instead of an
|
||||||
|
unconditional insert-if-absent followed by the update, which cost two round trips
|
||||||
|
on every packet and wasted the insert half for every packet after the first. The
|
||||||
|
existence re-check is load-bearing: a zero row count also means "the monotonic
|
||||||
|
guard rejected this packet", and inserting there would fork the mirror with a
|
||||||
|
second row for an id that already exists.
|
||||||
|
|
||||||
## Retry / Discard Relay
|
## Retry / Discard Relay
|
||||||
|
|
||||||
Parked cached calls live in the owning site's S&F buffer. Operator Retry/Discard
|
Parked cached calls live in the owning site's S&F buffer. Operator Retry/Discard
|
||||||
|
|||||||
@@ -18,8 +18,9 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
|||||||
/// ingest timestamp into DetailsJson (there is no promoted IngestedAtUtc
|
/// ingest timestamp into DetailsJson (there is no promoted IngestedAtUtc
|
||||||
/// column — the value is a DetailsJson field set via
|
/// column — the value is a DetailsJson field set via
|
||||||
/// <see cref="AuditRowProjection.WithIngestedAtUtc"/>) and inserted idempotently
|
/// <see cref="AuditRowProjection.WithIngestedAtUtc"/>) and inserted idempotently
|
||||||
/// via <see cref="IAuditLogRepository.InsertIfNotExistsAsync"/> — duplicates are
|
/// via <see cref="IAuditLogRepository.InsertManyIfNotExistsAsync"/> — duplicates
|
||||||
/// silently swallowed (first-write-wins).
|
/// are silently swallowed (first-write-wins), whether they repeat inside one
|
||||||
|
/// packet or across packets.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
/// <para>
|
/// <para>
|
||||||
@@ -28,9 +29,11 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
|||||||
/// consistent and the site is free to flip its local row to <c>Forwarded</c>.
|
/// consistent and the site is free to flip its local row to <c>Forwarded</c>.
|
||||||
/// </para>
|
/// </para>
|
||||||
/// <para>
|
/// <para>
|
||||||
/// Audit-write failures must NEVER abort the user-facing action. The actor
|
/// Audit-write failures must NEVER abort the user-facing action. Each message
|
||||||
/// wraps each repository call in its own try/catch so a single bad row cannot
|
/// is written as ONE set-based statement (or one transaction, for the cached
|
||||||
/// cause the rest of the batch to be lost, and it guards scope/repository
|
/// dual-write); if that fails the actor retries the same work row-by-row inside
|
||||||
|
/// per-row try/catch, so a single bad row still cannot cause the rest of the
|
||||||
|
/// batch to be lost. It also guards scope/repository
|
||||||
/// resolution so a transient DI fault cannot restart the singleton — those
|
/// resolution so a transient DI fault cannot restart the singleton — those
|
||||||
/// catches are what keep this actor alive across handler throws, not the
|
/// catches are what keep this actor alive across handler throws, not the
|
||||||
/// supervisor strategy. The <see cref="SupervisorStrategy"/> override returns
|
/// supervisor strategy. The <see cref="SupervisorStrategy"/> override returns
|
||||||
@@ -50,6 +53,31 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
|||||||
/// </remarks>
|
/// </remarks>
|
||||||
public class AuditLogIngestActor : ReceiveActor
|
public class AuditLogIngestActor : ReceiveActor
|
||||||
{
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Overall budget for one ingest message's database work, deliberately
|
||||||
|
/// SHORTER than the gRPC Ask that wraps it.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// The path used to stack three identical 30 s budgets — the site's Ask
|
||||||
|
/// (<c>CommunicationOptions.NotificationForwardTimeout</c>), the central gRPC
|
||||||
|
/// handler's Ask (<c>SiteStreamGrpcServer.AuditIngestAskTimeout</c>) and the
|
||||||
|
/// ADO.NET command default — so they all expired at the same instant. The
|
||||||
|
/// caller therefore learned nothing except "it took 30 s": no partial ack, no
|
||||||
|
/// distinction between a slow database and a wedged singleton. Making the
|
||||||
|
/// innermost budget strictly smallest means a slow batch is abandoned by the
|
||||||
|
/// actor FIRST, with the accepted-so-far ids still replied, while the outer
|
||||||
|
/// Asks are still waiting.
|
||||||
|
/// </remarks>
|
||||||
|
internal static readonly TimeSpan IngestBudget = TimeSpan.FromSeconds(20);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Per-statement SQL timeout for the ingest write, strictly inside
|
||||||
|
/// <see cref="IngestBudget"/> so a single wedged statement surfaces as a SQL
|
||||||
|
/// timeout the per-batch catch can log, rather than consuming the whole
|
||||||
|
/// actor-level budget and starving the rest of the batch.
|
||||||
|
/// </summary>
|
||||||
|
internal static readonly TimeSpan IngestSqlCommandTimeout = TimeSpan.FromSeconds(15);
|
||||||
|
|
||||||
private readonly IServiceProvider? _serviceProvider;
|
private readonly IServiceProvider? _serviceProvider;
|
||||||
private readonly IAuditLogRepository? _injectedRepository;
|
private readonly IAuditLogRepository? _injectedRepository;
|
||||||
private readonly ILogger<AuditLogIngestActor> _logger;
|
private readonly ILogger<AuditLogIngestActor> _logger;
|
||||||
@@ -185,39 +213,75 @@ public class AuditLogIngestActor : ReceiveActor
|
|||||||
DateTime nowUtc,
|
DateTime nowUtc,
|
||||||
List<Guid> accepted)
|
List<Guid> accepted)
|
||||||
{
|
{
|
||||||
|
// Stamp the ingest timestamp here, not at the site. Redact BEFORE the
|
||||||
|
// IngestedAtUtc stamp so the redacted copy carries the central-side
|
||||||
|
// ingest timestamp. The redactor is contract-bound to never throw; a null
|
||||||
|
// redactor (test composition root, no IAuditRedactor registered) falls
|
||||||
|
// back to the SafeDefault rather than pass-through, so HTTP header
|
||||||
|
// redaction always runs. IngestedAtUtc is a DetailsJson field on the
|
||||||
|
// canonical record, so stamp it via the projection helper.
|
||||||
|
var safeRedactor = redactor ?? SafeDefaultAuditRedactor.Instance;
|
||||||
|
var projected = new List<AuditEvent>(cmd.Events.Count);
|
||||||
foreach (var evt in cmd.Events)
|
foreach (var evt in cmd.Events)
|
||||||
{
|
{
|
||||||
|
var filtered = safeRedactor.Apply(evt);
|
||||||
|
projected.Add(AuditRowProjection.WithIngestedAtUtc(filtered, nowUtc));
|
||||||
|
}
|
||||||
|
|
||||||
|
// ONE set-based statement for the whole packet (WP2.2) instead of one
|
||||||
|
// IF NOT EXISTS … INSERT round trip per event. A telemetry packet of 200
|
||||||
|
// rows used to cost 200 sequential round trips on the central singleton's
|
||||||
|
// dispatcher; it now costs two. Idempotency is unchanged — duplicates
|
||||||
|
// within the packet collapse first-write-wins, duplicates against
|
||||||
|
// committed rows are eliminated by the anti-semi-join, and a concurrent
|
||||||
|
// writer degrades to the repository's per-row fallback.
|
||||||
|
using var budget = new CancellationTokenSource(IngestBudget);
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
// Stamp the ingest timestamp here, not at the site. The
|
await repository
|
||||||
// repository's duplicate-key hardening already swallows
|
.InsertManyIfNotExistsAsync(projected, IngestSqlCommandTimeout, budget.Token)
|
||||||
// duplicate-key races, so the same id arriving twice (site
|
.ConfigureAwait(false);
|
||||||
// retry, reconciliation) is a silent no-op.
|
|
||||||
// Redact BEFORE the IngestedAtUtc stamp so the redacted
|
// A batch that returned without throwing means every row is now
|
||||||
// copy carries the central-side ingest timestamp. The redactor
|
// present — inserted here or already committed by an earlier delivery.
|
||||||
// is contract-bound to never throw. A null
|
// Both count as accepted: the storage state is consistent and the site
|
||||||
// redactor (test composition root, no IAuditRedactor
|
// is free to flip its local rows to Forwarded.
|
||||||
// registered) now falls back to the SafeDefault rather than
|
foreach (var evt in cmd.Events)
|
||||||
// pass-through, so HTTP header redaction always runs.
|
{
|
||||||
// IngestedAtUtc is a DetailsJson field on
|
|
||||||
// the canonical record, so stamp it via the projection helper.
|
|
||||||
var safeRedactor = redactor ?? SafeDefaultAuditRedactor.Instance;
|
|
||||||
var filtered = safeRedactor.Apply(evt);
|
|
||||||
var ingested = AuditRowProjection.WithIngestedAtUtc(filtered, nowUtc);
|
|
||||||
await repository.InsertIfNotExistsAsync(ingested).ConfigureAwait(false);
|
|
||||||
accepted.Add(evt.EventId);
|
accepted.Add(evt.EventId);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
// The batch is a throughput optimisation, NOT a change to the
|
||||||
|
// failure grain. The documented invariant — "a single bad row cannot
|
||||||
|
// cause the rest of the batch to be lost" — is preserved by falling
|
||||||
|
// back to the per-row path here, so a poison row still costs only
|
||||||
|
// itself. Re-running rows the failed batch may already have committed
|
||||||
|
// is safe: every insert is idempotent on EventId.
|
||||||
|
_logger.LogWarning(ex,
|
||||||
|
"Set-based ingest of {Count} audit event(s) failed; falling back to per-row inserts so one bad row does not sink the batch.",
|
||||||
|
cmd.Events.Count);
|
||||||
|
|
||||||
|
for (var i = 0; i < projected.Count; i++)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await repository.InsertIfNotExistsAsync(projected[i], budget.Token).ConfigureAwait(false);
|
||||||
|
accepted.Add(cmd.Events[i].EventId);
|
||||||
|
}
|
||||||
|
catch (Exception rowEx)
|
||||||
{
|
{
|
||||||
// Per-row catch — one bad row never sinks the whole batch.
|
// Per-row catch — one bad row never sinks the whole batch.
|
||||||
// The row stays Pending at the site; the next drain retries.
|
// The row stays Pending at the site; the next drain retries.
|
||||||
// Bump the central health counter so a
|
// Bump the central health counter so a sustained insert-throw
|
||||||
// sustained insert-throw failure surfaces on the dashboard.
|
// failure surfaces on the dashboard.
|
||||||
try { failureCounter?.Increment(); }
|
try { failureCounter?.Increment(); }
|
||||||
catch { /* counter must never throw — defence in depth */ }
|
catch { /* counter must never throw — defence in depth */ }
|
||||||
_logger.LogError(ex,
|
_logger.LogError(rowEx,
|
||||||
"Failed to persist audit event {EventId} during batch ingest; row will be retried by the site.",
|
"Failed to persist audit event {EventId} during batch ingest; row will be retried by the site.",
|
||||||
evt.EventId);
|
cmd.Events[i].EventId);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -262,6 +326,24 @@ public class AuditLogIngestActor : ReceiveActor
|
|||||||
|
|
||||||
var strategy = dbContext.Database.CreateExecutionStrategy();
|
var strategy = dbContext.Database.CreateExecutionStrategy();
|
||||||
|
|
||||||
|
// Fast path (WP2.2): the WHOLE packet in ONE transaction — one
|
||||||
|
// set-based audit insert plus one single-statement upsert per entry.
|
||||||
|
// The predecessor opened a transaction PER entry and issued three
|
||||||
|
// statements inside it (IF NOT EXISTS insert, then the two-statement
|
||||||
|
// SiteCalls upsert), so a 50-entry packet cost ~200 round trips and 50
|
||||||
|
// commits. If anything faults, the per-entry loop below re-runs the
|
||||||
|
// packet with its original per-entry isolation, so the documented
|
||||||
|
// "one entry's failure does not abort the others" invariant survives —
|
||||||
|
// it is simply no longer paid for on the healthy path.
|
||||||
|
using var budget = new CancellationTokenSource(IngestBudget);
|
||||||
|
if (await TryIngestCachedBatchAsync(
|
||||||
|
strategy, dbContext, auditRepo, siteCallRepo, redactor, cmd, accepted, budget.Token)
|
||||||
|
.ConfigureAwait(false))
|
||||||
|
{
|
||||||
|
replyTo.Tell(new IngestCachedTelemetryReply(accepted));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
foreach (var entry in cmd.Entries)
|
foreach (var entry in cmd.Entries)
|
||||||
{
|
{
|
||||||
try
|
try
|
||||||
@@ -330,6 +412,90 @@ public class AuditLogIngestActor : ReceiveActor
|
|||||||
replyTo.Tell(new IngestCachedTelemetryReply(accepted));
|
replyTo.Tell(new IngestCachedTelemetryReply(accepted));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Attempts the whole cached-telemetry packet as ONE transaction: a single
|
||||||
|
/// set-based audit insert followed by one monotonic <c>SiteCalls</c> upsert
|
||||||
|
/// per entry. Returns <see langword="true"/> when it committed (and only then
|
||||||
|
/// appends to <paramref name="accepted"/>), <see langword="false"/> when the
|
||||||
|
/// caller should fall back to the per-entry transaction loop.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// The batch is all-or-nothing by construction — it is a single transaction —
|
||||||
|
/// which is why a failure MUST fall back rather than report partial success:
|
||||||
|
/// the per-entry loop is what turns one poison entry into one lost entry
|
||||||
|
/// instead of a lost packet. Nothing is appended to
|
||||||
|
/// <paramref name="accepted"/> until the commit returns, so a failed attempt
|
||||||
|
/// leaves the caller's list untouched and the retry starts from a clean slate.
|
||||||
|
/// </remarks>
|
||||||
|
private async Task<bool> TryIngestCachedBatchAsync(
|
||||||
|
Microsoft.EntityFrameworkCore.Storage.IExecutionStrategy strategy,
|
||||||
|
ScadaBridgeDbContext dbContext,
|
||||||
|
IAuditLogRepository auditRepo,
|
||||||
|
ISiteCallAuditRepository siteCallRepo,
|
||||||
|
IAuditRedactor? redactor,
|
||||||
|
IngestCachedTelemetryCommand cmd,
|
||||||
|
List<Guid> accepted,
|
||||||
|
CancellationToken ct)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await strategy.ExecuteAsync(async () =>
|
||||||
|
{
|
||||||
|
await using var tx = await dbContext.Database
|
||||||
|
.BeginTransactionAsync(ct)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
|
||||||
|
// One central-side instant for the whole packet so a join across
|
||||||
|
// the two tables sees matching timestamps (debugging convenience,
|
||||||
|
// not a correctness invariant).
|
||||||
|
var ingestedAt = DateTime.UtcNow;
|
||||||
|
var safeRedactor = redactor ?? SafeDefaultAuditRedactor.Instance;
|
||||||
|
|
||||||
|
var auditRows = new List<AuditEvent>(cmd.Entries.Count);
|
||||||
|
foreach (var entry in cmd.Entries)
|
||||||
|
{
|
||||||
|
// Only the AuditLog row's payload columns are redactable;
|
||||||
|
// SiteCalls carries operational state only (status, retry
|
||||||
|
// count) and is left untouched.
|
||||||
|
var filteredAudit = safeRedactor.Apply(entry.Audit);
|
||||||
|
auditRows.Add(AuditRowProjection.WithIngestedAtUtc(filteredAudit, ingestedAt));
|
||||||
|
}
|
||||||
|
|
||||||
|
await auditRepo
|
||||||
|
.InsertManyIfNotExistsAsync(auditRows, IngestSqlCommandTimeout, ct)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
|
||||||
|
foreach (var entry in cmd.Entries)
|
||||||
|
{
|
||||||
|
await siteCallRepo
|
||||||
|
.UpsertAsync(entry.SiteCall with { IngestedAtUtc = ingestedAt }, ct)
|
||||||
|
.ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
await tx.CommitAsync(ct).ConfigureAwait(false);
|
||||||
|
}).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (Exception ex)
|
||||||
|
{
|
||||||
|
// No health-counter bump here — the per-entry retry is the authority
|
||||||
|
// on whether this packet genuinely failed, and double-counting a
|
||||||
|
// batch that the fallback then writes successfully would make the
|
||||||
|
// dashboard read as a sustained fault during normal contention.
|
||||||
|
_logger.LogWarning(
|
||||||
|
ex,
|
||||||
|
"Batched cached-telemetry dual-write of {Count} entr(ies) failed; falling back to per-entry transactions.",
|
||||||
|
cmd.Entries.Count);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach (var entry in cmd.Entries)
|
||||||
|
{
|
||||||
|
accepted.Add(entry.Audit.EventId);
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Fallback handler installed on the single-repository test ctor — that
|
/// Fallback handler installed on the single-repository test ctor — that
|
||||||
/// ctor has no DbContext and no <see cref="ISiteCallAuditRepository"/>, so
|
/// ctor has no DbContext and no <see cref="ISiteCallAuditRepository"/>, so
|
||||||
|
|||||||
@@ -17,8 +17,9 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
|||||||
/// for monthly boundaries whose latest <c>OccurredAtUtc</c> is older
|
/// for monthly boundaries whose latest <c>OccurredAtUtc</c> is older
|
||||||
/// than <c>DateTime.UtcNow - RetentionDays</c>.</item>
|
/// than <c>DateTime.UtcNow - RetentionDays</c>.</item>
|
||||||
/// <item>For each eligible boundary, calls
|
/// <item>For each eligible boundary, calls
|
||||||
/// <see cref="IAuditLogRepository.SwitchOutPartitionAsync"/> which runs
|
/// <see cref="IAuditLogRepository.SwitchOutPartitionAsync"/>, a
|
||||||
/// the drop-and-rebuild dance around <c>UX_AuditLog_EventId</c>.</item>
|
/// metadata-only staging-table switch (WP2.2 removed the index
|
||||||
|
/// drop/rebuild that used to bracket it).</item>
|
||||||
/// <item>Publishes <see cref="AuditLogPurgedEvent"/> on the actor-system
|
/// <item>Publishes <see cref="AuditLogPurgedEvent"/> on the actor-system
|
||||||
/// EventStream so the central health collector + ops surfaces
|
/// EventStream so the central health collector + ops surfaces
|
||||||
/// can subscribe without coupling to this actor.</item>
|
/// can subscribe without coupling to this actor.</item>
|
||||||
@@ -26,11 +27,10 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
/// <para>
|
/// <para>
|
||||||
/// <b>Daily cadence.</b> Partition switch is metadata-only but the
|
/// <b>Daily cadence.</b> The partition switch is metadata-only, but it still
|
||||||
/// drop-and-rebuild dance briefly removes <c>UX_AuditLog_EventId</c>; running
|
/// takes schema-modification locks on a table the ingest path is writing to;
|
||||||
/// more often than necessary trades unique-index rebuild outages for
|
/// running more often than necessary trades contention for negligible freshness
|
||||||
/// negligible freshness wins. The default 24-hour interval matches
|
/// wins. The default 24-hour interval matches alog.md §10's retention policy.
|
||||||
/// alog.md §10's retention policy.
|
|
||||||
/// </para>
|
/// </para>
|
||||||
/// <para>
|
/// <para>
|
||||||
/// <b>Continue-on-error.</b> A single boundary that throws (transient SQL
|
/// <b>Continue-on-error.</b> A single boundary that throws (transient SQL
|
||||||
|
|||||||
@@ -10,12 +10,12 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
/// <para>
|
/// <para>
|
||||||
/// The purge actor is a daily-cadence singleton, not a hot-loop, because
|
/// The purge actor is a daily-cadence singleton, not a hot-loop. The
|
||||||
/// partition-switch I/O is metadata-only but the drop-and-rebuild dance
|
/// partition switch itself is metadata-only (since WP2.2's
|
||||||
/// briefly removes the <c>UX_AuditLog_EventId</c> unique index — running
|
/// <c>AlignAuditLogEventIdUniqueness</c> it no longer drops and rebuilds a
|
||||||
/// more often than necessary trades index-rebuild outages for marginal
|
/// non-aligned unique index around it), but it still takes schema-modification
|
||||||
/// freshness gains. Lower this only when an operator can prove they need
|
/// locks and competes with ingest for the same table. Lower this only when an
|
||||||
/// sub-daily purge granularity.
|
/// operator can prove they need sub-daily purge granularity.
|
||||||
/// </para>
|
/// </para>
|
||||||
/// <para>
|
/// <para>
|
||||||
/// <see cref="IntervalOverride"/> exists for tests to drop the cadence to
|
/// <see cref="IntervalOverride"/> exists for tests to drop the cadence to
|
||||||
@@ -58,15 +58,14 @@ public sealed class AuditLogPurgeOptions
|
|||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Per-command timeout (in minutes) for the maintenance SQL the purge tick issues —
|
/// Per-command timeout (in minutes) for the maintenance SQL the purge tick issues —
|
||||||
/// both the partition switch-out drop-and-rebuild dance
|
/// both the partition switch-out staging batch
|
||||||
/// (<see cref="ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories.IAuditLogRepository.SwitchOutPartitionAsync"/>)
|
/// (<see cref="ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories.IAuditLogRepository.SwitchOutPartitionAsync"/>)
|
||||||
/// and each per-channel <c>DELETE TOP</c> batch. Default 30 minutes.
|
/// and each per-channel <c>DELETE TOP</c> batch. Default 30 minutes.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
/// The ADO.NET default command timeout is ~30 seconds. On a large or contended partition the
|
/// The ADO.NET default command timeout is ~30 seconds. On a large or contended partition the
|
||||||
/// SWITCH dance (which briefly drops <c>UX_AuditLog_EventId</c>) can exceed that and abort
|
/// SWITCH batch can exceed that and abort mid-flight, leaving an orphaned staging table for the
|
||||||
/// mid-flight — leaving the live table without its idempotency-supporting unique index until a
|
/// next tick's CATCH branch to clean up (arch-review 04, S2). A generous maintenance timeout
|
||||||
/// later tick's CATCH branch rebuilds it (arch-review 04, S2). A generous maintenance timeout
|
|
||||||
/// lets the metadata-only switch complete rather than self-locking. Resolved via
|
/// lets the metadata-only switch complete rather than self-locking. Resolved via
|
||||||
/// <see cref="ResolvedMaintenanceCommandTimeout"/>, clamped to a 1-minute floor.
|
/// <see cref="ResolvedMaintenanceCommandTimeout"/>, clamped to a 1-minute floor.
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
|
|||||||
@@ -34,6 +34,62 @@ public interface IAuditLogRepository
|
|||||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||||
Task InsertIfNotExistsAsync(AuditEvent evt, CancellationToken ct = default);
|
Task InsertIfNotExistsAsync(AuditEvent evt, CancellationToken ct = default);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Set-based form of <see cref="InsertIfNotExistsAsync"/>: inserts every
|
||||||
|
/// event in <paramref name="events"/> that does not already exist, in as few
|
||||||
|
/// round trips as the provider allows, and returns the number of rows
|
||||||
|
/// actually written. First-write-wins idempotency is unchanged — an EventId
|
||||||
|
/// already present is silently skipped.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// <para>
|
||||||
|
/// <b>Duplicate EventIds are tolerated both within and across packets.</b>
|
||||||
|
/// Duplicates inside one call collapse to the first occurrence before the
|
||||||
|
/// statement is built (an append-only row is immutable, so later copies of an
|
||||||
|
/// EventId carry the same content); duplicates against already-committed rows
|
||||||
|
/// are eliminated by the anti-semi-join. A concurrent writer that commits a
|
||||||
|
/// row mid-statement is handled by falling back to the per-row path, so the
|
||||||
|
/// batch is a throughput optimisation and never a correctness dependency.
|
||||||
|
/// </para>
|
||||||
|
/// <para>
|
||||||
|
/// <b>Default implementation.</b> The interface supplies a per-row loop so a
|
||||||
|
/// test double or an alternative store keeps working unchanged; the EF Core
|
||||||
|
/// implementation overrides it with a genuine set-based statement. Callers on
|
||||||
|
/// the ingest hot path should always prefer this method — the per-row form
|
||||||
|
/// cost one <c>IF NOT EXISTS … INSERT</c> round trip per audit event.
|
||||||
|
/// </para>
|
||||||
|
/// </remarks>
|
||||||
|
/// <param name="events">Audit events to insert; may contain duplicates.</param>
|
||||||
|
/// <param name="commandTimeout">
|
||||||
|
/// Optional per-statement timeout. The ingest actor passes a budget strictly
|
||||||
|
/// SHORTER than the gRPC Ask that wraps it, so the reply is produced before
|
||||||
|
/// the caller gives up rather than at the same instant.
|
||||||
|
/// </param>
|
||||||
|
/// <param name="ct">Cancellation token.</param>
|
||||||
|
/// <returns>A task that resolves to the number of rows inserted (duplicates excluded).</returns>
|
||||||
|
async Task<int> InsertManyIfNotExistsAsync(
|
||||||
|
IReadOnlyList<AuditEvent> events,
|
||||||
|
TimeSpan? commandTimeout = null,
|
||||||
|
CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(events);
|
||||||
|
|
||||||
|
var inserted = 0;
|
||||||
|
var seen = new HashSet<Guid>(events.Count);
|
||||||
|
foreach (var evt in events)
|
||||||
|
{
|
||||||
|
if (evt is null || !seen.Add(evt.EventId))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
await InsertIfNotExistsAsync(evt, ct).ConfigureAwait(false);
|
||||||
|
inserted++;
|
||||||
|
}
|
||||||
|
|
||||||
|
return inserted;
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Returns up to <see cref="AuditLogPaging.PageSize"/> rows matching
|
/// Returns up to <see cref="AuditLogPaging.PageSize"/> rows matching
|
||||||
/// <paramref name="filter"/>, ordered by <c>(OccurredAtUtc DESC, EventId DESC)</c>.
|
/// <paramref name="filter"/>, ordered by <c>(OccurredAtUtc DESC, EventId DESC)</c>.
|
||||||
@@ -60,36 +116,32 @@ public interface IAuditLogRepository
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
/// <para>
|
/// <para>
|
||||||
/// <b>Drop-and-rebuild dance.</b> <c>UX_AuditLog_EventId</c> is intentionally
|
/// <b>Partition-aligned uniqueness — no index drop.</b> EventId uniqueness is
|
||||||
/// non-partition-aligned (it lives on <c>[PRIMARY]</c> so single-column
|
/// enforced by the clustered <c>PK_AuditLog (EventId, OccurredAtUtc)</c>,
|
||||||
/// EventId uniqueness — required by <see cref="InsertIfNotExistsAsync"/> —
|
/// which is aligned on <c>ps_AuditLog_Month(OccurredAtUtc)</c>, so
|
||||||
/// can be enforced cheaply). SQL Server rejects
|
/// <c>ALTER TABLE … SWITCH PARTITION</c> has no non-aligned unique index to
|
||||||
/// <c>ALTER TABLE … SWITCH PARTITION</c> while a non-aligned unique index
|
/// object to. The implementation creates a staging table with byte-identical
|
||||||
/// is present, so the implementation drops the index, creates a staging
|
/// schema, switches the partition's data into staging, and drops staging
|
||||||
/// table with byte-identical schema, switches the partition's data into
|
/// (discarding the rows). Nothing is dropped from the live table, so there is
|
||||||
/// staging, drops staging (discarding the rows), and rebuilds the unique
|
/// no window in which the idempotency enforcement is absent and no offline
|
||||||
/// index. The CATCH branch guarantees the index is rebuilt even on partial
|
/// index rebuild inside the switch transaction.
|
||||||
/// failure so the table never returns to live traffic without its
|
|
||||||
/// idempotency-supporting index.
|
|
||||||
/// </para>
|
/// </para>
|
||||||
/// <para>
|
/// <para>
|
||||||
/// <b>Outage window.</b> The dance briefly removes the unique index, so
|
/// The predecessor design carried a non-aligned
|
||||||
/// concurrent <see cref="InsertIfNotExistsAsync"/> calls during the switch
|
/// <c>UX_AuditLog_EventId</c> on <c>[PRIMARY]</c> that had to be dropped and
|
||||||
/// could in principle race past the IF NOT EXISTS check without the index
|
/// rebuilt around every switch. A defensive guarded <c>DROP INDEX</c> remains
|
||||||
/// catching the duplicate. This is acceptable for the daily purge cadence
|
/// in the batch so a database restored from a pre-alignment backup still
|
||||||
/// — the inserts that the IF NOT EXISTS check guards are themselves rare
|
/// purges; it is a one-way cleanup, never rebuilt.
|
||||||
/// enough that a sub-second collision window is operationally negligible,
|
|
||||||
/// and the composite PK still rejects same-(EventId, OccurredAtUtc) rows.
|
|
||||||
/// </para>
|
/// </para>
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
/// <param name="monthBoundary">Lower-bound datetime of the monthly partition to switch out.</param>
|
/// <param name="monthBoundary">Lower-bound datetime of the monthly partition to switch out.</param>
|
||||||
/// <param name="commandTimeout">
|
/// <param name="commandTimeout">
|
||||||
/// Optional per-command timeout for the maintenance SQL (the row-count sample plus the
|
/// Optional per-command timeout for the maintenance SQL (the row-count sample plus the
|
||||||
/// drop-and-rebuild dance). When null the provider default applies. The purge actor passes
|
/// staging/switch batch). When null the provider default applies. The purge actor passes
|
||||||
/// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout"/>
|
/// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout"/>
|
||||||
/// (default 30 min) because the ~30s ADO.NET default can abort the switch mid-dance on a large
|
/// (default 30 min) because the ~30s ADO.NET default can abort the switch mid-batch on a large
|
||||||
/// or contended partition, leaving the table without <c>UX_AuditLog_EventId</c> until the next
|
/// or contended partition, leaving an orphaned staging table for the next tick's CATCH to clean
|
||||||
/// tick recovers (arch-review 04, S2).
|
/// up (arch-review 04, S2).
|
||||||
/// </param>
|
/// </param>
|
||||||
/// <param name="ct">Cancellation token.</param>
|
/// <param name="ct">Cancellation token.</param>
|
||||||
/// <returns>A task that resolves to the approximate number of rows discarded by the partition switch.</returns>
|
/// <returns>A task that resolves to the approximate number of rows discarded by the partition switch.</returns>
|
||||||
|
|||||||
+18
-3
@@ -46,10 +46,25 @@ public interface INotificationOutboxRepository
|
|||||||
Task<Notification?> GetByIdAsync(string notificationId, CancellationToken cancellationToken = default);
|
Task<Notification?> GetByIdAsync(string notificationId, CancellationToken cancellationToken = default);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Marks <paramref name="n"/> modified and persists it (status transitions).
|
/// Persists <paramref name="n"/>'s <b>delivery-state</b> columns —
|
||||||
/// Commits internally — this call is its own transaction.
|
/// <c>Status</c>, <c>RetryCount</c>, <c>LastError</c>, <c>ResolvedTargets</c>,
|
||||||
|
/// <c>LastAttemptAt</c>, <c>NextAttemptAt</c>, <c>DeliveredAt</c>. Commits
|
||||||
|
/// internally — this call is its own transaction.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <param name="n">The notification to update.</param>
|
/// <remarks>
|
||||||
|
/// <b>Scope is deliberately narrow.</b> Those seven columns are the ONLY
|
||||||
|
/// mutable state a notification has: everything else (identity, type, list,
|
||||||
|
/// subject, body, type data, source/origin attribution, enqueue and creation
|
||||||
|
/// timestamps) is written once at ingest and is immutable by contract. Every
|
||||||
|
/// caller — the dispatcher's per-attempt write and the operator retry/discard
|
||||||
|
/// one-shots — touches only this set. Implementations are therefore free to
|
||||||
|
/// issue a targeted UPDATE of these columns rather than rewriting the whole
|
||||||
|
/// row, which matters because the row carries <c>nvarchar(max)</c> body and
|
||||||
|
/// payload columns and the dispatcher writes on EVERY attempt. A future
|
||||||
|
/// caller that needs to change an immutable column must add its own method
|
||||||
|
/// rather than widening this one.
|
||||||
|
/// </remarks>
|
||||||
|
/// <param name="n">The notification whose delivery state should be persisted.</param>
|
||||||
/// <param name="cancellationToken">Cancellation token.</param>
|
/// <param name="cancellationToken">Cancellation token.</param>
|
||||||
/// <returns>A task that completes when the notification has been persisted.</returns>
|
/// <returns>A task that completes when the notification has been persisted.</returns>
|
||||||
Task UpdateAsync(Notification n, CancellationToken cancellationToken = default);
|
Task UpdateAsync(Notification n, CancellationToken cancellationToken = default);
|
||||||
|
|||||||
+10
-7
@@ -172,15 +172,18 @@ public class AuditLogEntityTypeConfiguration : IEntityTypeConfiguration<AuditLog
|
|||||||
// ── Keys + indexes ───────────────────────────────────────────────────
|
// ── Keys + indexes ───────────────────────────────────────────────────
|
||||||
|
|
||||||
// Composite PK includes OccurredAtUtc — required by the monthly partition scheme
|
// Composite PK includes OccurredAtUtc — required by the monthly partition scheme
|
||||||
// (ps_AuditLog_Month) so the clustered key is partition-aligned. EventId still
|
// (ps_AuditLog_Month) so the clustered key is partition-aligned. It is ALSO the
|
||||||
// needs to be globally unique for InsertIfNotExistsAsync idempotency, so a
|
// only uniqueness enforcement the ingest path needs: EventId is a GUID minted
|
||||||
// separate (non-aligned) unique index is declared on EventId alone.
|
// once at the emitting site and never re-stamped, so a given EventId always
|
||||||
|
// arrives with the same OccurredAtUtc and can only ever land in one partition.
|
||||||
|
// Uniqueness of the pair is therefore uniqueness of EventId in practice, and the
|
||||||
|
// idempotency probe (WHERE EventId = @id) still seeks the clustered key's leading
|
||||||
|
// column. The predecessor non-aligned UX_AuditLog_EventId on [PRIMARY] was
|
||||||
|
// dropped by AlignAuditLogEventIdUniqueness — it existed only to give
|
||||||
|
// single-column uniqueness and its non-alignment forced an offline drop/rebuild
|
||||||
|
// around every partition-switch purge.
|
||||||
builder.HasKey(e => new { e.EventId, e.OccurredAtUtc });
|
builder.HasKey(e => new { e.EventId, e.OccurredAtUtc });
|
||||||
|
|
||||||
builder.HasIndex(e => e.EventId)
|
|
||||||
.IsUnique()
|
|
||||||
.HasDatabaseName("UX_AuditLog_EventId");
|
|
||||||
|
|
||||||
// Index names are locked for reconciliation/migration discoverability. The
|
// Index names are locked for reconciliation/migration discoverability. The
|
||||||
// column SETS migrate to the canonical/computed shape (alog.md §4 semantics
|
// column SETS migrate to the canonical/computed shape (alog.md §4 semantics
|
||||||
// preserved): Channel→Category, Site/Node/Execution/ParentExecution now read
|
// preserved): Channel→Category, Site/Node/Execution/ParentExecution now read
|
||||||
|
|||||||
+2090
File diff suppressed because it is too large
Load Diff
+93
@@ -0,0 +1,93 @@
|
|||||||
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
|
#nullable disable
|
||||||
|
|
||||||
|
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Makes <c>dbo.AuditLog</c>'s EventId uniqueness <b>partition-aligned</b> by
|
||||||
|
/// dropping the non-aligned <c>UX_AuditLog_EventId</c> and leaving the clustered
|
||||||
|
/// <c>PK_AuditLog (EventId, OccurredAtUtc)</c> — already aligned on
|
||||||
|
/// <c>ps_AuditLog_Month(OccurredAtUtc)</c> — as the sole enforcement.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// <para>
|
||||||
|
/// <b>Why.</b> <c>ALTER TABLE … SWITCH PARTITION</c> refuses to run while a
|
||||||
|
/// non-aligned index exists on the table, so the monthly retention purge
|
||||||
|
/// (<c>AuditLogRepository.SwitchOutPartitionAsync</c>) had to DROP
|
||||||
|
/// <c>UX_AuditLog_EventId</c>, switch, and then CREATE it again — an OFFLINE
|
||||||
|
/// whole-table unique-index build, inside the switch transaction, blocking every
|
||||||
|
/// audit writer for its duration. It also opened a window in which the index that
|
||||||
|
/// backs ingest idempotency did not exist at all, and a mid-dance failure could
|
||||||
|
/// leave the live table without it until a later tick's CATCH branch repaired it.
|
||||||
|
/// With alignment there is nothing to drop, so the switch is metadata-only and the
|
||||||
|
/// purge stops competing with ingest.
|
||||||
|
/// </para>
|
||||||
|
/// <para>
|
||||||
|
/// <b>Why dropping it is safe — EventId is globally unique by construction.</b>
|
||||||
|
/// The composite key enforces uniqueness of the PAIR, not of EventId alone, so in
|
||||||
|
/// principle the same EventId could now be stored twice under two different
|
||||||
|
/// <c>OccurredAtUtc</c> values (in two different partitions). That cannot happen
|
||||||
|
/// here: <c>EventId</c> is a GUID minted ONCE at the emitting site, in the same
|
||||||
|
/// operation that stamps <c>OccurredAtUtc</c>, and both travel together verbatim
|
||||||
|
/// through telemetry and reconciliation — nothing downstream re-stamps either
|
||||||
|
/// field. A given EventId therefore always arrives with the same OccurredAtUtc and
|
||||||
|
/// can only ever map to one partition, which makes pair-uniqueness equivalent to
|
||||||
|
/// EventId-uniqueness for every row this system produces. GUID collision across
|
||||||
|
/// partitions is not a real risk.
|
||||||
|
/// </para>
|
||||||
|
/// <para>
|
||||||
|
/// <b>The idempotency probe still seeks.</b> Both ingest forms test
|
||||||
|
/// <c>WHERE EventId = @id</c>, which is the LEADING column of the clustered PK, so
|
||||||
|
/// the probe remains an index seek. The cost changes shape rather than order: it
|
||||||
|
/// becomes one seek per partition (the partition column is not in the predicate, so
|
||||||
|
/// SQL Server cannot eliminate partitions) instead of a single seek on a
|
||||||
|
/// non-partitioned index. Against a monthly scheme that is a couple of dozen
|
||||||
|
/// shallow B-tree seeks — cheap, and paid on a path that now issues one statement
|
||||||
|
/// per telemetry packet rather than one per row.
|
||||||
|
/// </para>
|
||||||
|
/// <para>
|
||||||
|
/// <b>Edition note.</b> The alternative remedy — keeping the non-aligned index and
|
||||||
|
/// rebuilding it with <c>ONLINE = ON</c> outside the switch transaction — requires
|
||||||
|
/// Enterprise (or Azure SQL / Developer) edition; online index rebuild is not
|
||||||
|
/// available on Standard, which this deployment does not guarantee. Alignment
|
||||||
|
/// needs no edition-specific feature and removes the rebuild entirely, so it is
|
||||||
|
/// preferred regardless of edition.
|
||||||
|
/// </para>
|
||||||
|
/// <para>
|
||||||
|
/// <b>Down is a faithful reverse</b> and recreates the index on <c>[PRIMARY]</c>
|
||||||
|
/// exactly as <c>CollapseAuditLogToCanonical</c> created it. Reverting also
|
||||||
|
/// reinstates the SWITCH incompatibility, so the purge's guarded defensive
|
||||||
|
/// <c>DROP INDEX</c> (retained in <c>SwitchOutPartitionAsync</c> for databases
|
||||||
|
/// restored from pre-alignment backups) would remove it again on the next purge.
|
||||||
|
/// The partition function/scheme (<c>pf_AuditLog_Month</c> /
|
||||||
|
/// <c>ps_AuditLog_Month</c>) and every aligned index are untouched by both
|
||||||
|
/// directions.
|
||||||
|
/// </para>
|
||||||
|
/// </remarks>
|
||||||
|
public partial class AlignAuditLogEventIdUniqueness : Migration
|
||||||
|
{
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Up(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
// Raw, existence-guarded SQL rather than the scaffolded DropIndex: the
|
||||||
|
// AuditLog table is raw-SQL managed (partition scheme, persisted computed
|
||||||
|
// columns, append-only role grants), so its migrations stay explicit and
|
||||||
|
// re-runnable. The guard also lets this apply cleanly to a database whose
|
||||||
|
// index was already removed by the purge path's defensive cleanup.
|
||||||
|
migrationBuilder.Sql(@"
|
||||||
|
IF EXISTS (SELECT 1 FROM sys.indexes
|
||||||
|
WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
|
||||||
|
DROP INDEX UX_AuditLog_EventId ON dbo.AuditLog;");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
protected override void Down(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
migrationBuilder.Sql(@"
|
||||||
|
IF NOT EXISTS (SELECT 1 FROM sys.indexes
|
||||||
|
WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
|
||||||
|
CREATE UNIQUE NONCLUSTERED INDEX UX_AuditLog_EventId ON dbo.AuditLog (EventId) ON [PRIMARY];");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
-4
@@ -1806,10 +1806,6 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
|
|||||||
.HasDatabaseName("IX_AuditLog_CorrelationId")
|
.HasDatabaseName("IX_AuditLog_CorrelationId")
|
||||||
.HasFilter("[CorrelationId] IS NOT NULL");
|
.HasFilter("[CorrelationId] IS NOT NULL");
|
||||||
|
|
||||||
b.HasIndex("EventId")
|
|
||||||
.IsUnique()
|
|
||||||
.HasDatabaseName("UX_AuditLog_EventId");
|
|
||||||
|
|
||||||
b.HasIndex("ExecutionId")
|
b.HasIndex("ExecutionId")
|
||||||
.HasDatabaseName("IX_AuditLog_Execution");
|
.HasDatabaseName("IX_AuditLog_Execution");
|
||||||
|
|
||||||
|
|||||||
+241
-28
@@ -1,5 +1,8 @@
|
|||||||
|
using System.Data;
|
||||||
|
using System.Text;
|
||||||
using Microsoft.Data.SqlClient;
|
using Microsoft.Data.SqlClient;
|
||||||
using Microsoft.EntityFrameworkCore;
|
using Microsoft.EntityFrameworkCore;
|
||||||
|
using Microsoft.EntityFrameworkCore.Storage;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
using Microsoft.Extensions.Logging.Abstractions;
|
using Microsoft.Extensions.Logging.Abstractions;
|
||||||
using ZB.MOM.WW.Audit;
|
using ZB.MOM.WW.Audit;
|
||||||
@@ -17,15 +20,32 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
public class AuditLogRepository : IAuditLogRepository
|
public class AuditLogRepository : IAuditLogRepository
|
||||||
{
|
{
|
||||||
// SQL Server error numbers for duplicate-key violations on
|
// SQL Server error numbers for duplicate-key violations on the
|
||||||
// UX_AuditLog_EventId. 2601 is a unique-index violation; 2627 is a
|
// partition-aligned clustered PK_AuditLog (EventId, OccurredAtUtc).
|
||||||
// primary-key/unique-constraint violation. The IF NOT EXISTS … INSERT
|
// 2601 is a unique-index violation; 2627 is a primary-key/unique-constraint
|
||||||
// pattern has a check-then-act race window — two sessions can both pass
|
// violation. The IF NOT EXISTS … INSERT pattern has a check-then-act race
|
||||||
// the EXISTS check and then both attempt the INSERT — and the loser
|
// window — two sessions can both pass the EXISTS check and then both attempt
|
||||||
// surfaces as one of these errors. Idempotency demands we swallow them.
|
// the INSERT — and the loser surfaces as one of these errors. Idempotency
|
||||||
|
// demands we swallow them.
|
||||||
private const int SqlErrorUniqueIndexViolation = 2601;
|
private const int SqlErrorUniqueIndexViolation = 2601;
|
||||||
private const int SqlErrorPrimaryKeyViolation = 2627;
|
private const int SqlErrorPrimaryKeyViolation = 2627;
|
||||||
|
|
||||||
|
// Rows per set-based ingest statement. Ten bound parameters per row against
|
||||||
|
// SQL Server's 2,100-parameter ceiling leaves ample headroom at 100 rows
|
||||||
|
// (1,000 parameters) while still collapsing a typical telemetry packet into
|
||||||
|
// a single round trip. Larger chunks buy little — the win is round-trip
|
||||||
|
// elimination, not statement size — and would push plan-cache churn up
|
||||||
|
// (one cached plan per distinct row count).
|
||||||
|
private const int IngestChunkRows = 100;
|
||||||
|
|
||||||
|
// Ordinal-stable column list shared by the single-row and set-based inserts.
|
||||||
|
// The five persisted computed columns (Kind/Status/SourceSiteId/ExecutionId/
|
||||||
|
// ParentExecutionId) plus the non-persisted IngestedAtUtc are derived
|
||||||
|
// server-side from DetailsJson and must NEVER appear here — writing a
|
||||||
|
// computed column is an error.
|
||||||
|
private const string CanonicalColumnList =
|
||||||
|
"EventId, OccurredAtUtc, Actor, Action, Outcome, Category, Target, SourceNode, CorrelationId, DetailsJson";
|
||||||
|
|
||||||
private readonly ScadaBridgeDbContext _context;
|
private readonly ScadaBridgeDbContext _context;
|
||||||
private readonly ILogger<AuditLogRepository> _logger;
|
private readonly ILogger<AuditLogRepository> _logger;
|
||||||
|
|
||||||
@@ -83,7 +103,7 @@ VALUES
|
|||||||
{
|
{
|
||||||
// Two concurrent sessions both passed the IF NOT EXISTS check and
|
// Two concurrent sessions both passed the IF NOT EXISTS check and
|
||||||
// both attempted the INSERT — the loser raises 2601/2627 against
|
// both attempted the INSERT — the loser raises 2601/2627 against
|
||||||
// UX_AuditLog_EventId. First-write-wins idempotency is already the
|
// the clustered PK. First-write-wins idempotency is already the
|
||||||
// documented contract for this method, so the race outcome is
|
// documented contract for this method, so the race outcome is
|
||||||
// semantically a no-op. Swallow at Debug; other SqlExceptions
|
// semantically a no-op. Swallow at Debug; other SqlExceptions
|
||||||
// bubble.
|
// bubble.
|
||||||
@@ -95,6 +115,196 @@ VALUES
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public async Task<int> InsertManyIfNotExistsAsync(
|
||||||
|
IReadOnlyList<AuditEvent> events,
|
||||||
|
TimeSpan? commandTimeout = null,
|
||||||
|
CancellationToken ct = default)
|
||||||
|
{
|
||||||
|
ArgumentNullException.ThrowIfNull(events);
|
||||||
|
|
||||||
|
if (events.Count == 0)
|
||||||
|
{
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
// De-duplicate WITHIN the packet before the statement is built. The
|
||||||
|
// set-based INSERT … SELECT … WHERE NOT EXISTS only tests rows that are
|
||||||
|
// ALREADY committed, so two copies of one EventId inside a single VALUES
|
||||||
|
// constructor would both pass the anti-semi-join and collide on the
|
||||||
|
// clustered PK — losing the whole statement to a duplicate-key fault.
|
||||||
|
// First-write-wins matches the single-row contract exactly (a later copy
|
||||||
|
// of the same EventId is by definition the same immutable append-only
|
||||||
|
// row), so keeping the first occurrence is not merely convenient, it is
|
||||||
|
// the documented semantics.
|
||||||
|
var seen = new HashSet<Guid>(events.Count);
|
||||||
|
var distinct = new List<AuditEvent>(events.Count);
|
||||||
|
foreach (var evt in events)
|
||||||
|
{
|
||||||
|
if (evt is not null && seen.Add(evt.EventId))
|
||||||
|
{
|
||||||
|
distinct.Add(evt);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var inserted = 0;
|
||||||
|
for (var offset = 0; offset < distinct.Count; offset += IngestChunkRows)
|
||||||
|
{
|
||||||
|
var length = Math.Min(IngestChunkRows, distinct.Count - offset);
|
||||||
|
var chunk = distinct.GetRange(offset, length);
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
inserted += await InsertChunkAsync(chunk, commandTimeout, ct).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
catch (SqlException ex) when (
|
||||||
|
ex.Number == SqlErrorUniqueIndexViolation
|
||||||
|
|| ex.Number == SqlErrorPrimaryKeyViolation)
|
||||||
|
{
|
||||||
|
// A CONCURRENT writer committed one of this chunk's EventIds
|
||||||
|
// between the anti-semi-join and the insert (the same
|
||||||
|
// check-then-act window the single-row path documents), and the
|
||||||
|
// whole set-based statement rolled back with it. Fall back to
|
||||||
|
// the per-row path so the rows that are genuinely new still land
|
||||||
|
// — the batch is a throughput optimisation, never a correctness
|
||||||
|
// dependency. Every row is retried, including the one that
|
||||||
|
// collided, because InsertIfNotExistsAsync swallows its own
|
||||||
|
// duplicate-key fault as a no-op.
|
||||||
|
_logger.LogDebug(
|
||||||
|
ex,
|
||||||
|
"Set-based audit ingest chunk of {Count} row(s) hit a duplicate-key violation (error {SqlErrorNumber}); falling back to per-row inserts.",
|
||||||
|
chunk.Count,
|
||||||
|
ex.Number);
|
||||||
|
|
||||||
|
foreach (var evt in chunk)
|
||||||
|
{
|
||||||
|
await InsertIfNotExistsAsync(evt, ct).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return inserted;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Executes one set-based idempotent insert: a VALUES table constructor
|
||||||
|
/// anti-semi-joined against the committed rows, so a whole telemetry packet
|
||||||
|
/// costs ONE round trip instead of one <c>IF NOT EXISTS … INSERT</c> per row.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Raw ADO.NET (rather than <c>ExecuteSqlInterpolated</c>) because the
|
||||||
|
/// statement's parameter count varies with the chunk size and every parameter
|
||||||
|
/// needs an explicit <see cref="SqlDbType"/>: the VALUES constructor's column
|
||||||
|
/// types are inferred from the first row's parameters, so leaving a null
|
||||||
|
/// <c>Target</c>/<c>SourceNode</c> untyped would give the derived column the
|
||||||
|
/// wrong type and defeat the seek on the anti-semi-join. The command enlists
|
||||||
|
/// in the DbContext's ambient transaction when one is open — the cached
|
||||||
|
/// telemetry dual-write runs the audit insert and the SiteCalls upsert inside
|
||||||
|
/// a single transaction and both must commit or roll back together.
|
||||||
|
/// </remarks>
|
||||||
|
private async Task<int> InsertChunkAsync(
|
||||||
|
IReadOnlyList<AuditEvent> chunk, TimeSpan? commandTimeout, CancellationToken ct)
|
||||||
|
{
|
||||||
|
var sql = new StringBuilder(256 + (chunk.Count * 64));
|
||||||
|
sql.Append("INSERT INTO dbo.AuditLog (").Append(CanonicalColumnList).Append(")\n");
|
||||||
|
sql.Append("SELECT v.EventId, v.OccurredAtUtc, v.Actor, v.Action, v.Outcome, v.Category, ");
|
||||||
|
sql.Append("v.Target, v.SourceNode, v.CorrelationId, v.DetailsJson\n");
|
||||||
|
sql.Append("FROM (VALUES\n");
|
||||||
|
for (var i = 0; i < chunk.Count; i++)
|
||||||
|
{
|
||||||
|
if (i > 0)
|
||||||
|
{
|
||||||
|
sql.Append(",\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
sql.Append(" (@e").Append(i)
|
||||||
|
.Append(",@t").Append(i)
|
||||||
|
.Append(",@a").Append(i)
|
||||||
|
.Append(",@n").Append(i)
|
||||||
|
.Append(",@o").Append(i)
|
||||||
|
.Append(",@c").Append(i)
|
||||||
|
.Append(",@g").Append(i)
|
||||||
|
.Append(",@s").Append(i)
|
||||||
|
.Append(",@r").Append(i)
|
||||||
|
.Append(",@d").Append(i)
|
||||||
|
.Append(')');
|
||||||
|
}
|
||||||
|
|
||||||
|
sql.Append("\n) AS v (").Append(CanonicalColumnList).Append(")\n");
|
||||||
|
sql.Append("WHERE NOT EXISTS (SELECT 1 FROM dbo.AuditLog x WHERE x.EventId = v.EventId);");
|
||||||
|
|
||||||
|
var conn = _context.Database.GetDbConnection();
|
||||||
|
var openedHere = false;
|
||||||
|
if (conn.State != ConnectionState.Open)
|
||||||
|
{
|
||||||
|
await conn.OpenAsync(ct).ConfigureAwait(false);
|
||||||
|
openedHere = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await using var cmd = conn.CreateCommand();
|
||||||
|
cmd.CommandText = sql.ToString();
|
||||||
|
cmd.Transaction = _context.Database.CurrentTransaction?.GetDbTransaction();
|
||||||
|
if (commandTimeout is { } timeout)
|
||||||
|
{
|
||||||
|
cmd.CommandTimeout = (int)timeout.TotalSeconds;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (var i = 0; i < chunk.Count; i++)
|
||||||
|
{
|
||||||
|
var evt = chunk[i];
|
||||||
|
|
||||||
|
// Same canonical projection as the single-row path: UTC-kind
|
||||||
|
// OccurredAtUtc, empty Actor collapses to NULL, Outcome/Category
|
||||||
|
// bound as their varchar storage form.
|
||||||
|
var occurred = DateTime.SpecifyKind(evt.OccurredAtUtc.UtcDateTime, DateTimeKind.Utc);
|
||||||
|
object actor = string.IsNullOrEmpty(evt.Actor) ? DBNull.Value : evt.Actor;
|
||||||
|
|
||||||
|
AddParameter(cmd, "@e" + i, SqlDbType.UniqueIdentifier, size: 0, evt.EventId);
|
||||||
|
AddParameter(cmd, "@t" + i, SqlDbType.DateTime2, size: 0, occurred);
|
||||||
|
AddParameter(cmd, "@a" + i, SqlDbType.NVarChar, size: 256, actor);
|
||||||
|
AddParameter(cmd, "@n" + i, SqlDbType.VarChar, size: 64, evt.Action);
|
||||||
|
AddParameter(cmd, "@o" + i, SqlDbType.VarChar, size: 16, evt.Outcome.ToString());
|
||||||
|
AddParameter(cmd, "@c" + i, SqlDbType.VarChar, size: 32, evt.Category);
|
||||||
|
AddParameter(cmd, "@g" + i, SqlDbType.NVarChar, size: 256, evt.Target);
|
||||||
|
AddParameter(cmd, "@s" + i, SqlDbType.VarChar, size: 64, evt.SourceNode);
|
||||||
|
AddParameter(cmd, "@r" + i, SqlDbType.UniqueIdentifier, size: 0, evt.CorrelationId);
|
||||||
|
AddParameter(cmd, "@d" + i, SqlDbType.NVarChar, size: -1, evt.DetailsJson);
|
||||||
|
}
|
||||||
|
|
||||||
|
return await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
if (openedHere)
|
||||||
|
{
|
||||||
|
await conn.CloseAsync().ConfigureAwait(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Binds one explicitly-typed parameter. A null CLR value binds as
|
||||||
|
/// <see cref="DBNull"/> while KEEPING its declared <see cref="SqlDbType"/>,
|
||||||
|
/// which is what makes the VALUES constructor's derived column types stable
|
||||||
|
/// regardless of which rows happen to carry nulls.
|
||||||
|
/// </summary>
|
||||||
|
private static void AddParameter(
|
||||||
|
System.Data.Common.DbCommand cmd, string name, SqlDbType type, int size, object? value)
|
||||||
|
{
|
||||||
|
var p = (SqlParameter)cmd.CreateParameter();
|
||||||
|
p.ParameterName = name;
|
||||||
|
p.SqlDbType = type;
|
||||||
|
if (size != 0)
|
||||||
|
{
|
||||||
|
p.Size = size;
|
||||||
|
}
|
||||||
|
|
||||||
|
p.Value = value ?? DBNull.Value;
|
||||||
|
cmd.Parameters.Add(p);
|
||||||
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public async Task<IReadOnlyList<AuditEvent>> QueryAsync(
|
public async Task<IReadOnlyList<AuditEvent>> QueryAsync(
|
||||||
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default)
|
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default)
|
||||||
@@ -229,13 +439,23 @@ VALUES
|
|||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public async Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default)
|
public async Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
// The drop-and-rebuild batch below runs via
|
// The switch batch below runs via
|
||||||
// ExecuteSqlRaw with NO EF user-transaction — it carries its own server-side
|
// ExecuteSqlRaw with NO EF user-transaction — it carries its own server-side
|
||||||
// BEGIN TRANSACTION / TRY-CATCH / ROLLBACK — so the DbContext's retrying
|
// BEGIN TRANSACTION / TRY-CATCH / ROLLBACK — so the DbContext's retrying
|
||||||
// execution strategy (EnableRetryOnFailure) MAY auto-replay the whole batch on
|
// execution strategy (EnableRetryOnFailure) MAY auto-replay the whole batch on
|
||||||
// a transient fault. That replay is safe: every step is IF-EXISTS / IF-NOT-EXISTS
|
// a transient fault. That replay is safe: every step is IF-EXISTS / IF-NOT-EXISTS
|
||||||
// guarded and the staging table is GUID-suffixed, so a re-run is idempotent.
|
// guarded and the staging table is GUID-suffixed, so a re-run is idempotent.
|
||||||
//
|
//
|
||||||
|
// ALIGNED UNIQUENESS (WP2.2): there is no longer an index drop/rebuild around
|
||||||
|
// the SWITCH. EventId uniqueness is now enforced solely by the clustered
|
||||||
|
// PK_AuditLog (EventId, OccurredAtUtc), which is partition-aligned on
|
||||||
|
// ps_AuditLog_Month(OccurredAtUtc) — so ALTER TABLE … SWITCH PARTITION has no
|
||||||
|
// non-aligned index to object to. The former dance dropped UX_AuditLog_EventId,
|
||||||
|
// switched, then rebuilt it OFFLINE inside the same transaction: a whole-table
|
||||||
|
// unique-index build blocking every writer for the duration of the purge, and a
|
||||||
|
// window in which the idempotency-supporting index did not exist at all. Both
|
||||||
|
// are gone. See migration AlignAuditLogEventIdUniqueness for the reasoning.
|
||||||
|
//
|
||||||
// Maintenance timeout in whole seconds (ADO.NET CommandTimeout unit). Null leaves the
|
// Maintenance timeout in whole seconds (ADO.NET CommandTimeout unit). Null leaves the
|
||||||
// provider default in place. See AuditLogPurgeOptions.MaintenanceCommandTimeoutMinutes /
|
// provider default in place. See AuditLogPurgeOptions.MaintenanceCommandTimeoutMinutes /
|
||||||
// arch-review 04 S2 for why the ~30s default is unsafe for the switch-out dance.
|
// arch-review 04 S2 for why the ~30s default is unsafe for the switch-out dance.
|
||||||
@@ -270,8 +490,13 @@ VALUES
|
|||||||
BEGIN TRY
|
BEGIN TRY
|
||||||
BEGIN TRANSACTION;
|
BEGIN TRANSACTION;
|
||||||
|
|
||||||
-- 1. Drop the non-aligned unique index. ALTER TABLE SWITCH refuses
|
-- 1. Defensive cleanup for databases created before
|
||||||
-- to run while it exists.
|
-- AlignAuditLogEventIdUniqueness: the migration drops the
|
||||||
|
-- non-aligned UX_AuditLog_EventId, but a database restored from an
|
||||||
|
-- older backup could still carry it and SWITCH refuses to run while
|
||||||
|
-- a non-aligned unique index exists. Dropping it here is idempotent
|
||||||
|
-- and permanent — the aligned clustered PK is the only uniqueness
|
||||||
|
-- enforcement the ingest path needs, so there is nothing to rebuild.
|
||||||
IF EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
|
IF EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
|
||||||
DROP INDEX UX_AuditLog_EventId ON dbo.AuditLog;
|
DROP INDEX UX_AuditLog_EventId ON dbo.AuditLog;
|
||||||
|
|
||||||
@@ -313,31 +538,19 @@ VALUES
|
|||||||
-- 4. Drop staging — the rows are discarded here. This is the purge.
|
-- 4. Drop staging — the rows are discarded here. This is the purge.
|
||||||
DROP TABLE dbo.[{stagingTableName}];
|
DROP TABLE dbo.[{stagingTableName}];
|
||||||
|
|
||||||
-- 5. Rebuild the non-aligned unique index. Live traffic that hit the
|
|
||||||
-- table during steps 1-4 saw composite-PK uniqueness only; from
|
|
||||||
-- here on, single-column EventId uniqueness is restored.
|
|
||||||
CREATE UNIQUE NONCLUSTERED INDEX UX_AuditLog_EventId ON dbo.AuditLog (EventId) ON [PRIMARY];
|
|
||||||
|
|
||||||
COMMIT TRANSACTION;
|
COMMIT TRANSACTION;
|
||||||
END TRY
|
END TRY
|
||||||
BEGIN CATCH
|
BEGIN CATCH
|
||||||
IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION;
|
IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION;
|
||||||
|
|
||||||
-- Best-effort staging cleanup. The DROP INDEX in step 1 is now
|
-- Best-effort staging cleanup. The staging table from step 2 may or
|
||||||
-- rolled back (so the index is back), but the staging table from
|
-- may not survive the rollback depending on the failure point. Guard
|
||||||
-- step 2 may or may not survive the rollback depending on the
|
-- the DROP so a missing staging table doesn't mask the original error.
|
||||||
-- failure point. Guard the DROP so a missing staging table doesn't
|
-- Nothing else needs repairing: uniqueness lives on the clustered PK,
|
||||||
-- mask the original error.
|
-- which the switch never touches, so a failed purge can no longer
|
||||||
|
-- leave the live table without its idempotency enforcement.
|
||||||
IF OBJECT_ID('dbo.[{stagingTableName}]', 'U') IS NOT NULL DROP TABLE dbo.[{stagingTableName}];
|
IF OBJECT_ID('dbo.[{stagingTableName}]', 'U') IS NOT NULL DROP TABLE dbo.[{stagingTableName}];
|
||||||
|
|
||||||
-- Idempotent index rebuild — covers the niche case where ROLLBACK
|
|
||||||
-- failed to restore UX_AuditLog_EventId (or the failure happened
|
|
||||||
-- AFTER the COMMIT, which shouldn't be possible inside this TRY
|
|
||||||
-- but is cheap insurance). Without this, a failed switch could
|
|
||||||
-- leave the live table without its idempotency-supporting index.
|
|
||||||
IF NOT EXISTS (SELECT 1 FROM sys.indexes WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog'))
|
|
||||||
CREATE UNIQUE NONCLUSTERED INDEX UX_AuditLog_EventId ON dbo.AuditLog (EventId) ON [PRIMARY];
|
|
||||||
|
|
||||||
-- Surface the original error to the caller — the purge actor logs
|
-- Surface the original error to the caller — the purge actor logs
|
||||||
-- and continues with the next boundary.
|
-- and continues with the next boundary.
|
||||||
THROW;
|
THROW;
|
||||||
|
|||||||
+36
-12
@@ -128,6 +128,37 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
|
|||||||
var groups = samples.GroupBy(s => new SeriesHourKey(
|
var groups = samples.GroupBy(s => new SeriesHourKey(
|
||||||
s.Source, s.Metric, s.Scope, s.ScopeKey, TruncateToHour(s.CapturedAtUtc)));
|
s.Source, s.Metric, s.Scope, s.ScopeKey, TruncateToHour(s.CapturedAtUtc)));
|
||||||
|
|
||||||
|
// Preload every rollup row already covering this window in ONE query and
|
||||||
|
// index it by series+hour (WP2.2). The predecessor issued a
|
||||||
|
// FirstOrDefaultAsync existence probe PER (series, hour) group — an N+1
|
||||||
|
// that scaled with the metric catalogue times the lookback: a 3 h re-fold
|
||||||
|
// over ~40 series cost ~120 sequential round trips before a single row was
|
||||||
|
// written. The window is bounded by the caller's small trailing lookback
|
||||||
|
// and the rollup table holds exactly one row per series-hour, so the
|
||||||
|
// preload is a narrow range seek on IX_KpiRollupHourly_Series.
|
||||||
|
//
|
||||||
|
// Deliberately TRACKED (not a projection): the re-fold path mutates the
|
||||||
|
// existing entity in place and relies on the change tracker to emit the
|
||||||
|
// UPDATE. The dictionary's ScopeKey comparison is ordinal where the
|
||||||
|
// previous SQL predicate used the database collation; both sides are
|
||||||
|
// written from the same KpiSample.ScopeKey values, so they are
|
||||||
|
// byte-identical in practice and a residual mismatch degrades to the
|
||||||
|
// already-handled upsert-race path rather than a wrong aggregate.
|
||||||
|
var existingRollups = await _context.KpiRollupHourly
|
||||||
|
.Where(r => r.HourStartUtc >= from && r.HourStartUtc < to)
|
||||||
|
.ToListAsync(cancellationToken);
|
||||||
|
|
||||||
|
var existingByKey = new Dictionary<SeriesHourKey, KpiRollupHourly>(existingRollups.Count);
|
||||||
|
foreach (var row in existingRollups)
|
||||||
|
{
|
||||||
|
existingByKey[new SeriesHourKey(
|
||||||
|
row.Source,
|
||||||
|
row.Metric,
|
||||||
|
row.Scope,
|
||||||
|
row.ScopeKey,
|
||||||
|
DateTime.SpecifyKind(row.HourStartUtc, DateTimeKind.Utc))] = row;
|
||||||
|
}
|
||||||
|
|
||||||
foreach (var group in groups)
|
foreach (var group in groups)
|
||||||
{
|
{
|
||||||
var key = group.Key;
|
var key = group.Key;
|
||||||
@@ -143,18 +174,11 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
|
|||||||
var maxValue = group.Max(s => s.Value);
|
var maxValue = group.Max(s => s.Value);
|
||||||
var sampleCount = group.Count();
|
var sampleCount = group.Count();
|
||||||
|
|
||||||
// Idempotent upsert on the unique series+hour key. The ScopeKey == key.ScopeKey
|
// Idempotent upsert on the unique series+hour key, resolved against the
|
||||||
// comparison matches null against the Global-scope rows (IS NULL) exactly as the
|
// preloaded dictionary. A null ScopeKey keys the Global-scope rows exactly
|
||||||
// UNIQUE IX_KpiRollupHourly_Series index treats a null key as participating.
|
// as the UNIQUE IX_KpiRollupHourly_Series index treats a null key as
|
||||||
var existing = await _context.KpiRollupHourly.FirstOrDefaultAsync(
|
// participating.
|
||||||
r => r.Source == key.Source
|
if (!existingByKey.TryGetValue(key, out var existing))
|
||||||
&& r.Metric == key.Metric
|
|
||||||
&& r.Scope == key.Scope
|
|
||||||
&& r.ScopeKey == key.ScopeKey
|
|
||||||
&& r.HourStartUtc == key.HourStartUtc,
|
|
||||||
cancellationToken);
|
|
||||||
|
|
||||||
if (existing is null)
|
|
||||||
{
|
{
|
||||||
_context.KpiRollupHourly.Add(new KpiRollupHourly
|
_context.KpiRollupHourly.Add(new KpiRollupHourly
|
||||||
{
|
{
|
||||||
|
|||||||
+74
-7
@@ -140,7 +140,14 @@ VALUES
|
|||||||
public async Task<IReadOnlyList<Notification>> GetDueAsync(
|
public async Task<IReadOnlyList<Notification>> GetDueAsync(
|
||||||
DateTimeOffset now, int batchSize, CancellationToken cancellationToken = default)
|
DateTimeOffset now, int batchSize, CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
|
// AsNoTracking (WP2.2): the dispatcher mutates each row's delivery state in
|
||||||
|
// memory and persists it through UpdateAsync, which is now a targeted
|
||||||
|
// server-side ExecuteUpdate and needs no change tracker. Tracking a batch
|
||||||
|
// of notifications — each carrying an nvarchar(max) Body and TypeData —
|
||||||
|
// paid for a full snapshot copy per row plus a DetectChanges scan of the
|
||||||
|
// whole batch on every save.
|
||||||
return await _context.Notifications
|
return await _context.Notifications
|
||||||
|
.AsNoTracking()
|
||||||
.Where(n => n.Status == NotificationStatus.Pending
|
.Where(n => n.Status == NotificationStatus.Pending
|
||||||
|| (n.Status == NotificationStatus.Retrying
|
|| (n.Status == NotificationStatus.Retrying
|
||||||
&& n.NextAttemptAt != null
|
&& n.NextAttemptAt != null
|
||||||
@@ -150,22 +157,59 @@ VALUES
|
|||||||
.ToListAsync(cancellationToken);
|
.ToListAsync(cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
|
||||||
public async Task<Notification?> GetByIdAsync(string notificationId, CancellationToken cancellationToken = default)
|
|
||||||
=> await _context.Notifications.FindAsync(new object[] { notificationId }, cancellationToken);
|
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public async Task UpdateAsync(Notification n, CancellationToken cancellationToken = default)
|
public async Task UpdateAsync(Notification n, CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
_context.Notifications.Update(n);
|
ArgumentNullException.ThrowIfNull(n);
|
||||||
await _context.SaveChangesAsync(cancellationToken);
|
|
||||||
|
// Targeted server-side UPDATE of the seven mutable delivery-state columns
|
||||||
|
// (WP2.2). The predecessor called DbSet.Update(n) + SaveChanges, which
|
||||||
|
// marks EVERY property modified and rewrites all 21 columns — including
|
||||||
|
// the immutable nvarchar(max) Body/TypeData payloads — on every single
|
||||||
|
// delivery attempt. ExecuteUpdate also bypasses the change tracker
|
||||||
|
// entirely, so it composes with the untracked GetDueAsync read.
|
||||||
|
//
|
||||||
|
// Immutable-by-contract columns (NotificationId, Type, ListName, Subject,
|
||||||
|
// Body, TypeData, Source*, Origin*, SiteEnqueuedAt, CreatedAt) are
|
||||||
|
// deliberately absent — see the interface contract: nothing in the
|
||||||
|
// notification lifecycle ever changes them, and omitting them is what
|
||||||
|
// makes the write narrow.
|
||||||
|
var status = n.Status;
|
||||||
|
var retryCount = n.RetryCount;
|
||||||
|
var lastError = n.LastError;
|
||||||
|
var resolvedTargets = n.ResolvedTargets;
|
||||||
|
var lastAttemptAt = n.LastAttemptAt;
|
||||||
|
var nextAttemptAt = n.NextAttemptAt;
|
||||||
|
var deliveredAt = n.DeliveredAt;
|
||||||
|
|
||||||
|
var notificationId = n.NotificationId;
|
||||||
|
|
||||||
|
await _context.Notifications
|
||||||
|
.Where(row => row.NotificationId == notificationId)
|
||||||
|
.ExecuteUpdateAsync(
|
||||||
|
setters => setters
|
||||||
|
.SetProperty(row => row.Status, status)
|
||||||
|
.SetProperty(row => row.RetryCount, retryCount)
|
||||||
|
.SetProperty(row => row.LastError, lastError)
|
||||||
|
.SetProperty(row => row.ResolvedTargets, resolvedTargets)
|
||||||
|
.SetProperty(row => row.LastAttemptAt, lastAttemptAt)
|
||||||
|
.SetProperty(row => row.NextAttemptAt, nextAttemptAt)
|
||||||
|
.SetProperty(row => row.DeliveredAt, deliveredAt),
|
||||||
|
cancellationToken);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <inheritdoc />
|
||||||
|
public async Task<Notification?> GetByIdAsync(string notificationId, CancellationToken cancellationToken = default)
|
||||||
|
=> await _context.Notifications.FindAsync(new object[] { notificationId }, cancellationToken);
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public async Task<(IReadOnlyList<Notification> Rows, int TotalCount)> QueryAsync(
|
public async Task<(IReadOnlyList<Notification> Rows, int TotalCount)> QueryAsync(
|
||||||
NotificationOutboxFilter filter, int pageNumber, int pageSize, CancellationToken cancellationToken = default)
|
NotificationOutboxFilter filter, int pageNumber, int pageSize, CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
var query = _context.Notifications.AsQueryable();
|
// AsNoTracking (WP2.2): this is the Central UI's read-only list page. The
|
||||||
|
// rows are projected to the wire and never saved, so tracking them cost a
|
||||||
|
// snapshot copy of every nvarchar(max) Body in the page for nothing.
|
||||||
|
var query = _context.Notifications.AsNoTracking().AsQueryable();
|
||||||
|
|
||||||
if (filter.Status is { } status)
|
if (filter.Status is { } status)
|
||||||
{
|
{
|
||||||
@@ -218,8 +262,14 @@ VALUES
|
|||||||
|
|
||||||
var totalCount = await query.CountAsync(cancellationToken);
|
var totalCount = await query.CountAsync(cancellationToken);
|
||||||
|
|
||||||
|
// NotificationId breaks CreatedAt ties so the OFFSET window is deterministic —
|
||||||
|
// without it two rows sharing a CreatedAt could appear on both pages or on
|
||||||
|
// neither. (This page keeps OFFSET paging rather than the sibling repos'
|
||||||
|
// keyset cursor because its contract surfaces a page number and a total
|
||||||
|
// count, neither of which a keyset cursor can express.)
|
||||||
var rows = await query
|
var rows = await query
|
||||||
.OrderByDescending(n => n.CreatedAt)
|
.OrderByDescending(n => n.CreatedAt)
|
||||||
|
.ThenByDescending(n => n.NotificationId)
|
||||||
.Skip((pageNumber - 1) * pageSize)
|
.Skip((pageNumber - 1) * pageSize)
|
||||||
.Take(pageSize)
|
.Take(pageSize)
|
||||||
.ToListAsync(cancellationToken);
|
.ToListAsync(cancellationToken);
|
||||||
@@ -265,7 +315,24 @@ VALUES
|
|||||||
// One conditional-aggregation pass replaces four sequential COUNT round trips:
|
// One conditional-aggregation pass replaces four sequential COUNT round trips:
|
||||||
// each metric is a COUNT(CASE WHEN <predicate> THEN 1 END) over the same scan
|
// each metric is a COUNT(CASE WHEN <predicate> THEN 1 END) over the same scan
|
||||||
// (arch-review 04). GroupBy(_ => 1) yields a single group (no rows → no group).
|
// (arch-review 04). GroupBy(_ => 1) yields a single group (no rows → no group).
|
||||||
|
//
|
||||||
|
// WP2.2 — the aggregation is now PREDICATE-RESTRICTED instead of scanning the
|
||||||
|
// whole table. Every KPI here is about the live queue (Pending/Retrying), the
|
||||||
|
// parked backlog, or the last delivery interval; the overwhelming bulk of the
|
||||||
|
// Notifications table is historical Delivered and Discarded rows that
|
||||||
|
// contribute to NONE of them. The pre-filter below is the union of the
|
||||||
|
// metric-contributing predicates, which lets the optimizer seek the status
|
||||||
|
// index for the live/parked legs and the filtered
|
||||||
|
// (DeliveredAt) WHERE Status='Delivered' index (WP1.4) for the interval leg,
|
||||||
|
// instead of paying a full scan whose cost grows with retained history.
|
||||||
|
// Mirrors the pre-filter ComputePerSiteKpisAsync/ComputePerNodeKpisAsync
|
||||||
|
// already use — the global snapshot was the odd one out.
|
||||||
var counts = await _context.Notifications
|
var counts = await _context.Notifications
|
||||||
|
.Where(n => n.Status == NotificationStatus.Pending
|
||||||
|
|| n.Status == NotificationStatus.Retrying
|
||||||
|
|| n.Status == NotificationStatus.Parked
|
||||||
|
|| (n.Status == NotificationStatus.Delivered
|
||||||
|
&& n.DeliveredAt != null && n.DeliveredAt >= deliveredSince))
|
||||||
.GroupBy(_ => 1)
|
.GroupBy(_ => 1)
|
||||||
.Select(g => new
|
.Select(g => new
|
||||||
{
|
{
|
||||||
|
|||||||
+66
-43
@@ -72,49 +72,36 @@ public class SiteCallAuditRepository : ISiteCallAuditRepository
|
|||||||
var idText = siteCall.TrackedOperationId.Value.ToString("D");
|
var idText = siteCall.TrackedOperationId.Value.ToString("D");
|
||||||
var incomingRank = GetRankOrThrow(siteCall.Status);
|
var incomingRank = GetRankOrThrow(siteCall.Status);
|
||||||
|
|
||||||
// Step 1: insert-if-not-exists. Like AuditLogRepository.InsertIfNotExistsAsync
|
// ONE round trip, UPDATE-first (WP2.2). The predecessor issued an
|
||||||
// this is check-then-act so a duplicate-key violation may surface under
|
// unconditional IF NOT EXISTS … INSERT and THEN a monotonic UPDATE — two
|
||||||
// concurrent inserts on the same id — caught + logged at Debug.
|
// statements, two round trips, on every single packet, of which the insert
|
||||||
|
// half was wasted work for every packet after the first (the steady state:
|
||||||
|
// a cached call emits Submitted → Forwarded → Attempted → terminal, so
|
||||||
|
// three of four packets hit an existing row).
|
||||||
//
|
//
|
||||||
// SourceNode-stamping: the column is included in the INSERT
|
// The combined batch below runs UPDATE first and inserts only when the
|
||||||
// column list / VALUES so a fresh row carries the originating node
|
// UPDATE matched nothing AND the row genuinely does not exist. The
|
||||||
// name (node-a/node-b for site rows). A null SourceNode (legacy hosts
|
// NOT EXISTS re-check is load-bearing: @@ROWCOUNT = 0 is ALSO what a
|
||||||
// / unstamped reconciled rows) writes NULL straight through.
|
// monotonic REJECTION looks like (a stale or regressive packet against an
|
||||||
try
|
// existing row), and inserting there would resurrect a row the guard just
|
||||||
{
|
// refused. Both statements ship in one command text, so this is one
|
||||||
await _context.Database.ExecuteSqlInterpolatedAsync(
|
// round trip, not two.
|
||||||
$@"IF NOT EXISTS (SELECT 1 FROM dbo.SiteCalls WHERE TrackedOperationId = {idText})
|
|
||||||
INSERT INTO dbo.SiteCalls
|
|
||||||
(TrackedOperationId, Channel, Target, SourceSite, SourceNode, Status, RetryCount,
|
|
||||||
LastError, HttpStatus, CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, IngestedAtUtc)
|
|
||||||
VALUES
|
|
||||||
({idText}, {siteCall.Channel}, {siteCall.Target}, {siteCall.SourceSite}, {siteCall.SourceNode}, {siteCall.Status}, {siteCall.RetryCount},
|
|
||||||
{siteCall.LastError}, {siteCall.HttpStatus}, {siteCall.CreatedAtUtc}, {siteCall.UpdatedAtUtc}, {siteCall.TerminalAtUtc}, {siteCall.IngestedAtUtc});",
|
|
||||||
ct);
|
|
||||||
}
|
|
||||||
catch (SqlException ex) when (
|
|
||||||
ex.Number == SqlErrorUniqueIndexViolation
|
|
||||||
|| ex.Number == SqlErrorPrimaryKeyViolation)
|
|
||||||
{
|
|
||||||
_logger.LogDebug(
|
|
||||||
ex,
|
|
||||||
"SiteCallAuditRepository.UpsertAsync swallowed duplicate-key violation (error {SqlErrorNumber}) for TrackedOperationId {TrackedOperationId}; falling through to monotonic update.",
|
|
||||||
ex.Number,
|
|
||||||
idText);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Step 2: monotonic update with a same-rank freshness tiebreaker. The
|
|
||||||
// CASE expression maps the stored Status string to the same rank table
|
|
||||||
// the caller uses. We mutate when EITHER the incoming rank is strictly
|
|
||||||
// greater, OR the incoming rank equals the stored rank AND that rank is
|
|
||||||
// non-terminal (< TerminalRank) AND the incoming UpdatedAtUtc is strictly
|
|
||||||
// newer than the stored one — so a retrying call's Attempted-phase
|
|
||||||
// RetryCount/LastError/HttpStatus stay live instead of freezing at the
|
|
||||||
// first Attempted packet. Terminal ranks are excluded from the
|
|
||||||
// tiebreaker, so a later terminal NEVER overwrites an earlier one; equal
|
|
||||||
// stamps are inert (idempotent replay) and a lower rank is always a no-op.
|
|
||||||
//
|
//
|
||||||
// SourceNode-stamping: SourceNode is updated via
|
// Monotonic update semantics are unchanged: mutate when EITHER the
|
||||||
|
// incoming rank is strictly greater, OR the incoming rank equals the
|
||||||
|
// stored rank AND that rank is non-terminal (< TerminalRank) AND the
|
||||||
|
// incoming UpdatedAtUtc is strictly newer than the stored one — so a
|
||||||
|
// retrying call's Attempted-phase RetryCount/LastError/HttpStatus stay
|
||||||
|
// live instead of freezing at the first Attempted packet. Terminal ranks
|
||||||
|
// are excluded from the tiebreaker, so a later terminal NEVER overwrites
|
||||||
|
// an earlier one; equal stamps are inert (idempotent replay) and a lower
|
||||||
|
// rank is always a no-op.
|
||||||
|
//
|
||||||
|
// SourceNode-stamping: the column is included in the INSERT column list /
|
||||||
|
// VALUES so a fresh row carries the originating node name (node-a/node-b
|
||||||
|
// for site rows). A null SourceNode (legacy hosts / unstamped reconciled
|
||||||
|
// rows) writes NULL straight through. On the UPDATE leg SourceNode is
|
||||||
|
// written via
|
||||||
// COALESCE(@SourceNode, SourceNode). The operator returns @SourceNode
|
// COALESCE(@SourceNode, SourceNode). The operator returns @SourceNode
|
||||||
// when it is non-null, otherwise the stored value — so the column
|
// when it is non-null, otherwise the stored value — so the column
|
||||||
// behaves protectively: a later packet that carries a null
|
// behaves protectively: a later packet that carries a null
|
||||||
@@ -128,8 +115,12 @@ VALUES
|
|||||||
// lifecycle every packet should carry the same SourceNode value (one
|
// lifecycle every packet should carry the same SourceNode value (one
|
||||||
// execution, one node) so the "overwrite" path is in practice
|
// execution, one node) so the "overwrite" path is in practice
|
||||||
// idempotent.
|
// idempotent.
|
||||||
|
try
|
||||||
|
{
|
||||||
await _context.Database.ExecuteSqlInterpolatedAsync(
|
await _context.Database.ExecuteSqlInterpolatedAsync(
|
||||||
$@"UPDATE dbo.SiteCalls
|
$@"DECLARE @updated int;
|
||||||
|
|
||||||
|
UPDATE dbo.SiteCalls
|
||||||
SET Status = {siteCall.Status},
|
SET Status = {siteCall.Status},
|
||||||
RetryCount = {siteCall.RetryCount},
|
RetryCount = {siteCall.RetryCount},
|
||||||
LastError = {siteCall.LastError},
|
LastError = {siteCall.LastError},
|
||||||
@@ -162,9 +153,41 @@ WHERE TrackedOperationId = {idText}
|
|||||||
ELSE -1
|
ELSE -1
|
||||||
END)
|
END)
|
||||||
AND {incomingRank} < {TerminalRank}
|
AND {incomingRank} < {TerminalRank}
|
||||||
AND UpdatedAtUtc < {siteCall.UpdatedAtUtc} ) );",
|
AND UpdatedAtUtc < {siteCall.UpdatedAtUtc} ) );
|
||||||
|
|
||||||
|
-- Captured IMMEDIATELY after the UPDATE: @@ROWCOUNT is reset by the next
|
||||||
|
-- statement, and reading it inline inside a compound IF condition alongside a
|
||||||
|
-- subquery is not safe (the subquery's own execution can clobber it).
|
||||||
|
SET @updated = @@ROWCOUNT;
|
||||||
|
|
||||||
|
IF @updated = 0 AND NOT EXISTS (SELECT 1 FROM dbo.SiteCalls WHERE TrackedOperationId = {idText})
|
||||||
|
INSERT INTO dbo.SiteCalls
|
||||||
|
(TrackedOperationId, Channel, Target, SourceSite, SourceNode, Status, RetryCount,
|
||||||
|
LastError, HttpStatus, CreatedAtUtc, UpdatedAtUtc, TerminalAtUtc, IngestedAtUtc)
|
||||||
|
VALUES
|
||||||
|
({idText}, {siteCall.Channel}, {siteCall.Target}, {siteCall.SourceSite}, {siteCall.SourceNode}, {siteCall.Status}, {siteCall.RetryCount},
|
||||||
|
{siteCall.LastError}, {siteCall.HttpStatus}, {siteCall.CreatedAtUtc}, {siteCall.UpdatedAtUtc}, {siteCall.TerminalAtUtc}, {siteCall.IngestedAtUtc});",
|
||||||
ct);
|
ct);
|
||||||
}
|
}
|
||||||
|
catch (SqlException ex) when (
|
||||||
|
ex.Number == SqlErrorUniqueIndexViolation
|
||||||
|
|| ex.Number == SqlErrorPrimaryKeyViolation)
|
||||||
|
{
|
||||||
|
// Two concurrent sessions both found the row absent and both raced to
|
||||||
|
// INSERT; the loser raises 2601/2627 against the TrackedOperationId
|
||||||
|
// primary key. The winner's row IS the first-write, and this packet's
|
||||||
|
// content is by construction the same lifecycle state, so the race
|
||||||
|
// outcome is semantically a no-op. Swallow at Debug — the same
|
||||||
|
// check-then-act contract the sibling AuditLog/Notification repos
|
||||||
|
// document. Note the loser's UPDATE leg already ran (against no row),
|
||||||
|
// so nothing is left half-applied.
|
||||||
|
_logger.LogDebug(
|
||||||
|
ex,
|
||||||
|
"SiteCallAuditRepository.UpsertAsync swallowed duplicate-key violation (error {SqlErrorNumber}) for TrackedOperationId {TrackedOperationId}; treating as no-op.",
|
||||||
|
ex.Number,
|
||||||
|
idText);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
public async Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default)
|
public async Task<SiteCall?> GetAsync(TrackedOperationId id, CancellationToken ct = default)
|
||||||
|
|||||||
@@ -26,6 +26,22 @@ public static class ServiceCollectionExtensions
|
|||||||
// registers IDataProtectionProvider as a singleton; resolving it here does not recurse
|
// registers IDataProtectionProvider as a singleton; resolving it here does not recurse
|
||||||
// because key-ring loading is lazy (first Protect/Unprotect), not triggered by
|
// because key-ring loading is lazy (first Protect/Unprotect), not triggered by
|
||||||
// CreateProtector during model building.
|
// CreateProtector during model building.
|
||||||
|
//
|
||||||
|
// POOLING IS DELIBERATELY NOT USED (WP2.2 — verified, not overlooked).
|
||||||
|
// AddDbContextPool requires a context with a SINGLE public constructor taking
|
||||||
|
// only DbContextOptions<TContext>; EF Core constructs pooled instances through
|
||||||
|
// its own activator and cannot supply anything else. ScadaBridgeDbContext has
|
||||||
|
// two public constructors and the runtime one takes IDataProtectionProvider,
|
||||||
|
// because the encrypting value converter for secret-bearing columns is built
|
||||||
|
// during OnModelCreating from that provider. Worse, the model itself DIFFERS
|
||||||
|
// between the two constructors (no provider ⇒ no encrypting converter), so a
|
||||||
|
// pooled activator would silently produce a context that reads secret columns
|
||||||
|
// as ciphertext. Making this poolable means moving the protector out of the
|
||||||
|
// constructor and into a DbContextOptions extension — a change to the
|
||||||
|
// secrets-at-rest path, which is not a performance refactor. The registration
|
||||||
|
// below (a scoped factory overriding AddDbContext's activator) is what makes
|
||||||
|
// the provider reach the context at all, and it also bypasses pooling by
|
||||||
|
// construction. Revisit only alongside a deliberate secrets-plumbing change.
|
||||||
services.AddDbContext<ScadaBridgeDbContext>((serviceProvider, options) =>
|
services.AddDbContext<ScadaBridgeDbContext>((serviceProvider, options) =>
|
||||||
{
|
{
|
||||||
options.UseSqlServer(
|
options.UseSqlServer(
|
||||||
|
|||||||
@@ -158,6 +158,26 @@ public class SiteCallAuditActor : ReceiveActor
|
|||||||
/// </summary>
|
/// </summary>
|
||||||
private readonly Dictionary<string, bool> _reconciliationPinned = new();
|
private readonly Dictionary<string, bool> _reconciliationPinned = new();
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Actor-system EventStream captured on the actor thread at handler-registration
|
||||||
|
/// time. The reconciliation pass runs off-mailbox and publishes the pinned-state
|
||||||
|
/// transition from there, so it must not reach through <c>Context</c>.
|
||||||
|
/// </summary>
|
||||||
|
private Akka.Event.EventStream _eventStream = null!;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Single-flight guard for the off-mailbox reconciliation pass. Raised on the
|
||||||
|
/// actor thread when a tick launches a pass and lowered on the actor thread
|
||||||
|
/// when the piped <see cref="ReconciliationComplete"/> arrives, so the
|
||||||
|
/// per-site cursor/pinned dictionaries the pass mutates are only ever touched
|
||||||
|
/// by one task at a time and the mailbox supplies the memory barrier between
|
||||||
|
/// consecutive passes.
|
||||||
|
/// </summary>
|
||||||
|
private bool _reconciling;
|
||||||
|
|
||||||
|
/// <summary>Single-flight guard for the off-mailbox terminal-row purge pass.</summary>
|
||||||
|
private bool _purging;
|
||||||
|
|
||||||
private ICancelable? _reconciliationTimer;
|
private ICancelable? _reconciliationTimer;
|
||||||
private ICancelable? _purgeTimer;
|
private ICancelable? _purgeTimer;
|
||||||
|
|
||||||
@@ -338,8 +358,79 @@ public class SiteCallAuditActor : ReceiveActor
|
|||||||
// the daily terminal-row purge. Handlers stay alive across faults via
|
// the daily terminal-row purge. Handlers stay alive across faults via
|
||||||
// their own per-site / per-tick try/catch (mirroring the ingest path);
|
// their own per-site / per-tick try/catch (mirroring the ingest path);
|
||||||
// the timers are only started when their collaborators are available.
|
// the timers are only started when their collaborators are available.
|
||||||
ReceiveAsync<ReconciliationTick>(_ => OnReconciliationTickAsync());
|
//
|
||||||
ReceiveAsync<PurgeTick>(_ => OnPurgeTickAsync());
|
// OFF-MAILBOX (WP2.2). Both passes run as PipeTo-completed background
|
||||||
|
// tasks behind a single-flight guard rather than as ReceiveAsync bodies.
|
||||||
|
// A ReceiveAsync handler occupies the actor for its whole duration, and a
|
||||||
|
// reconciliation pass is unbounded work — every site, up to
|
||||||
|
// MaxReconciliationPagesPerTick network pulls each, one upsert per row.
|
||||||
|
// Post-outage catch-up therefore blocked telemetry ingest, UI queries and
|
||||||
|
// KPI Asks behind it until the drain finished, and those callers timed out
|
||||||
|
// rather than queued. NotificationOutboxActor's dispatch sweep is the
|
||||||
|
// in-repo reference for this shape.
|
||||||
|
Receive<ReconciliationTick>(_ => HandleReconciliationTick());
|
||||||
|
Receive<ReconciliationComplete>(_ => _reconciling = false);
|
||||||
|
Receive<PurgeTick>(_ => HandlePurgeTick());
|
||||||
|
Receive<PurgeComplete>(_ => _purging = false);
|
||||||
|
|
||||||
|
// Captured on the actor thread so the background passes never touch
|
||||||
|
// Context off-thread. EventStream itself is thread-safe.
|
||||||
|
_eventStream = Context.System.EventStream;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Launches a reconciliation pass unless one is already in flight, dropping
|
||||||
|
/// the tick if so. Overlapping passes are not merely wasteful — they would
|
||||||
|
/// race on the per-site cursor and pinned-latch dictionaries, which the
|
||||||
|
/// single-flight guard keeps confined to one task at a time (the guard itself
|
||||||
|
/// is only ever mutated on the actor thread: raised here, lowered by the
|
||||||
|
/// piped completion message).
|
||||||
|
/// </summary>
|
||||||
|
private void HandleReconciliationTick()
|
||||||
|
{
|
||||||
|
if (_reconciling)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
_reconciling = true;
|
||||||
|
|
||||||
|
// OnReconciliationTickAsync swallows its own per-site errors, but the
|
||||||
|
// failure projection is kept as a belt-and-braces guard so even a faulted
|
||||||
|
// task still lowers the guard — otherwise reconciliation would wedge
|
||||||
|
// permanently after a single unexpected throw.
|
||||||
|
OnReconciliationTickAsync().PipeTo(
|
||||||
|
Self,
|
||||||
|
success: () => ReconciliationComplete.Instance,
|
||||||
|
failure: ex =>
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "SiteCallAudit reconciliation pass faulted unexpectedly.");
|
||||||
|
return ReconciliationComplete.Instance;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Launches a purge pass unless one is already in flight. Same single-flight
|
||||||
|
/// discipline as <see cref="HandleReconciliationTick"/>: a purge that outlives
|
||||||
|
/// its interval (a large catch-up after an outage) must not stack.
|
||||||
|
/// </summary>
|
||||||
|
private void HandlePurgeTick()
|
||||||
|
{
|
||||||
|
if (_purging)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
_purging = true;
|
||||||
|
|
||||||
|
OnPurgeTickAsync().PipeTo(
|
||||||
|
Self,
|
||||||
|
success: () => PurgeComplete.Instance,
|
||||||
|
failure: ex =>
|
||||||
|
{
|
||||||
|
_logger.LogError(ex, "SiteCallAudit purge pass faulted unexpectedly.");
|
||||||
|
return PurgeComplete.Instance;
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <inheritdoc />
|
/// <inheritdoc />
|
||||||
@@ -743,7 +834,10 @@ public class SiteCallAuditActor : ReceiveActor
|
|||||||
}
|
}
|
||||||
|
|
||||||
_reconciliationPinned[siteId] = pinned;
|
_reconciliationPinned[siteId] = pinned;
|
||||||
Context.System.EventStream.Publish(new SiteCallReconciliationPinnedChanged(siteId, pinned));
|
|
||||||
|
// _eventStream, not Context.System.EventStream: this runs on the
|
||||||
|
// off-mailbox reconciliation pass, where Context must not be touched.
|
||||||
|
_eventStream.Publish(new SiteCallReconciliationPinnedChanged(siteId, pinned));
|
||||||
}
|
}
|
||||||
|
|
||||||
// ── Piece B: daily terminal-row purge scheduler ──
|
// ── Piece B: daily terminal-row purge scheduler ──
|
||||||
@@ -1452,6 +1546,24 @@ public class SiteCallAuditActor : ReceiveActor
|
|||||||
public static readonly PurgeTick Instance = new();
|
public static readonly PurgeTick Instance = new();
|
||||||
private PurgeTick() { }
|
private PurgeTick() { }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Piped back to <c>Self</c> when an off-mailbox reconciliation pass ends
|
||||||
|
/// (successfully or not) so the single-flight guard is lowered on the actor
|
||||||
|
/// thread rather than from the background task.
|
||||||
|
/// </summary>
|
||||||
|
internal sealed class ReconciliationComplete
|
||||||
|
{
|
||||||
|
public static readonly ReconciliationComplete Instance = new();
|
||||||
|
private ReconciliationComplete() { }
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>Purge counterpart of <see cref="ReconciliationComplete"/>.</summary>
|
||||||
|
internal sealed class PurgeComplete
|
||||||
|
{
|
||||||
|
public static readonly PurgeComplete Instance = new();
|
||||||
|
private PurgeComplete() { }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
|
|||||||
@@ -121,6 +121,105 @@ public class AuditLogIngestActorTests : TestKit, IClassFixture<MsSqlMigrationFix
|
|||||||
Assert.Equal(3, count);
|
Assert.Equal(3, count);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[SkippableFact]
|
||||||
|
public async Task Receive_DuplicateEventIds_WithinOnePacket_ProduceOneRow_AndAreAllAcked()
|
||||||
|
{
|
||||||
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||||
|
|
||||||
|
// WP2.2 writes each packet as ONE set-based statement, whose anti-semi-join
|
||||||
|
// only sees already-COMMITTED rows. Two copies of an EventId inside one
|
||||||
|
// packet would therefore both pass it and collide on the clustered PK,
|
||||||
|
// taking the whole packet down — the repository de-duplicates first.
|
||||||
|
// Every id is still acked: the site's contract is "this row is now
|
||||||
|
// present at central", which is true for both copies.
|
||||||
|
var siteId = NewSiteId();
|
||||||
|
var repeated = NewEvent(siteId);
|
||||||
|
var other = NewEvent(siteId);
|
||||||
|
var batch = new List<AuditEvent> { repeated, other, repeated, repeated };
|
||||||
|
|
||||||
|
await using var context = CreateContext();
|
||||||
|
var repo = new AuditLogRepository(context);
|
||||||
|
var actor = CreateActor(repo);
|
||||||
|
|
||||||
|
actor.Tell(new IngestAuditEventsCommand(batch), TestActor);
|
||||||
|
|
||||||
|
var reply = ExpectMsg<IngestAuditEventsReply>(TimeSpan.FromSeconds(10));
|
||||||
|
Assert.Equal(4, reply.AcceptedEventIds.Count);
|
||||||
|
Assert.True(
|
||||||
|
new[] { repeated.EventId, other.EventId }.ToHashSet()
|
||||||
|
.SetEquals(reply.AcceptedEventIds.ToHashSet()));
|
||||||
|
|
||||||
|
await using var readContext = CreateContext();
|
||||||
|
var rows = await readContext.Set<AuditLogRow>()
|
||||||
|
.Where(e => e.SourceSiteId == siteId)
|
||||||
|
.ToListAsync();
|
||||||
|
Assert.Equal(2, rows.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
[SkippableFact]
|
||||||
|
public async Task Receive_SamePacketTwice_IsIdempotent_AcrossPackets()
|
||||||
|
{
|
||||||
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||||
|
|
||||||
|
// A site whose ack was lost re-delivers the identical packet on the next
|
||||||
|
// drain, and the reconciliation pull can re-deliver it a third time. Each
|
||||||
|
// replay must ack fully (so the site can finally flip its rows to
|
||||||
|
// Forwarded) while writing nothing new.
|
||||||
|
var siteId = NewSiteId();
|
||||||
|
var events = Enumerable.Range(0, 6).Select(_ => NewEvent(siteId)).ToList();
|
||||||
|
|
||||||
|
await using var context = CreateContext();
|
||||||
|
var repo = new AuditLogRepository(context);
|
||||||
|
var actor = CreateActor(repo);
|
||||||
|
|
||||||
|
for (var attempt = 0; attempt < 3; attempt++)
|
||||||
|
{
|
||||||
|
actor.Tell(new IngestAuditEventsCommand(events), TestActor);
|
||||||
|
var reply = ExpectMsg<IngestAuditEventsReply>(TimeSpan.FromSeconds(10));
|
||||||
|
Assert.Equal(6, reply.AcceptedEventIds.Count);
|
||||||
|
Assert.True(
|
||||||
|
events.Select(e => e.EventId).ToHashSet()
|
||||||
|
.SetEquals(reply.AcceptedEventIds.ToHashSet()));
|
||||||
|
}
|
||||||
|
|
||||||
|
await using var readContext = CreateContext();
|
||||||
|
var rows = await readContext.Set<AuditLogRow>()
|
||||||
|
.Where(e => e.SourceSiteId == siteId)
|
||||||
|
.ToListAsync();
|
||||||
|
Assert.Equal(6, rows.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
[SkippableFact]
|
||||||
|
public async Task Receive_OverlappingPackets_InsertOnlyTheNewRows()
|
||||||
|
{
|
||||||
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||||
|
|
||||||
|
// Partially-overlapping packets are the normal reconciliation shape: the
|
||||||
|
// pull cursor re-serves a tail the push already delivered. The overlap
|
||||||
|
// must be a silent no-op and the new rows must land.
|
||||||
|
var siteId = NewSiteId();
|
||||||
|
var first = Enumerable.Range(0, 4).Select(_ => NewEvent(siteId)).ToList();
|
||||||
|
var second = first.Skip(2).Concat(
|
||||||
|
Enumerable.Range(0, 3).Select(_ => NewEvent(siteId))).ToList();
|
||||||
|
|
||||||
|
await using var context = CreateContext();
|
||||||
|
var repo = new AuditLogRepository(context);
|
||||||
|
var actor = CreateActor(repo);
|
||||||
|
|
||||||
|
actor.Tell(new IngestAuditEventsCommand(first), TestActor);
|
||||||
|
ExpectMsg<IngestAuditEventsReply>(TimeSpan.FromSeconds(10));
|
||||||
|
|
||||||
|
actor.Tell(new IngestAuditEventsCommand(second), TestActor);
|
||||||
|
var reply = ExpectMsg<IngestAuditEventsReply>(TimeSpan.FromSeconds(10));
|
||||||
|
Assert.Equal(5, reply.AcceptedEventIds.Count);
|
||||||
|
|
||||||
|
await using var readContext = CreateContext();
|
||||||
|
var rows = await readContext.Set<AuditLogRow>()
|
||||||
|
.Where(e => e.SourceSiteId == siteId)
|
||||||
|
.ToListAsync();
|
||||||
|
Assert.Equal(7, rows.Count);
|
||||||
|
}
|
||||||
|
|
||||||
[SkippableFact]
|
[SkippableFact]
|
||||||
public async Task Receive_Sets_IngestedAtUtc_Before_Insert()
|
public async Task Receive_Sets_IngestedAtUtc_Before_Insert()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -28,20 +28,19 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Integration;
|
|||||||
/// <list type="number">
|
/// <list type="number">
|
||||||
/// <item>The oldest partition (Jan) is removed.</item>
|
/// <item>The oldest partition (Jan) is removed.</item>
|
||||||
/// <item>Newer partitions (Feb + Mar) are untouched.</item>
|
/// <item>Newer partitions (Feb + Mar) are untouched.</item>
|
||||||
/// <item>The <c>UX_AuditLog_EventId</c> unique index survives the
|
/// <item>The switch leaves the aligned clustered <c>PK_AuditLog</c> intact and
|
||||||
/// drop-and-rebuild dance.</item>
|
/// does NOT (re)create the non-aligned <c>UX_AuditLog_EventId</c>.</item>
|
||||||
/// <item><see cref="IAuditLogRepository.InsertIfNotExistsAsync"/> remains
|
/// <item><see cref="IAuditLogRepository.InsertIfNotExistsAsync"/> remains
|
||||||
/// idempotent against the rebuilt index after the purge.</item>
|
/// idempotent against the aligned key after the purge.</item>
|
||||||
/// </list>
|
/// </list>
|
||||||
/// </summary>
|
/// </summary>
|
||||||
/// <remarks>
|
/// <remarks>
|
||||||
/// The brief calls out that direct INSERTs bypass the writer role's INSERT-only
|
/// The brief calls out that direct INSERTs bypass the writer role's INSERT-only
|
||||||
/// grant; the fixture connects as <c>sa</c> (see
|
/// grant; the fixture connects as <c>sa</c> (see
|
||||||
/// <see cref="MsSqlMigrationFixture"/>'s default admin connection string), so
|
/// <see cref="MsSqlMigrationFixture"/>'s default admin connection string), so
|
||||||
/// the seed step does not need the writer role at all. The drop-and-rebuild
|
/// the seed step does not need the writer role at all. The switch batch itself
|
||||||
/// dance itself runs under the same admin connection because the test owns
|
/// runs under the same admin connection because the test owns the database —
|
||||||
/// the database — the role granularity is exercised in the repository tests,
|
/// the role granularity is exercised in the repository tests, not here.
|
||||||
/// not here.
|
|
||||||
/// </remarks>
|
/// </remarks>
|
||||||
public class PartitionPurgeTests : TestKit, IClassFixture<MsSqlMigrationFixture>
|
public class PartitionPurgeTests : TestKit, IClassFixture<MsSqlMigrationFixture>
|
||||||
{
|
{
|
||||||
@@ -110,22 +109,39 @@ VALUES
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Asserts that <c>UX_AuditLog_EventId</c> exists in
|
/// Asserts the post-purge index state: the partition-ALIGNED clustered
|
||||||
/// <c>sys.indexes</c>. The drop-and-rebuild dance briefly removes the
|
/// <c>PK_AuditLog</c> is intact and the non-aligned <c>UX_AuditLog_EventId</c>
|
||||||
/// index inside its transaction; this check is meant to fire AFTER the
|
/// is absent (WP2.2 — <c>AlignAuditLogEventIdUniqueness</c>).
|
||||||
/// actor's purge tick has committed so the rebuilt index is observable.
|
|
||||||
/// </summary>
|
/// </summary>
|
||||||
private static async Task AssertUxIndexExistsAsync(SqlConnection conn)
|
/// <remarks>
|
||||||
|
/// The purge used to bracket its SWITCH with a DROP/CREATE of
|
||||||
|
/// <c>UX_AuditLog_EventId</c>, because a non-aligned unique index blocks
|
||||||
|
/// <c>ALTER TABLE … SWITCH PARTITION</c> — an offline whole-table index build
|
||||||
|
/// inside the switch transaction, plus a window with no idempotency index at
|
||||||
|
/// all. Uniqueness now rides the aligned clustered PK, so the switch is
|
||||||
|
/// metadata-only. Asserting the index's ABSENCE is what keeps that property:
|
||||||
|
/// anything that recreates it silently reinstates the rebuild.
|
||||||
|
/// </remarks>
|
||||||
|
private static async Task AssertAlignedUniquenessAsync(SqlConnection conn)
|
||||||
{
|
{
|
||||||
await using var cmd = conn.CreateCommand();
|
await using var cmd = conn.CreateCommand();
|
||||||
cmd.CommandText = @"
|
cmd.CommandText = @"
|
||||||
SELECT COUNT(*)
|
SELECT
|
||||||
FROM sys.indexes
|
(SELECT COUNT(*) FROM sys.indexes
|
||||||
WHERE name = 'UX_AuditLog_EventId'
|
WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog')) AS NonAligned,
|
||||||
AND object_id = OBJECT_ID('dbo.AuditLog');";
|
(SELECT COUNT(*) FROM sys.indexes
|
||||||
var raw = await cmd.ExecuteScalarAsync();
|
WHERE object_id = OBJECT_ID('dbo.AuditLog') AND is_primary_key = 1) AS ClusteredPk;";
|
||||||
var count = Convert.ToInt32(raw);
|
await using var reader = await cmd.ExecuteReaderAsync();
|
||||||
Assert.True(count == 1, $"UX_AuditLog_EventId should be present post-purge; sys.indexes count was {count}.");
|
Assert.True(await reader.ReadAsync());
|
||||||
|
var nonAligned = reader.GetInt32(0);
|
||||||
|
var clusteredPk = reader.GetInt32(1);
|
||||||
|
|
||||||
|
Assert.True(
|
||||||
|
nonAligned == 0,
|
||||||
|
$"UX_AuditLog_EventId must NOT exist post-purge (it blocks SWITCH PARTITION); sys.indexes count was {nonAligned}.");
|
||||||
|
Assert.True(
|
||||||
|
clusteredPk == 1,
|
||||||
|
$"The aligned clustered PK_AuditLog must survive the purge; sys.indexes primary-key count was {clusteredPk}.");
|
||||||
}
|
}
|
||||||
|
|
||||||
private IActorRef CreateActor(
|
private IActorRef CreateActor(
|
||||||
@@ -255,20 +271,19 @@ WHERE name = 'UX_AuditLog_EventId'
|
|||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
// 2. EndToEnd_UxIndexRebuilt_AfterPurge
|
// 2. EndToEnd_AlignedUniqueness_AfterPurge
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
|
|
||||||
[SkippableFact]
|
[SkippableFact]
|
||||||
public async Task EndToEnd_UxIndexRebuilt_AfterPurge()
|
public async Task EndToEnd_AlignedUniqueness_AfterPurge()
|
||||||
{
|
{
|
||||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||||
|
|
||||||
// Same shape as test 1 — purge the Jan-2026 partition and then assert the
|
// Same shape as test 1 — purge the Jan-2026 partition and then assert the
|
||||||
// UX_AuditLog_EventId index is still present. RetentionDays is computed
|
// index state. RetentionDays is computed dynamically so the threshold always
|
||||||
// dynamically so the threshold always lands near 2026-01-20 (see SeedOccurredAt()).
|
// lands near 2026-01-20 (see SeedOccurredAt()). Since WP2.2 the switch touches
|
||||||
// The drop-and-rebuild dance briefly removes the index inside its transaction
|
// no index at all: uniqueness rides the aligned clustered PK, so there is
|
||||||
// (the SWITCH PARTITION step requires the non-aligned unique index to be absent),
|
// nothing to drop before the SWITCH and nothing to rebuild after it.
|
||||||
// but step 5 rebuilds it before committing.
|
|
||||||
var siteId = "purge-uxidx-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
var siteId = "purge-uxidx-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||||
var oldEventId = Guid.NewGuid();
|
var oldEventId = Guid.NewGuid();
|
||||||
var (oldOccurred, _, _, retentionDays) = SeedOccurredAt();
|
var (oldOccurred, _, _, retentionDays) = SeedOccurredAt();
|
||||||
@@ -305,7 +320,7 @@ WHERE name = 'UX_AuditLog_EventId'
|
|||||||
// Open a fresh connection (the actor's pool is owned by EF) and
|
// Open a fresh connection (the actor's pool is owned by EF) and
|
||||||
// assert the index is present post-purge.
|
// assert the index is present post-purge.
|
||||||
await using var check = _fixture.OpenConnection();
|
await using var check = _fixture.OpenConnection();
|
||||||
await AssertUxIndexExistsAsync(check);
|
await AssertAlignedUniquenessAsync(check);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------
|
// ---------------------------------------------------------------------
|
||||||
@@ -320,7 +335,7 @@ WHERE name = 'UX_AuditLog_EventId'
|
|||||||
// Seed + purge the Jan-2026 row, THEN exercise InsertIfNotExistsAsync twice for
|
// Seed + purge the Jan-2026 row, THEN exercise InsertIfNotExistsAsync twice for
|
||||||
// a fresh recent EventId. The second call must be a no-op (duplicate-key collision
|
// a fresh recent EventId. The second call must be a no-op (duplicate-key collision
|
||||||
// swallowed by the repository, per M2 Bundle A's race-fix) — which means the
|
// swallowed by the repository, per M2 Bundle A's race-fix) — which means the
|
||||||
// rebuilt UX_AuditLog_EventId unique index is functioning as intended.
|
// aligned clustered PK is still enforcing uniqueness as intended.
|
||||||
// RetentionDays is computed dynamically so the threshold always lands near
|
// RetentionDays is computed dynamically so the threshold always lands near
|
||||||
// 2026-01-20 (see SeedOccurredAt()).
|
// 2026-01-20 (see SeedOccurredAt()).
|
||||||
var siteId = "purge-idem-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
var siteId = "purge-idem-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||||
@@ -357,11 +372,11 @@ WHERE name = 'UX_AuditLog_EventId'
|
|||||||
max: TimeSpan.FromSeconds(30));
|
max: TimeSpan.FromSeconds(30));
|
||||||
|
|
||||||
// Settle then exercise InsertIfNotExistsAsync twice for the same
|
// Settle then exercise InsertIfNotExistsAsync twice for the same
|
||||||
// EventId. The repository's idempotency relies on
|
// EventId. The repository's idempotency relies on the aligned clustered
|
||||||
// UX_AuditLog_EventId being present so the IF NOT EXISTS … INSERT
|
// PK (EventId, OccurredAtUtc) being intact so the IF NOT EXISTS … INSERT
|
||||||
// race window resolves to a duplicate-key violation the repo
|
// race window resolves to a duplicate-key violation the repo swallows.
|
||||||
// swallows. If the index were missing here, two rows would land
|
// If the switch had disturbed that key, two rows would land and the second
|
||||||
// and the second InsertIfNotExistsAsync would silently double-insert.
|
// InsertIfNotExistsAsync would silently double-insert.
|
||||||
await Task.Delay(TimeSpan.FromMilliseconds(500));
|
await Task.Delay(TimeSpan.FromMilliseconds(500));
|
||||||
|
|
||||||
var freshEventId = Guid.NewGuid();
|
var freshEventId = Guid.NewGuid();
|
||||||
@@ -482,7 +497,7 @@ WHERE name = 'UX_AuditLog_EventId'
|
|||||||
/// <summary>
|
/// <summary>
|
||||||
/// Task 4 (arch-review 04, S2): proves the explicit-<see cref="TimeSpan"/> maintenance-timeout
|
/// Task 4 (arch-review 04, S2): proves the explicit-<see cref="TimeSpan"/> maintenance-timeout
|
||||||
/// overload of <see cref="IAuditLogRepository.SwitchOutPartitionAsync"/> runs the real
|
/// overload of <see cref="IAuditLogRepository.SwitchOutPartitionAsync"/> runs the real
|
||||||
/// drop-and-rebuild dance to completion against SQL Server — the old row is purged, the kept row
|
/// staging/switch batch to completion against SQL Server — the old row is purged, the kept row
|
||||||
/// survives, and the returned sampled row-count reflects the switched partition. Exercising the
|
/// survives, and the returned sampled row-count reflects the switched partition. Exercising the
|
||||||
/// timeout path end-to-end guards against a regression that only sets the timeout on the sample
|
/// timeout path end-to-end guards against a regression that only sets the timeout on the sample
|
||||||
/// command and forgets the DDL batch (or vice versa).
|
/// command and forgets the DDL batch (or vice versa).
|
||||||
@@ -520,8 +535,8 @@ WHERE name = 'UX_AuditLog_EventId'
|
|||||||
Assert.DoesNotContain(rows, r => r.EventId == oldEventId);
|
Assert.DoesNotContain(rows, r => r.EventId == oldEventId);
|
||||||
Assert.Contains(rows, r => r.EventId == keptEventId);
|
Assert.Contains(rows, r => r.EventId == keptEventId);
|
||||||
|
|
||||||
// The dance must leave the idempotency-supporting unique index rebuilt.
|
// The switch must leave uniqueness enforcement exactly as it found it.
|
||||||
await using var check = _fixture.OpenConnection();
|
await using var check = _fixture.OpenConnection();
|
||||||
await AssertUxIndexExistsAsync(check);
|
await AssertAlignedUniquenessAsync(check);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+22
-12
@@ -52,22 +52,31 @@ public class AuditLogEntityTypeConfigurationTests : IDisposable
|
|||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public void Configure_DeclaresUniqueIndex_OnEventIdAlone_ForIdempotencyLookups()
|
public void Configure_DeclaresNoNonAlignedEventIdIndex_UniquenessRidesTheAlignedClusteredKey()
|
||||||
{
|
{
|
||||||
// EventId remains globally unique (the idempotency key for
|
// WP2.2 (AlignAuditLogEventIdUniqueness): the standalone non-aligned
|
||||||
// InsertIfNotExistsAsync) via a dedicated unique index independent of the
|
// UX_AuditLog_EventId is gone. EventId uniqueness now rides the clustered
|
||||||
// composite PK.
|
// PK (EventId, OccurredAtUtc), which is partition-aligned on
|
||||||
|
// ps_AuditLog_Month — so ALTER TABLE ... SWITCH PARTITION no longer needs
|
||||||
|
// an offline index drop/rebuild around every retention purge. EventId is a
|
||||||
|
// GUID minted once at the site alongside OccurredAtUtc, so pair-uniqueness
|
||||||
|
// is EventId-uniqueness in practice, and the idempotency probe
|
||||||
|
// (WHERE EventId = @id) still seeks the clustered key's leading column.
|
||||||
|
//
|
||||||
|
// Re-declaring a single-column unique index here would silently reinstate
|
||||||
|
// the SWITCH incompatibility, so this test pins its absence.
|
||||||
var entity = _context.Model.FindEntityType(typeof(AuditLogRow));
|
var entity = _context.Model.FindEntityType(typeof(AuditLogRow));
|
||||||
Assert.NotNull(entity);
|
Assert.NotNull(entity);
|
||||||
|
|
||||||
var eventIdIndex = entity!.GetIndexes()
|
Assert.DoesNotContain(
|
||||||
.SingleOrDefault(i => i.GetDatabaseName() == "UX_AuditLog_EventId");
|
entity!.GetIndexes(),
|
||||||
|
i => i.GetDatabaseName() == "UX_AuditLog_EventId");
|
||||||
|
|
||||||
Assert.NotNull(eventIdIndex);
|
var pk = entity.FindPrimaryKey();
|
||||||
Assert.True(eventIdIndex!.IsUnique);
|
Assert.NotNull(pk);
|
||||||
|
Assert.Equal(
|
||||||
var indexedProperty = Assert.Single(eventIdIndex.Properties);
|
new[] { nameof(AuditLogRow.EventId), nameof(AuditLogRow.OccurredAtUtc) },
|
||||||
Assert.Equal(nameof(AuditLogRow.EventId), indexedProperty.Name);
|
pk!.Properties.Select(p => p.Name).ToArray());
|
||||||
}
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
@@ -164,7 +173,8 @@ public class AuditLogEntityTypeConfigurationTests : IDisposable
|
|||||||
"IX_AuditLog_ParentExecution",
|
"IX_AuditLog_ParentExecution",
|
||||||
"IX_AuditLog_Site_Occurred",
|
"IX_AuditLog_Site_Occurred",
|
||||||
"IX_AuditLog_Target_Occurred",
|
"IX_AuditLog_Target_Occurred",
|
||||||
"UX_AuditLog_EventId",
|
// UX_AuditLog_EventId is intentionally absent — dropped by
|
||||||
|
// AlignAuditLogEventIdUniqueness; the aligned clustered PK carries it.
|
||||||
};
|
};
|
||||||
|
|
||||||
Assert.Equal(expected, indexNames);
|
Assert.Equal(expected, indexNames);
|
||||||
|
|||||||
+22
-8
@@ -85,22 +85,27 @@ public class AddAuditLogTableMigrationTests : IClassFixture<MsSqlMigrationFixtur
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Verifies all nine named non-clustered indexes exist on the final
|
/// Verifies all eight named non-clustered indexes exist on the final
|
||||||
/// <c>dbo.AuditLog</c> table after all migrations have been applied.
|
/// <c>dbo.AuditLog</c> table after all migrations have been applied, and that
|
||||||
|
/// the non-aligned <c>UX_AuditLog_EventId</c> does NOT.
|
||||||
/// The original five indexes were created by <c>AddAuditLogTable</c>;
|
/// The original five indexes were created by <c>AddAuditLogTable</c>;
|
||||||
/// the <c>CollapseAuditLogToCanonical</c> (C5, Task 2.5) migration rebuilt
|
/// the <c>CollapseAuditLogToCanonical</c> (C5, Task 2.5) migration rebuilt
|
||||||
/// the table and added <c>IX_AuditLog_Execution</c>,
|
/// the table and added <c>IX_AuditLog_Execution</c>,
|
||||||
/// <c>IX_AuditLog_ParentExecution</c>, <c>IX_AuditLog_Node_Occurred</c>,
|
/// <c>IX_AuditLog_ParentExecution</c> and <c>IX_AuditLog_Node_Occurred</c>.
|
||||||
/// and <c>UX_AuditLog_EventId</c> — nine in total.
|
/// <c>UX_AuditLog_EventId</c> was created alongside them but dropped again by
|
||||||
|
/// <c>AlignAuditLogEventIdUniqueness</c> (WP2.2) — its non-alignment forced an
|
||||||
|
/// offline drop/rebuild around every partition-switch purge, and the clustered
|
||||||
|
/// <c>PK_AuditLog (EventId, OccurredAtUtc)</c> already enforces the same
|
||||||
|
/// uniqueness partition-aligned.
|
||||||
/// </summary>
|
/// </summary>
|
||||||
[SkippableFact]
|
[SkippableFact]
|
||||||
public async Task AppliesMigration_CreatesNineNamedIndexes()
|
public async Task AppliesMigration_CreatesEightNamedIndexes_AndNoNonAlignedUniqueIndex()
|
||||||
{
|
{
|
||||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||||
|
|
||||||
// All nine named non-clustered indexes present on dbo.AuditLog after
|
// All eight named non-clustered indexes present on dbo.AuditLog after
|
||||||
// the full migration history is applied (AddAuditLogTable through
|
// the full migration history is applied (AddAuditLogTable through
|
||||||
// CollapseAuditLogToCanonical).
|
// AlignAuditLogEventIdUniqueness).
|
||||||
var expected = new[]
|
var expected = new[]
|
||||||
{
|
{
|
||||||
// Original five (AddAuditLogTable / AddAuditLogSourceNode):
|
// Original five (AddAuditLogTable / AddAuditLogSourceNode):
|
||||||
@@ -113,7 +118,6 @@ public class AddAuditLogTableMigrationTests : IClassFixture<MsSqlMigrationFixtur
|
|||||||
"IX_AuditLog_Execution",
|
"IX_AuditLog_Execution",
|
||||||
"IX_AuditLog_ParentExecution",
|
"IX_AuditLog_ParentExecution",
|
||||||
"IX_AuditLog_Node_Occurred",
|
"IX_AuditLog_Node_Occurred",
|
||||||
"UX_AuditLog_EventId",
|
|
||||||
};
|
};
|
||||||
|
|
||||||
foreach (var indexName in expected)
|
foreach (var indexName in expected)
|
||||||
@@ -124,6 +128,16 @@ public class AddAuditLogTableMigrationTests : IClassFixture<MsSqlMigrationFixtur
|
|||||||
$"WHERE o.name = 'AuditLog' AND i.name = '{indexName}';");
|
$"WHERE o.name = 'AuditLog' AND i.name = '{indexName}';");
|
||||||
Assert.True(count == 1, $"Expected index '{indexName}' to exist on AuditLog; found {count}.");
|
Assert.True(count == 1, $"Expected index '{indexName}' to exist on AuditLog; found {count}.");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var nonAligned = await ScalarAsync<int>(
|
||||||
|
"SELECT COUNT(*) FROM sys.indexes i " +
|
||||||
|
"INNER JOIN sys.objects o ON i.object_id = o.object_id " +
|
||||||
|
"WHERE o.name = 'AuditLog' AND i.name = 'UX_AuditLog_EventId';");
|
||||||
|
Assert.True(
|
||||||
|
nonAligned == 0,
|
||||||
|
"UX_AuditLog_EventId must NOT exist after AlignAuditLogEventIdUniqueness — "
|
||||||
|
+ "a non-aligned index blocks ALTER TABLE ... SWITCH PARTITION and reinstates "
|
||||||
|
+ $"the offline drop/rebuild the purge path was freed from; found {nonAligned}.");
|
||||||
}
|
}
|
||||||
|
|
||||||
[SkippableFact]
|
[SkippableFact]
|
||||||
|
|||||||
+10
@@ -50,6 +50,16 @@ public class NotificationOutboxRepositoryKpiQueryShapeTests
|
|||||||
$"ComputeKpisAsync issued {counter.Count} queries against Notifications; expected <= 2");
|
$"ComputeKpisAsync issued {counter.Count} queries against Notifications; expected <= 2");
|
||||||
// The oldest lookup must be bounded (LIMIT/TOP), never a full non-terminal SELECT.
|
// The oldest lookup must be bounded (LIMIT/TOP), never a full non-terminal SELECT.
|
||||||
Assert.Contains(counter.Commands, sql => sql.Contains("LIMIT", StringComparison.OrdinalIgnoreCase));
|
Assert.Contains(counter.Commands, sql => sql.Contains("LIMIT", StringComparison.OrdinalIgnoreCase));
|
||||||
|
// WP2.2: the aggregation must be PREDICATE-RESTRICTED, not an unrestricted
|
||||||
|
// scan. Every KPI here concerns the live queue, the parked backlog or the
|
||||||
|
// last delivery interval; historical Delivered/Discarded rows — which are
|
||||||
|
// the overwhelming bulk of a retained table — contribute to none of them.
|
||||||
|
// Without the WHERE the query's cost grows with retention rather than with
|
||||||
|
// the working set. (An index-restricted aggregation was already the shape
|
||||||
|
// of the per-site/per-node snapshots; the global one was the odd one out.)
|
||||||
|
Assert.All(
|
||||||
|
counter.Commands.Where(sql => sql.Contains("COUNT", StringComparison.OrdinalIgnoreCase)),
|
||||||
|
sql => Assert.Contains("WHERE", sql, StringComparison.OrdinalIgnoreCase));
|
||||||
Assert.Equal(3, global.QueueDepth);
|
Assert.Equal(3, global.QueueDepth);
|
||||||
Assert.NotNull(global.OldestPendingAge);
|
Assert.NotNull(global.OldestPendingAge);
|
||||||
|
|
||||||
|
|||||||
+198
-18
@@ -560,6 +560,164 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
|
|||||||
Assert.Equal(1, count);
|
Assert.Equal(1, count);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ------------------------------------------------------------------------
|
||||||
|
// WP2.2: set-based InsertManyIfNotExistsAsync
|
||||||
|
// ------------------------------------------------------------------------
|
||||||
|
|
||||||
|
[SkippableFact]
|
||||||
|
public async Task InsertManyIfNotExistsAsync_WritesEveryDistinctEvent()
|
||||||
|
{
|
||||||
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||||
|
|
||||||
|
var siteId = NewSiteId();
|
||||||
|
await using var context = CreateContext();
|
||||||
|
var repo = new AuditLogRepository(context);
|
||||||
|
|
||||||
|
var baseTime = new DateTime(2026, 5, 21, 8, 0, 0, DateTimeKind.Utc);
|
||||||
|
var events = Enumerable.Range(0, 12)
|
||||||
|
.Select(i => NewEvent(siteId, occurredAtUtc: baseTime.AddSeconds(i)))
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
var inserted = await repo.InsertManyIfNotExistsAsync(events);
|
||||||
|
|
||||||
|
Assert.Equal(12, inserted);
|
||||||
|
|
||||||
|
await using var readContext = CreateContext();
|
||||||
|
var rows = await readContext.Set<AuditLogRow>()
|
||||||
|
.Where(e => e.SourceSiteId == siteId)
|
||||||
|
.ToListAsync();
|
||||||
|
|
||||||
|
Assert.Equal(12, rows.Count);
|
||||||
|
Assert.Equal(
|
||||||
|
events.Select(e => e.EventId).OrderBy(g => g).ToArray(),
|
||||||
|
rows.Select(r => r.EventId).OrderBy(g => g).ToArray());
|
||||||
|
}
|
||||||
|
|
||||||
|
[SkippableFact]
|
||||||
|
public async Task InsertManyIfNotExistsAsync_DuplicateEventIdsWithinOnePacket_CollapseToOneRow()
|
||||||
|
{
|
||||||
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||||
|
|
||||||
|
var siteId = NewSiteId();
|
||||||
|
await using var context = CreateContext();
|
||||||
|
var repo = new AuditLogRepository(context);
|
||||||
|
|
||||||
|
// A set-based INSERT … SELECT … WHERE NOT EXISTS only tests rows that are
|
||||||
|
// already COMMITTED, so two copies of one EventId inside a single VALUES
|
||||||
|
// constructor would both pass the anti-semi-join and collide on the
|
||||||
|
// clustered PK — taking the whole statement down with them. The repository
|
||||||
|
// de-duplicates the packet first (first-write-wins, matching the
|
||||||
|
// single-row contract), which this pins.
|
||||||
|
var occurred = new DateTime(2026, 5, 21, 9, 0, 0, DateTimeKind.Utc);
|
||||||
|
var first = NewEvent(siteId, occurredAtUtc: occurred);
|
||||||
|
var duplicate = ScadaBridgeAuditEventFactory.Create(
|
||||||
|
channel: AuditChannel.ApiOutbound,
|
||||||
|
kind: AuditKind.ApiCall,
|
||||||
|
status: AuditStatus.Delivered,
|
||||||
|
eventId: first.EventId,
|
||||||
|
occurredAtUtc: occurred,
|
||||||
|
sourceSiteId: siteId,
|
||||||
|
errorMessage: "duplicate-within-packet-should-be-ignored");
|
||||||
|
var other = NewEvent(siteId, occurredAtUtc: occurred.AddSeconds(1));
|
||||||
|
|
||||||
|
var inserted = await repo.InsertManyIfNotExistsAsync(
|
||||||
|
new[] { first, duplicate, other, duplicate });
|
||||||
|
|
||||||
|
Assert.Equal(2, inserted);
|
||||||
|
|
||||||
|
await using var readContext = CreateContext();
|
||||||
|
var rows = await readContext.Set<AuditLogRow>()
|
||||||
|
.Where(e => e.SourceSiteId == siteId)
|
||||||
|
.ToListAsync();
|
||||||
|
|
||||||
|
Assert.Equal(2, rows.Count);
|
||||||
|
|
||||||
|
// First-write-wins: the surviving row is the FIRST occurrence, so the
|
||||||
|
// duplicate's ErrorMessage never lands.
|
||||||
|
var stored = Assert.Single(rows, r => r.EventId == first.EventId);
|
||||||
|
Assert.Null(AuditDetailsCodec.Deserialize(stored.DetailsJson).ErrorMessage);
|
||||||
|
}
|
||||||
|
|
||||||
|
[SkippableFact]
|
||||||
|
public async Task InsertManyIfNotExistsAsync_DuplicateEventIdsAcrossPackets_AreIdempotent()
|
||||||
|
{
|
||||||
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||||
|
|
||||||
|
var siteId = NewSiteId();
|
||||||
|
await using var context = CreateContext();
|
||||||
|
var repo = new AuditLogRepository(context);
|
||||||
|
|
||||||
|
var baseTime = new DateTime(2026, 5, 21, 10, 0, 0, DateTimeKind.Utc);
|
||||||
|
var packet = Enumerable.Range(0, 5)
|
||||||
|
.Select(i => NewEvent(siteId, occurredAtUtc: baseTime.AddSeconds(i)))
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
// A site retry / reconciliation pull re-delivers a packet that overlaps
|
||||||
|
// an already-ingested one. The second call must insert only the genuinely
|
||||||
|
// new rows and silently skip the rest (first-write-wins across packets).
|
||||||
|
var firstInserted = await repo.InsertManyIfNotExistsAsync(packet);
|
||||||
|
var extra = NewEvent(siteId, occurredAtUtc: baseTime.AddSeconds(99));
|
||||||
|
var secondInserted = await repo.InsertManyIfNotExistsAsync(
|
||||||
|
packet.Concat(new[] { extra }).ToList());
|
||||||
|
|
||||||
|
// A third, wholly-redundant replay writes nothing at all.
|
||||||
|
var thirdInserted = await repo.InsertManyIfNotExistsAsync(packet);
|
||||||
|
|
||||||
|
Assert.Equal(5, firstInserted);
|
||||||
|
Assert.Equal(1, secondInserted);
|
||||||
|
Assert.Equal(0, thirdInserted);
|
||||||
|
|
||||||
|
await using var readContext = CreateContext();
|
||||||
|
var rows = await readContext.Set<AuditLogRow>()
|
||||||
|
.Where(e => e.SourceSiteId == siteId)
|
||||||
|
.ToListAsync();
|
||||||
|
|
||||||
|
Assert.Equal(6, rows.Count);
|
||||||
|
}
|
||||||
|
|
||||||
|
[SkippableFact]
|
||||||
|
public async Task InsertManyIfNotExistsAsync_ChunksBeyondTheParameterCeiling()
|
||||||
|
{
|
||||||
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||||
|
|
||||||
|
var siteId = NewSiteId();
|
||||||
|
await using var context = CreateContext();
|
||||||
|
var repo = new AuditLogRepository(context);
|
||||||
|
|
||||||
|
// 250 rows × 10 bound parameters = 2,500 — past SQL Server's 2,100
|
||||||
|
// parameter ceiling for a single statement, so this only succeeds if the
|
||||||
|
// repository chunks. Nulls in the optional columns (Target, SourceNode,
|
||||||
|
// CorrelationId, Actor) are left null on purpose: the VALUES constructor
|
||||||
|
// derives its column types from the parameters, so an untyped null would
|
||||||
|
// surface here as a conversion failure rather than silently.
|
||||||
|
var baseTime = new DateTime(2026, 5, 21, 11, 0, 0, DateTimeKind.Utc);
|
||||||
|
var events = Enumerable.Range(0, 250)
|
||||||
|
.Select(i => NewEvent(siteId, occurredAtUtc: baseTime.AddSeconds(i)))
|
||||||
|
.ToList();
|
||||||
|
|
||||||
|
var inserted = await repo.InsertManyIfNotExistsAsync(events);
|
||||||
|
|
||||||
|
Assert.Equal(250, inserted);
|
||||||
|
|
||||||
|
await using var readContext = CreateContext();
|
||||||
|
var count = await readContext.Set<AuditLogRow>()
|
||||||
|
.Where(e => e.SourceSiteId == siteId)
|
||||||
|
.CountAsync();
|
||||||
|
|
||||||
|
Assert.Equal(250, count);
|
||||||
|
}
|
||||||
|
|
||||||
|
[SkippableFact]
|
||||||
|
public async Task InsertManyIfNotExistsAsync_EmptyBatch_IsANoOp()
|
||||||
|
{
|
||||||
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||||
|
|
||||||
|
await using var context = CreateContext();
|
||||||
|
var repo = new AuditLogRepository(context);
|
||||||
|
|
||||||
|
Assert.Equal(0, await repo.InsertManyIfNotExistsAsync(Array.Empty<AuditEvent>()));
|
||||||
|
}
|
||||||
|
|
||||||
[SkippableFact]
|
[SkippableFact]
|
||||||
public async Task QueryAsync_Keyset_SameOccurredAtUtc_TiebreaksOnEventId()
|
public async Task QueryAsync_Keyset_SameOccurredAtUtc_TiebreaksOnEventId()
|
||||||
{
|
{
|
||||||
@@ -622,16 +780,17 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
|
|||||||
// ------------------------------------------------------------------------
|
// ------------------------------------------------------------------------
|
||||||
//
|
//
|
||||||
// The partition-switch path replaces M1's NotSupportedException stub with
|
// The partition-switch path replaces M1's NotSupportedException stub with
|
||||||
// the production drop-DROP-INDEX → CREATE-staging → SWITCH PARTITION →
|
// the production CREATE-staging → SWITCH PARTITION → DROP-staging batch
|
||||||
// DROP-staging → CREATE-INDEX dance documented in alog.md §4. These tests
|
// documented in alog.md §4. WP2.2 removed the index drop/rebuild that used to
|
||||||
|
// bracket it: uniqueness rides the partition-ALIGNED clustered PK
|
||||||
|
// (EventId, OccurredAtUtc), so SWITCH has nothing to object to. These tests
|
||||||
// verify the side effects an outsider can observe:
|
// verify the side effects an outsider can observe:
|
||||||
// * rows in the targeted month are removed
|
// * rows in the targeted month are removed
|
||||||
// * rows in OTHER months are NOT touched
|
// * rows in OTHER months are NOT touched
|
||||||
// * UX_AuditLog_EventId still exists after a successful switch
|
// * no non-aligned UX_AuditLog_EventId is (re)created by a switch
|
||||||
// * InsertIfNotExistsAsync's first-write-wins idempotency still holds
|
// * InsertIfNotExistsAsync's first-write-wins idempotency still holds
|
||||||
// after a switch (the rebuilt index is real)
|
// after a switch (the aligned key really does enforce it)
|
||||||
// * a thrown SqlException leaves UX_AuditLog_EventId rebuilt (the CATCH
|
// * a thrown SqlException leaves the table intact with no orphaned staging
|
||||||
// branch's recovery path runs)
|
|
||||||
|
|
||||||
[SkippableFact]
|
[SkippableFact]
|
||||||
public async Task SwitchOutPartitionAsync_OldPartition_RemovesRows_NewPartitionsKept()
|
public async Task SwitchOutPartitionAsync_OldPartition_RemovesRows_NewPartitionsKept()
|
||||||
@@ -669,7 +828,7 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
|
|||||||
}
|
}
|
||||||
|
|
||||||
[SkippableFact]
|
[SkippableFact]
|
||||||
public async Task SwitchOutPartitionAsync_RebuildsUxIndex_AfterSwitch()
|
public async Task SwitchOutPartitionAsync_LeavesNoNonAlignedUniqueIndex_AfterSwitch()
|
||||||
{
|
{
|
||||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||||
|
|
||||||
@@ -680,12 +839,23 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
|
|||||||
// the fixture's MSSQL database) don't tread on each other.
|
// the fixture's MSSQL database) don't tread on each other.
|
||||||
await repo.SwitchOutPartitionAsync(new DateTime(2026, 4, 1, 0, 0, 0, DateTimeKind.Utc));
|
await repo.SwitchOutPartitionAsync(new DateTime(2026, 4, 1, 0, 0, 0, DateTimeKind.Utc));
|
||||||
|
|
||||||
|
// WP2.2: the switch must NOT recreate UX_AuditLog_EventId. Re-adding it
|
||||||
|
// would reinstate the offline drop/rebuild the purge was freed from and
|
||||||
|
// block the next SWITCH until something dropped it again.
|
||||||
await using var verifyContext = CreateContext();
|
await using var verifyContext = CreateContext();
|
||||||
var indexExists = await ScalarAsync<int>(
|
var indexExists = await ScalarAsync<int>(
|
||||||
verifyContext,
|
verifyContext,
|
||||||
"SELECT COUNT(*) FROM sys.indexes " +
|
"SELECT COUNT(*) FROM sys.indexes " +
|
||||||
"WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog');");
|
"WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog');");
|
||||||
Assert.Equal(1, indexExists);
|
Assert.Equal(0, indexExists);
|
||||||
|
|
||||||
|
// The aligned clustered PK is what enforces uniqueness now, and the
|
||||||
|
// switch must leave it untouched.
|
||||||
|
var pkExists = await ScalarAsync<int>(
|
||||||
|
verifyContext,
|
||||||
|
"SELECT COUNT(*) FROM sys.indexes " +
|
||||||
|
"WHERE name = 'PK_AuditLog' AND object_id = OBJECT_ID('dbo.AuditLog') AND is_primary_key = 1;");
|
||||||
|
Assert.Equal(1, pkExists);
|
||||||
}
|
}
|
||||||
|
|
||||||
[SkippableFact]
|
[SkippableFact]
|
||||||
@@ -705,10 +875,11 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
|
|||||||
// Switch out the June 2026 partition (different month, empty).
|
// Switch out the June 2026 partition (different month, empty).
|
||||||
await repo.SwitchOutPartitionAsync(new DateTime(2026, 6, 1, 0, 0, 0, DateTimeKind.Utc));
|
await repo.SwitchOutPartitionAsync(new DateTime(2026, 6, 1, 0, 0, 0, DateTimeKind.Utc));
|
||||||
|
|
||||||
// Re-attempting the same EventId after the switch must STILL be a no-op
|
// Re-attempting the same EventId after the switch must STILL be a no-op.
|
||||||
// (UX_AuditLog_EventId is the index that enables idempotency; if the
|
// The idempotency probe now seeks the aligned clustered PK's leading
|
||||||
// rebuild left it broken, this insert would silently produce a duplicate
|
// column instead of a dedicated unique index; if the switch had disturbed
|
||||||
// row and the count assertion below would catch it).
|
// that key, this insert would silently produce a duplicate row and the
|
||||||
|
// count assertion below would catch it.
|
||||||
// C3 (Task 2.5): rebuild a sibling row with the same EventId via the factory
|
// C3 (Task 2.5): rebuild a sibling row with the same EventId via the factory
|
||||||
// (ErrorMessage rides in DetailsJson, so a top-level `with` no longer applies).
|
// (ErrorMessage rides in DetailsJson, so a top-level `with` no longer applies).
|
||||||
var dup = ScadaBridgeAuditEventFactory.Create(
|
var dup = ScadaBridgeAuditEventFactory.Create(
|
||||||
@@ -734,7 +905,7 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
|
|||||||
}
|
}
|
||||||
|
|
||||||
[SkippableFact]
|
[SkippableFact]
|
||||||
public async Task SwitchOutPartitionAsync_PartialFailure_RebuildsUxIndex_RaisesException()
|
public async Task SwitchOutPartitionAsync_PartialFailure_RaisesException_LeavesNoOrphanedStaging()
|
||||||
{
|
{
|
||||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||||
|
|
||||||
@@ -745,8 +916,8 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
|
|||||||
// ALTER TABLE … SWITCH refuses to move rows out of a partition that's
|
// ALTER TABLE … SWITCH refuses to move rows out of a partition that's
|
||||||
// referenced by an FK from another table, raising msg 4928
|
// referenced by an FK from another table, raising msg 4928
|
||||||
// ("ALTER TABLE SWITCH statement failed because target table … has a
|
// ("ALTER TABLE SWITCH statement failed because target table … has a
|
||||||
// foreign key …"). The CATCH branch then rolls back and rebuilds the
|
// foreign key …"). The CATCH branch then rolls back and drops the staging
|
||||||
// unique index — which the assertion below verifies.
|
// table — which the assertion below verifies.
|
||||||
//
|
//
|
||||||
// The probe table is uniquely named with a guid suffix so reruns of
|
// The probe table is uniquely named with a guid suffix so reruns of
|
||||||
// this test inside the same fixture DB never collide. We clean it up
|
// this test inside the same fixture DB never collide. We clean it up
|
||||||
@@ -786,14 +957,23 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
|
|||||||
await cmd.ExecuteNonQueryAsync();
|
await cmd.ExecuteNonQueryAsync();
|
||||||
}
|
}
|
||||||
|
|
||||||
// The CATCH block in the production SQL guarantees UX_AuditLog_EventId
|
// The CATCH block drops the GUID-suffixed staging table regardless of
|
||||||
// is rebuilt regardless of which step failed inside the TRY.
|
// which step failed inside the TRY, so a failed purge leaves no orphaned
|
||||||
|
// AuditLog_Staging_* object behind for the next tick to trip over.
|
||||||
await using var verifyContext = CreateContext();
|
await using var verifyContext = CreateContext();
|
||||||
|
var orphanedStaging = await ScalarAsync<int>(
|
||||||
|
verifyContext,
|
||||||
|
"SELECT COUNT(*) FROM sys.tables WHERE name LIKE 'AuditLog\\_Staging\\_%' ESCAPE '\\';");
|
||||||
|
Assert.Equal(0, orphanedStaging);
|
||||||
|
|
||||||
|
// And it must NOT have (re)created the non-aligned unique index: WP2.2
|
||||||
|
// deleted that rebuild entirely, so a failed switch cannot reintroduce
|
||||||
|
// the object that blocks the NEXT switch.
|
||||||
var indexExists = await ScalarAsync<int>(
|
var indexExists = await ScalarAsync<int>(
|
||||||
verifyContext,
|
verifyContext,
|
||||||
"SELECT COUNT(*) FROM sys.indexes " +
|
"SELECT COUNT(*) FROM sys.indexes " +
|
||||||
"WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog');");
|
"WHERE name = 'UX_AuditLog_EventId' AND object_id = OBJECT_ID('dbo.AuditLog');");
|
||||||
Assert.Equal(1, indexExists);
|
Assert.Equal(0, indexExists);
|
||||||
}
|
}
|
||||||
|
|
||||||
// ------------------------------------------------------------------------
|
// ------------------------------------------------------------------------
|
||||||
|
|||||||
+87
@@ -325,6 +325,57 @@ public class KpiHistoryRepositoryTests
|
|||||||
Assert.Equal(2, row.SampleCount);
|
Assert.Equal(2, row.SampleCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task FoldHourlyRollupsAsync_PreloadsExistingRollups_WithoutAPerSeriesHourProbe()
|
||||||
|
{
|
||||||
|
// WP2.2: the fold used to issue one FirstOrDefaultAsync existence probe per
|
||||||
|
// (series, hour) group before writing anything — an N+1 that scaled with
|
||||||
|
// the metric catalogue times the lookback window. It now preloads the whole
|
||||||
|
// window in a single query and resolves each group from a dictionary.
|
||||||
|
//
|
||||||
|
// Six distinct series across two hours = twelve groups, so the old shape
|
||||||
|
// issued twelve SELECTs against KpiRollupHourly. The new shape issues one
|
||||||
|
// (plus the KpiSample read). This asserts the count stays small and
|
||||||
|
// constant rather than tracking the group count.
|
||||||
|
var counter = new RollupSelectCountingInterceptor();
|
||||||
|
await using var ctx = SqliteTestHelper.CreateInMemoryContext(counter);
|
||||||
|
var repo = new KpiHistoryRepository(ctx);
|
||||||
|
|
||||||
|
var samples = new List<KpiSample>();
|
||||||
|
for (var series = 0; series < 6; series++)
|
||||||
|
{
|
||||||
|
foreach (var hourOffset in new[] { 0, 1 })
|
||||||
|
{
|
||||||
|
samples.Add(Sample(
|
||||||
|
"NotificationOutbox",
|
||||||
|
"metric" + series,
|
||||||
|
"Global",
|
||||||
|
null,
|
||||||
|
value: series + hourOffset,
|
||||||
|
capturedAtUtc: Base.AddHours(hourOffset).AddMinutes(10)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await repo.RecordSamplesAsync(samples);
|
||||||
|
|
||||||
|
// Seed the rollups so the SECOND fold takes the re-fold (update) path —
|
||||||
|
// the branch that most needed the per-group probe.
|
||||||
|
await repo.FoldHourlyRollupsAsync(Base, Base.AddHours(2));
|
||||||
|
|
||||||
|
counter.Reset();
|
||||||
|
await repo.FoldHourlyRollupsAsync(Base, Base.AddHours(2));
|
||||||
|
|
||||||
|
Assert.True(
|
||||||
|
counter.RollupSelectCount <= 1,
|
||||||
|
$"expected the fold to preload existing rollups in a single query; it issued {counter.RollupSelectCount} SELECTs against KpiRollupHourly");
|
||||||
|
|
||||||
|
// And the fold is still correct: twelve series-hours, values unchanged by
|
||||||
|
// the re-fold.
|
||||||
|
var rollups = await ctx.KpiRollupHourly.AsNoTracking().ToListAsync();
|
||||||
|
Assert.Equal(12, rollups.Count);
|
||||||
|
Assert.All(rollups, r => Assert.Equal(1, r.SampleCount));
|
||||||
|
}
|
||||||
|
|
||||||
[Fact]
|
[Fact]
|
||||||
public async Task GetHourlySeriesAsync_ReturnsAscending_AndHonorsNullVsSiteScopeKey()
|
public async Task GetHourlySeriesAsync_ReturnsAscending_AndHonorsNullVsSiteScopeKey()
|
||||||
{
|
{
|
||||||
@@ -540,6 +591,42 @@ public class KpiHistoryRepositoryTests
|
|||||||
/// async non-query entry points (<c>ExecuteDeleteAsync</c> routes through the
|
/// async non-query entry points (<c>ExecuteDeleteAsync</c> routes through the
|
||||||
/// async path).
|
/// async path).
|
||||||
/// </summary>
|
/// </summary>
|
||||||
|
/// <summary>
|
||||||
|
/// Counts reader commands that SELECT from <c>KpiRollupHourly</c>, so a test
|
||||||
|
/// can prove the hourly fold resolves existing rows from a single preload
|
||||||
|
/// rather than one probe per (series, hour) group.
|
||||||
|
/// </summary>
|
||||||
|
private sealed class RollupSelectCountingInterceptor : DbCommandInterceptor
|
||||||
|
{
|
||||||
|
public int RollupSelectCount { get; private set; }
|
||||||
|
|
||||||
|
public void Reset() => RollupSelectCount = 0;
|
||||||
|
|
||||||
|
private void CountIfRollupSelect(DbCommand command)
|
||||||
|
{
|
||||||
|
if (command.CommandText.Contains("KpiRollupHourly", StringComparison.OrdinalIgnoreCase)
|
||||||
|
&& command.CommandText.Contains("SELECT", StringComparison.OrdinalIgnoreCase))
|
||||||
|
{
|
||||||
|
RollupSelectCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public override InterceptionResult<DbDataReader> ReaderExecuting(
|
||||||
|
DbCommand command, CommandEventData eventData, InterceptionResult<DbDataReader> result)
|
||||||
|
{
|
||||||
|
CountIfRollupSelect(command);
|
||||||
|
return base.ReaderExecuting(command, eventData, result);
|
||||||
|
}
|
||||||
|
|
||||||
|
public override ValueTask<InterceptionResult<DbDataReader>> ReaderExecutingAsync(
|
||||||
|
DbCommand command, CommandEventData eventData, InterceptionResult<DbDataReader> result,
|
||||||
|
CancellationToken cancellationToken = default)
|
||||||
|
{
|
||||||
|
CountIfRollupSelect(command);
|
||||||
|
return base.ReaderExecutingAsync(command, eventData, result, cancellationToken);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
private sealed class DeleteCountingInterceptor : DbCommandInterceptor
|
private sealed class DeleteCountingInterceptor : DbCommandInterceptor
|
||||||
{
|
{
|
||||||
public int DeleteCount { get; private set; }
|
public int DeleteCount { get; private set; }
|
||||||
|
|||||||
+41
@@ -96,6 +96,47 @@ public class SiteCallAuditRepositoryTests : IClassFixture<MsSqlMigrationFixture>
|
|||||||
Assert.Equal(attemptedSnapshot!.UpdatedAtUtc, afterStale.UpdatedAtUtc);
|
Assert.Equal(attemptedSnapshot!.UpdatedAtUtc, afterStale.UpdatedAtUtc);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
[SkippableFact]
|
||||||
|
public async Task UpsertAsync_RejectedMonotonicUpdate_DoesNotFallThroughToInsert()
|
||||||
|
{
|
||||||
|
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||||
|
|
||||||
|
// WP2.2 collapsed the two-round-trip "insert-if-absent then monotonic
|
||||||
|
// update" into ONE batch that updates first and inserts only when nothing
|
||||||
|
// matched. That makes @@ROWCOUNT = 0 ambiguous: it means "no such row"
|
||||||
|
// AND "the monotonic guard rejected this packet". Guarding the insert on
|
||||||
|
// @@ROWCOUNT alone would therefore let every stale/regressive packet
|
||||||
|
// append a SECOND row for an id that already exists — silently forking
|
||||||
|
// the mirror. The re-check for the row's existence is what prevents that,
|
||||||
|
// and this pins it: after a rejected regressive upsert there is still
|
||||||
|
// exactly ONE row, still carrying the advanced state.
|
||||||
|
var id = TrackedOperationId.New();
|
||||||
|
await using var context = CreateContext();
|
||||||
|
var repo = new SiteCallAuditRepository(context);
|
||||||
|
|
||||||
|
await repo.UpsertAsync(NewRow(id, status: "Delivered", retryCount: 3, lastError: null));
|
||||||
|
|
||||||
|
// Every flavour of rejection: lower rank, equal terminal rank, and an
|
||||||
|
// equal non-terminal rank with a stale timestamp.
|
||||||
|
await repo.UpsertAsync(NewRow(id, status: "Submitted", retryCount: 0));
|
||||||
|
await repo.UpsertAsync(NewRow(id, status: "Parked", retryCount: 9, lastError: "should-not-apply"));
|
||||||
|
await repo.UpsertAsync(NewRow(
|
||||||
|
id,
|
||||||
|
status: "Delivered",
|
||||||
|
retryCount: 99,
|
||||||
|
updatedAtUtc: new DateTime(2020, 1, 1, 0, 0, 0, DateTimeKind.Utc)));
|
||||||
|
|
||||||
|
await using var readContext = CreateContext();
|
||||||
|
var loaded = await readContext.Set<SiteCall>()
|
||||||
|
.Where(s => s.TrackedOperationId == id)
|
||||||
|
.ToListAsync();
|
||||||
|
|
||||||
|
Assert.Single(loaded);
|
||||||
|
Assert.Equal("Delivered", loaded[0].Status);
|
||||||
|
Assert.Equal(3, loaded[0].RetryCount);
|
||||||
|
Assert.Null(loaded[0].LastError);
|
||||||
|
}
|
||||||
|
|
||||||
[SkippableFact]
|
[SkippableFact]
|
||||||
public async Task UpsertAsync_SameStatus_EqualUpdatedAt_IsNoOp()
|
public async Task UpsertAsync_SameStatus_EqualUpdatedAt_IsNoOp()
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging.Abstractions;
|
|||||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||||
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
|
||||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Integration;
|
||||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||||
@@ -582,4 +583,122 @@ public class SiteCallAuditReconciliationTests : TestKit
|
|||||||
Assert.Equal(siteId, evt.SiteId);
|
Assert.Equal(siteId, evt.SiteId);
|
||||||
Assert.True(evt.Pinned, "a legacy site that ignores after_id must publish Pinned=true");
|
Assert.True(evt.Pinned, "a legacy site that ignores after_id must publish Pinned=true");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------
|
||||||
|
// 9. WP2.2: the reconciliation drain runs OFF the mailbox, so ingest,
|
||||||
|
// query and KPI messages are answered while a long post-outage
|
||||||
|
// catch-up is still pulling.
|
||||||
|
// ---------------------------------------------------------------------
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Pull client whose first call blocks until released, simulating a
|
||||||
|
/// post-outage catch-up that takes far longer than the caller's Ask timeout.
|
||||||
|
/// </summary>
|
||||||
|
private sealed class BlockingPullClient : IPullSiteCallsClient
|
||||||
|
{
|
||||||
|
private readonly TaskCompletionSource _release =
|
||||||
|
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
private readonly TaskCompletionSource _entered =
|
||||||
|
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
|
||||||
|
/// <summary>Completes once the drain has actually started pulling.</summary>
|
||||||
|
public Task Entered => _entered.Task;
|
||||||
|
|
||||||
|
public void Release() => _release.TrySetResult();
|
||||||
|
|
||||||
|
public async Task<PullSiteCallsResponse> PullAsync(
|
||||||
|
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
|
||||||
|
{
|
||||||
|
_entered.TrySetResult();
|
||||||
|
await _release.Task.ConfigureAwait(false);
|
||||||
|
return new PullSiteCallsResponse(Array.Empty<SiteCall>(), MoreAvailable: false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ReconciliationDrain_InFlight_DoesNotBlockIngestUpsert()
|
||||||
|
{
|
||||||
|
// The drain used to run inside ReceiveAsync, which occupies the actor for
|
||||||
|
// its whole duration. A post-outage catch-up (every site, many paged
|
||||||
|
// network pulls, one upsert per row) therefore parked telemetry ingest,
|
||||||
|
// UI queries and KPI Asks behind it — and those callers timed out rather
|
||||||
|
// than queued, so a slow site could make central look dead. This pins the
|
||||||
|
// fix: with the drain off-mailbox behind a single-flight guard, an ingest
|
||||||
|
// Ask completes promptly while the pull is still blocked.
|
||||||
|
var siteId = "siteSlow";
|
||||||
|
var sites = new StaticEnumerator(new SiteEntry(siteId, "http://siteSlow:8083"));
|
||||||
|
var client = new BlockingPullClient();
|
||||||
|
var repo = new RecordingRepo();
|
||||||
|
|
||||||
|
var actor = CreateActor(sites, client, repo, FastTickOptions());
|
||||||
|
|
||||||
|
// Wait until the drain is genuinely in flight and blocked inside PullAsync.
|
||||||
|
await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
|
||||||
|
|
||||||
|
// The mailbox must still be serving. A generous-but-finite budget: this
|
||||||
|
// fails at the pre-fix behaviour (the reply only arrives once the pull
|
||||||
|
// unblocks, which never happens until Release below).
|
||||||
|
var id = TrackedOperationId.New();
|
||||||
|
var reply = await actor.Ask<UpsertSiteCallReply>(
|
||||||
|
new UpsertSiteCallCommand(NewRow(id, sourceSite: siteId)),
|
||||||
|
TimeSpan.FromSeconds(3));
|
||||||
|
|
||||||
|
Assert.True(reply.Accepted);
|
||||||
|
Assert.Equal(id, reply.TrackedOperationId);
|
||||||
|
|
||||||
|
// Let the drain finish so the actor shuts down cleanly.
|
||||||
|
client.Release();
|
||||||
|
}
|
||||||
|
|
||||||
|
[Fact]
|
||||||
|
public async Task ReconciliationTicks_DoNotOverlap_WhileADrainIsInFlight()
|
||||||
|
{
|
||||||
|
// Single-flight guard: with a 100 ms tick and a drain blocked for far
|
||||||
|
// longer, every subsequent tick must be dropped rather than starting a
|
||||||
|
// second concurrent pass — overlapping passes would race on the per-site
|
||||||
|
// cursor and pinned-latch dictionaries the drain mutates.
|
||||||
|
var siteId = "siteSlow";
|
||||||
|
var sites = new StaticEnumerator(new SiteEntry(siteId, "http://siteSlow:8083"));
|
||||||
|
var client = new CountingBlockingPullClient();
|
||||||
|
var repo = new RecordingRepo();
|
||||||
|
|
||||||
|
CreateActor(sites, client, repo, FastTickOptions());
|
||||||
|
|
||||||
|
await client.Entered.WaitAsync(TimeSpan.FromSeconds(5));
|
||||||
|
|
||||||
|
// Several tick intervals elapse while the first pass is still blocked.
|
||||||
|
await Task.Delay(TimeSpan.FromMilliseconds(600));
|
||||||
|
|
||||||
|
Assert.Equal(1, client.CallCount);
|
||||||
|
|
||||||
|
client.Release();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// <see cref="BlockingPullClient"/> that also counts invocations, so a test
|
||||||
|
/// can prove no second pass started while the first was blocked.
|
||||||
|
/// </summary>
|
||||||
|
private sealed class CountingBlockingPullClient : IPullSiteCallsClient
|
||||||
|
{
|
||||||
|
private readonly TaskCompletionSource _release =
|
||||||
|
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
private readonly TaskCompletionSource _entered =
|
||||||
|
new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||||
|
private int _callCount;
|
||||||
|
|
||||||
|
public Task Entered => _entered.Task;
|
||||||
|
|
||||||
|
public int CallCount => Volatile.Read(ref _callCount);
|
||||||
|
|
||||||
|
public void Release() => _release.TrySetResult();
|
||||||
|
|
||||||
|
public async Task<PullSiteCallsResponse> PullAsync(
|
||||||
|
string siteId, DateTime sinceUtc, string? afterId, int batchSize, CancellationToken ct)
|
||||||
|
{
|
||||||
|
Interlocked.Increment(ref _callCount);
|
||||||
|
_entered.TrySetResult();
|
||||||
|
await _release.Task.ConfigureAwait(false);
|
||||||
|
return new PullSiteCallsResponse(Array.Empty<SiteCall>(), MoreAvailable: false);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user