Compare commits

...

12 Commits

Author SHA1 Message Date
Joseph Doherty 1dd993ec2f docs(archreview): sync PLAN-04 tasks.json — Tasks 7-9 completed (Batch 3, append-only-honesty docs + SiteCalls index + exec-tree window bound) 2026-07-09 07:11:26 -04:00
Joseph Doherty 5d0fcd4e66 perf(audit-log): bound GetExecutionTreeAsync edge scan to a root-anchored time window (partition elimination) 2026-07-09 07:10:35 -04:00
Joseph Doherty f1957606ff perf(site-call-audit): filtered IX_SiteCalls_NonTerminal index — KPI predicates seek the live queue instead of scanning 365 days 2026-07-09 07:06:50 -04:00
Joseph Doherty eb69b93947 docs(audit-log): document append-only enforcement honestly (CI guard default; DB roles are optional DBA hardening; purger now has CREATE TABLE + scoped DELETE) 2026-07-09 07:03:10 -04:00
Joseph Doherty 2a080a3e12 docs(archreview): sync PLAN-04 tasks.json — Tasks 4-6 completed (Batch 2, maintenance-timeout + purge-failure health + purger grants) 2026-07-09 06:53:46 -04:00
Joseph Doherty 7555e65746 fix(config-db): grant CREATE TABLE + scoped DELETE to scadabridge_audit_purger so a segregated maintenance principal can actually run the purge 2026-07-09 06:52:44 -04:00
Joseph Doherty 14c4df54f9 feat(audit-log): surface partition-purge failure as health event + counter 2026-07-09 06:49:28 -04:00
Joseph Doherty b59aa2d717 fix(audit-log): explicit long CommandTimeout on partition-switch and per-channel purge maintenance paths 2026-07-09 06:43:23 -04:00
Joseph Doherty 1b6b0aab2a docs(archreview): sync PLAN-04 tasks.json — Tasks 1-3 completed (Batch 1, site retention chain) 2026-07-09 06:22:42 -04:00
Joseph Doherty 74aa872c07 feat(audit-log): daily site SQLite retention purge job (closes unbounded site DB growth)
SiteAuditRetentionService (site-only IHostedService) ticks PurgeExpiredAsync on
InitialDelay(5m)/PurgeInterval(24h, clamped >=1m), cutoff = UtcNow - RetentionDays.
Registered site-only in AddAuditLogHealthMetricsBridge; options bound from
AuditLog:SiteRetention. Per-tick failures swallowed. Added a test-only
PurgeIntervalOverride (mirrors SiteCallAuditOptions) so the ms-cadence tick test
can bypass the 1-min production clamp. Doc updated. (PLAN-04 Task 3, S1/U2)
2026-07-09 06:22:19 -04:00
Joseph Doherty b5271da5c8 feat(audit-log): SiteAuditRetentionOptions (7-day default, clamped 1-90)
RetentionDays clamped [1,90]; PurgeInterval clamped to >= 1 min (Timer spin
footgun); 5-min InitialDelay so a daily-recycled node still purges. (PLAN-04 Task 2)
2026-07-09 06:13:05 -04:00
Joseph Doherty 731ee69797 feat(audit-log): add site SQLite PurgeExpiredAsync honoring the ForwardState invariant
Retention purge deletes Forwarded/Reconciled rows older than the cutoff and
reclaims pages via incremental_vacuum; Pending rows are never purged on age
alone (would lose audit data that never reached central). Temp-table purge set
avoids the SQLite 999-param limit; runs under _writeLock. (PLAN-04 Task 1, S1/U2)
2026-07-09 06:11:54 -04:00
39 changed files with 5459 additions and 57 deletions
@@ -1,15 +1,15 @@
{
"planPath": "archreview/plans/PLAN-04-data-audit-backbone.md",
"tasks": [
{ "id": 1, "subject": "Task 1: Site SQLite retention purge — PurgeExpiredAsync on the writer", "status": "pending", "blockedBy": [] },
{ "id": 2, "subject": "Task 2: Site retention options", "status": "pending", "blockedBy": [] },
{ "id": 3, "subject": "Task 3: Site retention hosted job + registration + design doc", "status": "pending", "blockedBy": [1, 2] },
{ "id": 4, "subject": "Task 4: Maintenance command timeout on partition switch + per-channel purge", "status": "pending", "blockedBy": [] },
{ "id": 5, "subject": "Task 5: Purge-failure health event (not just a log line)", "status": "pending", "blockedBy": [4] },
{ "id": 6, "subject": "Task 6: Fix purger role grants (migration)", "status": "pending", "blockedBy": [] },
{ "id": 7, "subject": "Task 7: Honest append-only enforcement story (docs)", "status": "pending", "blockedBy": [6] },
{ "id": 8, "subject": "Task 8: SiteCalls filtered non-terminal index (migration)", "status": "pending", "blockedBy": [6] },
{ "id": 9, "subject": "Task 9: Bound the execution-tree edge scan with a root time window", "status": "pending", "blockedBy": [] },
{ "id": 1, "subject": "Task 1: Site SQLite retention purge — PurgeExpiredAsync on the writer", "status": "completed", "blockedBy": [] },
{ "id": 2, "subject": "Task 2: Site retention options", "status": "completed", "blockedBy": [] },
{ "id": 3, "subject": "Task 3: Site retention hosted job + registration + design doc", "status": "completed", "blockedBy": [1, 2] },
{ "id": 4, "subject": "Task 4: Maintenance command timeout on partition switch + per-channel purge", "status": "completed", "blockedBy": [] },
{ "id": 5, "subject": "Task 5: Purge-failure health event (not just a log line)", "status": "completed", "blockedBy": [4] },
{ "id": 6, "subject": "Task 6: Fix purger role grants (migration)", "status": "completed", "blockedBy": [] },
{ "id": 7, "subject": "Task 7: Honest append-only enforcement story (docs)", "status": "completed", "blockedBy": [6] },
{ "id": 8, "subject": "Task 8: SiteCalls filtered non-terminal index (migration)", "status": "completed", "blockedBy": [6] },
{ "id": 9, "subject": "Task 9: Bound the execution-tree edge scan with a root time window", "status": "completed", "blockedBy": [] },
{ "id": 10, "subject": "Task 10: Fix the hardcoded-zero backlogTotal KPI history", "status": "pending", "blockedBy": [] },
{ "id": 11, "subject": "Task 11: SiteCalls same-rank freshness upsert (unfreeze RetryCount/LastError)", "status": "pending", "blockedBy": [] },
{ "id": 12, "subject": "Task 12: EF Core connection resiliency (EnableRetryOnFailure)", "status": "pending", "blockedBy": [] },
@@ -28,5 +28,5 @@
{ "id": 25, "subject": "Task 25: Bounded parallel notification delivery", "status": "pending", "blockedBy": [13] },
{ "id": 26, "subject": "Final verification: full solution build + test sweep (infra up for MSSQL suites)", "status": "pending", "blockedBy": [3, 8, 9, 10, 12, 14, 18, 19, 20, 21, 22, 23, 24, 25] }
],
"lastUpdated": "2026-07-08"
"lastUpdated": "2026-07-09 (Batch 3: Tasks 7-9 completed)"
}
+4 -4
View File
@@ -42,14 +42,14 @@ A `SecretAwareModelCacheKeyFactory` folds `HasSecretEncryptionProvider` into the
### Append-only AuditLog and DB-role enforcement
The central `dbo.AuditLog` table has two dedicated SQL Server roles:
The central `dbo.AuditLog` table has two dedicated SQL Server roles — **optional DBA hardening** on top of the default application-level enforcement (CI grep guard + code review; the default deployment runs one connection principal for both paths):
| Role | Grants |
|------|--------|
| `scadabridge_audit_writer` | `INSERT`, `SELECT` on `AuditLog` only — no `UPDATE`, no `DELETE` |
| `scadabridge_audit_purger` | `ALTER ON SCHEMA::dbo` (required for `SPLIT RANGE` and partition switch-out) |
| `scadabridge_audit_writer` | `INSERT`, `SELECT` on `AuditLog` only; `DENY UPDATE` + `DENY DELETE` |
| `scadabridge_audit_purger` | `SELECT` + `ALTER ON SCHEMA::dbo` (partition switch-out), **plus** `CREATE TABLE` (switch-out CREATEs a staging table) and scoped `DELETE ON dbo.AuditLog` (per-channel retention override) — the latter two added by the `FixAuditPurgerRoleGrants` migration |
Row-level `DELETE` on `AuditLog` is not granted even to the purge role; retention is always a partition switch, never a row delete.
The primary retention purge is a monthly partition switch. The one sanctioned row-level `DELETE` against `AuditLog` is the per-channel retention override on the maintenance path (guarded by the `// AUDIT-PURGE-ALLOWED` marker); every other `UPDATE`/`DELETE` trips the append-only CI guard.
## Architecture
@@ -0,0 +1,21 @@
BEGIN TRANSACTION;
IF NOT EXISTS (
SELECT * FROM [__EFMigrationsHistory]
WHERE [MigrationId] = N'20260709110614_AddSiteCallsNonTerminalIndex'
)
BEGIN
EXEC(N'CREATE INDEX [IX_SiteCalls_NonTerminal] ON [SiteCalls] ([CreatedAtUtc]) INCLUDE ([SourceSite], [SourceNode], [Status]) WHERE [TerminalAtUtc] IS NULL');
END;
IF NOT EXISTS (
SELECT * FROM [__EFMigrationsHistory]
WHERE [MigrationId] = N'20260709110614_AddSiteCallsNonTerminalIndex'
)
BEGIN
INSERT INTO [__EFMigrationsHistory] ([MigrationId], [ProductVersion])
VALUES (N'20260709110614_AddSiteCallsNonTerminalIndex', N'10.0.7');
END;
COMMIT;
GO
@@ -0,0 +1,29 @@
BEGIN TRANSACTION;
IF NOT EXISTS (
SELECT * FROM [__EFMigrationsHistory]
WHERE [MigrationId] = N'20260709105112_FixAuditPurgerRoleGrants'
)
BEGIN
IF DATABASE_PRINCIPAL_ID('scadabridge_audit_purger') IS NULL
EXEC sp_executesql N'CREATE ROLE scadabridge_audit_purger';
-- The switch-out dance CREATEs a staging table; ALTER ON SCHEMA::dbo alone does not
-- confer the database-level CREATE TABLE permission (review finding S3).
GRANT CREATE TABLE TO scadabridge_audit_purger;
-- The per-channel retention override (PurgeChannelOlderThanAsync) is a bounded row
-- DELETE on the maintenance path; a segregated purger principal needs this grant.
GRANT DELETE ON dbo.AuditLog TO scadabridge_audit_purger;
END;
IF NOT EXISTS (
SELECT * FROM [__EFMigrationsHistory]
WHERE [MigrationId] = N'20260709105112_FixAuditPurgerRoleGrants'
)
BEGIN
INSERT INTO [__EFMigrationsHistory] ([MigrationId], [ProductVersion])
VALUES (N'20260709105112_FixAuditPurgerRoleGrants', N'10.0.7');
END;
COMMIT;
GO
+68 -11
View File
@@ -209,6 +209,17 @@ tag subscription) are also wired: trigger-driven runs carry `ParentExecutionId =
NULL` (top-level roots), and any nested `CallScript`/`CallShared` they perform
chains as above. The schema is unchanged — no further tag-cascade work is deferred.
**Execution-tree traversal bound.** `GetExecutionTreeAsync` first walks up
`ParentExecutionId` to the chain root, then walks down via a recursive CTE. The
down-walk's edge scan is bounded to a window anchored at the root's **first event**:
`[rootFirst − 1 h, rootFirst + 7 d)` (configurable constants). Execution trees span
minutes, not years — an inbound request and everything it spawns fire inside one
operational burst — so the window lets SQL Server eliminate every `AuditLog`
partition outside it, turning a full-table `DISTINCT` scan into a seek on
`IX_AuditLog_Execution` (arch-review 04, P2). A genuine descendant stamped **beyond
the 7-day span is excluded by design**; a row-less stub root (its first event purged)
has no anchor and falls back to the unbounded scan (correctness over speed).
## The Site-Local `AuditLog` (SQLite)
A SQLite database file on each site node, alongside the Store-and-Forward
@@ -390,9 +401,25 @@ MS SQL for direct-write events). Unredacted secrets never persist.
- **Purge actor:** `AuditLogPurgeActor` singleton on the active central node
runs daily, switches out any partition whose latest `OccurredAtUtc` is older
than the retention window, then applies any per-channel overrides (see below),
and emits an `AuditLog:Purged` event (partition range, rowcount, duration) per
and emits an `AuditLogPurgedEvent` (partition range, rowcount, duration) per
switched partition. A partition-maintenance step rolls forward each month,
creating the next month's partition ahead of time.
- **Purge failure is a health metric, not just a log line:** every purge step that
throws — a partition switch-out, a per-channel override DELETE, or the pre-purge
boundary enumeration — publishes an `AuditLogPurgeFailedEvent` (month boundary,
error message, elapsed ms; `DateTime.MinValue` boundary for the enumeration/channel
phases that have no single month in hand) and increments the `PurgeFailures` central
health counter on `AuditCentralHealthSnapshot` (alongside `CentralAuditWriteFailures`).
This is the symmetric failure twin of the success-path `AuditLogPurgedEvent`: a
silently failing retention job would otherwise be invisible until the table grew
unbounded. Per-boundary/per-channel error isolation is unchanged — a single failure
still never abandons the rest of the tick.
- **Maintenance command timeout:** the switch-out drop-and-rebuild dance and each
per-channel `DELETE TOP` batch run with an explicit command timeout
(`AuditLog:Purge:MaintenanceCommandTimeoutMinutes`, default 30, floor 1 min) rather
than the ~30 s ADO.NET default, which could abort the metadata-only SWITCH mid-dance
on a large or contended partition and leave the live table without
`UX_AuditLog_EventId` until a later tick's CATCH branch rebuilt it.
- **Per-channel retention overrides (M5.5 T3):** `AuditLog:PerChannelRetentionDays`
is a dictionary keyed by canonical channel name (`ApiOutbound`, `DbOutbound`,
`Notification`, `ApiInbound`) whose value is a retention window in days that
@@ -409,19 +436,49 @@ MS SQL for direct-write events). Unredacted secrets never persist.
partition switch-out loop. Values are validated to be in
`[30, RetentionDays]`; keys that are not a recognized `AuditChannel` enum name
are rejected at startup.
- **Sites:** daily site job; default 7-day retention (configurable, min 1,
max 90). Respects the hard `ForwardState` invariant — `Pending` rows are
never purged on age alone.
- **Sites:** `SiteAuditRetentionService` (site-only `IHostedService`) runs the
site SQLite retention purge on a timer — first tick after
`AuditLog:SiteRetention:InitialDelay` (default 5 min, short so a daily-recycled
node still purges soon after start), then every
`AuditLog:SiteRetention:PurgeInterval` (default 24 h, clamped to ≥ 1 min).
Each tick calls `ISiteAuditQueue.PurgeExpiredAsync(UtcNow − RetentionDays)`,
which deletes eligible rows in one transaction (sidecar first, then the
canonical row) and then runs `PRAGMA incremental_vacuum` to return the freed
pages to the OS. `AuditLog:SiteRetention:RetentionDays` defaults to 7 and is
clamped to `[1, 90]`. Respects the hard `ForwardState` invariant — a `Pending`
row is never purged on age alone; only `Forwarded`/`Reconciled` rows older than
the cutoff are removed. Per-tick failures are swallowed and logged so a transient
SQLite fault never tears the service down.
## Security & Tamper-Evidence
- **Append-only enforcement.** The application accesses `AuditLog` via a
dedicated DB role `scadabridge_audit_writer` granted `INSERT` + `SELECT` only —
no `UPDATE`, no `DELETE`. Purge runs under a separate role
`scadabridge_audit_purger` whose permissions are limited to the partition-switch
operation; row-level `DELETE` is not granted even to purge.
- **CI grep guard.** The build greps the data layer for any
`UPDATE … AuditLog` or `DELETE … AuditLog` text and fails on a hit.
- **Append-only enforcement — what actually holds in the default deployment.**
ScadaBridge runs with **one** connection principal for both the writer and the
maintenance/purge paths (the runtime does not open a second, lower-privilege
connection for purge). So in the default deployment the append-only invariant is
enforced by two application-level controls, not by database permissions:
1. **CI grep guard** — `AuditLogAppendOnlyGuardTests` scans the ConfigurationDatabase
source for any `UPDATE`/`DELETE` DML targeting `dbo.AuditLog` and fails the build
on a hit. Exactly two maintenance-path mutations are allow-listed by an explicit
`// AUDIT-PURGE-ALLOWED` marker (the per-channel retention `DELETE TOP` and the
one-time `SourceNode` sentinel backfill `UPDATE`); every other UPDATE/DELETE trips
the guard.
2. **Code review** — the marker is deliberately specific so an unrelated mutation
cannot inherit the exemption without a reviewer noticing.
- **Optional DBA hardening (two DB roles).** For a deployment that wants
database-level enforcement on top of the application controls, migrations provision
two roles: `scadabridge_audit_writer` (`INSERT` + `SELECT` only; `DENY UPDATE` +
`DENY DELETE` so a later `db_datawriter` membership cannot silently re-enable
mutation) and `scadabridge_audit_purger` (the maintenance principal). To use them,
a DBA provisions **two logins** — the runtime connection mapped to the writer role,
and a separate maintenance job/connection mapped to the purger role. The purger role
now carries the permissions its purge path genuinely needs: `SELECT` + `ALTER ON
SCHEMA::dbo` **plus** `CREATE TABLE` (the switch-out dance CREATEs a staging table)
and a scoped `DELETE ON dbo.AuditLog` (the per-channel retention override is a bounded
row DELETE) — granted by the `FixAuditPurgerRoleGrants` migration (arch-review 04, S3).
Without those two grants the purger role could not actually execute the switch-out or
the per-channel purge; the earlier claim that "row-level `DELETE` is not granted even
to purge" was stale once `PerChannelRetentionDays` shipped.
- **Authorization.** Reading the Audit Log requires the existing **Audit** role
extended with a new **OperationalAudit** permission. Per-site row scoping
reuses the existing site-permission model; bulk export requires an additional
@@ -100,7 +100,7 @@ The configuration database stores all central system data, organized by domain a
- `IX_AuditLog_ParentExecution (ParentExecutionId)` — unfiltered
- `IX_AuditLog_Node_Occurred (SourceNode, OccurredAtUtc)` (per-node queries)
**Monthly partitioning** on `OccurredAtUtc` via partition function `pf_AuditLog_Month` and partition scheme `ps_AuditLog_Month`; retention purge is a partition switch, never a row-level delete. The table is populated only by Audit Log writers (site telemetry, central direct-write, reconciliation pulls); central ingest is **insert-if-not-exists** keyed on `EventId`. See Component-AuditLog.md for the full lifecycle, payload-capture policy, and ingestion paths.
**Monthly partitioning** on `OccurredAtUtc` via partition function `pf_AuditLog_Month` and partition scheme `ps_AuditLog_Month`; the primary retention purge is a partition switch (per-channel retention overrides add one sanctioned bounded row DELETE on the maintenance path — see AuditLog Table Purge). The table is populated only by Audit Log writers (site telemetry, central direct-write, reconciliation pulls); central ingest is **insert-if-not-exists** keyed on `EventId`. See Component-AuditLog.md for the full lifecycle, payload-capture policy, and ingestion paths.
### Inbound API
- **API Keys**: The `ApiKeys` SQL Server table was **retired** by migration `RetireInboundApiKeyStore`. Inbound API keys now live in the shared `ZB.MOM.WW.Auth.ApiKeys` SQLite store, not the configuration MS SQL database. The `IInboundApiRepository` no longer exposes any `ApiKey` operations; key management goes through `IInboundApiKeyAdmin` (Commons, `Interfaces/Security/`). The `ApprovedApiKeyIds` column on `ApiMethods` was removed at the same time — key-to-method scoping is now managed by the auth library.
@@ -327,12 +327,12 @@ Results are returned in reverse chronological order (most recent first) with pag
## Database Roles
The configuration database defines dedicated SQL Server roles for the append-only `AuditLog` table so that the application can never accidentally mutate audit history:
The configuration database defines dedicated SQL Server roles for the append-only `AuditLog` table. These roles are **optional DBA hardening**: the default deployment runs with a single connection principal for both writer and maintenance paths, so append-only is enforced application-side (CI grep guard + code review — see Component-AuditLog.md). A deployment that wants database-level enforcement provisions two logins mapped to these roles:
- **`scadabridge_audit_writer`** — the role used by application code that ingests audit events (the `AuditLogIngestActor`, central direct-write paths, and the Notification Outbox dispatcher). Granted `INSERT` and `SELECT` on `AuditLog` only — explicitly **no** `UPDATE` and **no** `DELETE`. Audit ingest is `INSERT … WHERE NOT EXISTS` keyed on `EventId`, which this grant set fully supports.
- **`scadabridge_audit_purger`** — the role used by the `AuditLogPurgeActor`. Granted only the permissions required to execute the monthly partition-switch operation (switch out a partition to a staging table and drop the staging table). Row-level `DELETE` on `AuditLog` is **not** granted even to the purge role; retention is a partition switch, never a row-by-row delete.
- **`scadabridge_audit_writer`** — the role for application code that ingests audit events (the `AuditLogIngestActor`, central direct-write paths, and the Notification Outbox dispatcher). Granted `INSERT` and `SELECT` on `AuditLog` only, with `DENY UPDATE` + `DENY DELETE` so a later `db_datawriter` membership cannot silently re-enable mutation (`DENY` > `GRANT`). Audit ingest is `INSERT … WHERE NOT EXISTS` keyed on `EventId`, which this grant set fully supports.
- **`scadabridge_audit_purger`** — the maintenance role for the `AuditLogPurgeActor`. Granted `SELECT` + `ALTER ON SCHEMA::dbo` for the monthly partition-switch dance, **plus** `CREATE TABLE` (the switch-out CREATEs a staging table — `ALTER ON SCHEMA::dbo` alone does not confer the database-level `CREATE TABLE` permission) and a scoped `DELETE ON dbo.AuditLog` (the per-channel retention override is a bounded row DELETE on the maintenance path). The `CREATE TABLE` + `DELETE` grants were added by the `FixAuditPurgerRoleGrants` migration (arch-review 04, S3); the earlier claim that row-level `DELETE` is "not granted even to the purge role" was stale once per-channel retention overrides shipped.
A CI grep guard fails the build on any occurrence of `UPDATE … AuditLog` or `DELETE … AuditLog` in the data-access layer source, backstopping the DB-grant enforcement at code-review time. See Component-AuditLog.md (Security & Tamper-Evidence) for the full enforcement contract.
A CI grep guard (`AuditLogAppendOnlyGuardTests`) fails the build on any `UPDATE`/`DELETE` DML targeting `dbo.AuditLog` in the data-access layer source, save the two `// AUDIT-PURGE-ALLOWED`-marked maintenance-path mutations. In the default single-principal deployment this guard (with code review) IS the enforcement; the DB roles above harden it further when a DBA opts in. See Component-AuditLog.md (Security & Tamper-Evidence) for the full contract.
---
@@ -396,7 +396,7 @@ The `SiteCalls` table grows one row per cached site call and is never trimmed by
### AuditLog Table Purge
The `AuditLog` table is append-only and grows by every script-trust-boundary event across all channels. Unlike `Notifications` and `SiteCalls`, purge is **never a row-level `DELETE`** — it is a **monthly partition switch** against the `ps_AuditLog_Month` scheme. A daily job switches out any partition whose latest `OccurredAtUtc` is older than the global retention window (default 365 days, configurable via the `AuditLog:RetentionDays` Audit Log option — single global value in v1, no per-channel overrides) and drops the resulting staging table. The job is owned and scheduled by the Audit Log component (`AuditLogPurgeActor` — see Component-AuditLog.md), which is also the consumer of the `AuditLog:RetentionDays` option. The Configuration Database component contributes only the table, the partition function/scheme, the indexes, and the DB roles that constrain the purge to a partition switch.
The `AuditLog` table is append-only and grows by every script-trust-boundary event across all channels. Unlike `Notifications` and `SiteCalls`, the primary purge is a **monthly partition switch** against the `ps_AuditLog_Month` scheme, not a row-level `DELETE`. A daily job switches out any partition whose latest `OccurredAtUtc` is older than the global retention window (default 365 days, configurable via `AuditLog:RetentionDays`) and drops the resulting staging table. On top of the global window, **per-channel retention overrides** (`AuditLog:PerChannelRetentionDays`, a dictionary keyed by canonical channel name with windows strictly shorter than the global one) expire a single channel's rows earlier via a bounded, batched `DELETE TOP` on the maintenance path — the one sanctioned row-level DELETE against `AuditLog` (see Component-AuditLog.md). The job is owned and scheduled by the Audit Log component (`AuditLogPurgeActor` — see Component-AuditLog.md), which is also the consumer of both options. The Configuration Database component contributes only the table, the partition function/scheme, the indexes, and the optional DB roles.
---
@@ -40,11 +40,13 @@ public sealed class AuditCentralHealthSnapshot
: IAuditCentralHealthSnapshot,
ICentralAuditWriteFailureCounter,
IAuditRedactionFailureCounter,
IAuditInboundCeilingHitsCounter
IAuditInboundCeilingHitsCounter,
IAuditPurgeFailureCounter
{
private int _centralAuditWriteFailures;
private int _auditRedactionFailure;
private int _auditInboundCeilingHits;
private int _auditPurgeFailures;
private readonly ConcurrentDictionary<string, bool> _stalled = new();
/// <inheritdoc/>
@@ -59,6 +61,10 @@ public sealed class AuditCentralHealthSnapshot
public int AuditInboundCeilingHits =>
Interlocked.CompareExchange(ref _auditInboundCeilingHits, 0, 0);
/// <inheritdoc/>
public int PurgeFailures =>
Interlocked.CompareExchange(ref _auditPurgeFailures, 0, 0);
/// <inheritdoc/>
public IReadOnlyDictionary<string, bool> SiteAuditTelemetryStalled =>
new Dictionary<string, bool>(_stalled);
@@ -88,4 +94,8 @@ public sealed class AuditCentralHealthSnapshot
/// <inheritdoc/>
void IAuditInboundCeilingHitsCounter.Increment() =>
Interlocked.Increment(ref _auditInboundCeilingHits);
/// <inheritdoc/>
void IAuditPurgeFailureCounter.Increment() =>
Interlocked.Increment(ref _auditPurgeFailures);
}
@@ -149,7 +149,14 @@ public class AuditLogPurgeActor : ReceiveActor
return;
}
// Optional purge-failure counter — resolved from the same per-tick scope the repository
// comes from. NoOp default on site/test roots; the central composition root binds the
// AuditCentralHealthSnapshot so failures surface on the central health dashboard. Mirrors
// AuditLogIngestActor's ICentralAuditWriteFailureCounter resolution shape.
var purgeFailureCounter = scope.ServiceProvider.GetService<IAuditPurgeFailureCounter>();
IReadOnlyList<DateTime> boundaries;
var enumSw = Stopwatch.StartNew();
try
{
boundaries = await repository
@@ -158,10 +165,14 @@ public class AuditLogPurgeActor : ReceiveActor
}
catch (Exception ex)
{
enumSw.Stop();
_logger.LogError(
ex,
"Failed to enumerate eligible AuditLog partition boundaries (threshold {ThresholdUtc:o}); skipping purge tick.",
threshold);
// Enumeration threw before any boundary was selected — publish with the
// DateTime.MinValue sentinel so the failure is still observable on the health surface.
PublishPurgeFailure(eventStream, purgeFailureCounter, DateTime.MinValue, ex, enumSw.ElapsedMilliseconds);
return;
}
@@ -169,7 +180,7 @@ public class AuditLogPurgeActor : ReceiveActor
{
// No whole-month partitions are eligible, but per-channel overrides may
// still expire rows earlier than the global window — run them below.
await RunPerChannelOverridesAsync(repository).ConfigureAwait(false);
await RunPerChannelOverridesAsync(repository, eventStream, purgeFailureCounter).ConfigureAwait(false);
return;
}
@@ -182,7 +193,7 @@ public class AuditLogPurgeActor : ReceiveActor
try
{
var rowsDeleted = await repository
.SwitchOutPartitionAsync(boundary)
.SwitchOutPartitionAsync(boundary, _purgeOptions.ResolvedMaintenanceCommandTimeout)
.ConfigureAwait(false);
sw.Stop();
@@ -203,6 +214,7 @@ public class AuditLogPurgeActor : ReceiveActor
"Failed to purge AuditLog partition {MonthBoundary:yyyy-MM-dd}; other partitions continue. Elapsed {DurationMs} ms.",
boundary,
sw.ElapsedMilliseconds);
PublishPurgeFailure(eventStream, purgeFailureCounter, boundary, ex, sw.ElapsedMilliseconds);
}
}
@@ -212,7 +224,24 @@ public class AuditLogPurgeActor : ReceiveActor
// switch-out has already dropped whole months older than RetentionDays; these
// deletes only ever expire rows EARLIER than that, so they run last and are a
// strict tightening.
await RunPerChannelOverridesAsync(repository).ConfigureAwait(false);
await RunPerChannelOverridesAsync(repository, eventStream, purgeFailureCounter).ConfigureAwait(false);
}
/// <summary>
/// Publishes an <see cref="AuditLogPurgeFailedEvent"/> on the EventStream and increments the
/// central purge-failure health counter (when wired). Best-effort — the symmetric failure twin
/// of the success-path <see cref="AuditLogPurgedEvent"/> publish, so a silently failing
/// retention job is a health-surface condition, not just a log line (arch-review 04, S2).
/// </summary>
private static void PublishPurgeFailure(
Akka.Event.EventStream eventStream,
IAuditPurgeFailureCounter? counter,
DateTime monthBoundary,
Exception ex,
long elapsedMilliseconds)
{
eventStream.Publish(new AuditLogPurgeFailedEvent(monthBoundary, ex.Message, elapsedMilliseconds));
counter?.Increment();
}
/// <summary>
@@ -224,7 +253,12 @@ public class AuditLogPurgeActor : ReceiveActor
/// per-boundary error isolation of the partition switch-out loop.
/// </summary>
/// <param name="repository">The repository resolved for this tick's DI scope.</param>
private async Task RunPerChannelOverridesAsync(IAuditLogRepository repository)
/// <param name="eventStream">The actor-system EventStream captured before the first await.</param>
/// <param name="purgeFailureCounter">Optional central purge-failure health counter (NoOp on site/test roots).</param>
private async Task RunPerChannelOverridesAsync(
IAuditLogRepository repository,
Akka.Event.EventStream eventStream,
IAuditPurgeFailureCounter? purgeFailureCounter)
{
var overrides = _auditOptions.PerChannelRetentionDays;
if (overrides is null || overrides.Count == 0)
@@ -250,7 +284,11 @@ public class AuditLogPurgeActor : ReceiveActor
try
{
var rowsDeleted = await repository
.PurgeChannelOlderThanAsync(channel, channelThreshold, _purgeOptions.ChannelPurgeBatchSize)
.PurgeChannelOlderThanAsync(
channel,
channelThreshold,
_purgeOptions.ChannelPurgeBatchSize,
_purgeOptions.ResolvedMaintenanceCommandTimeout)
.ConfigureAwait(false);
sw.Stop();
@@ -277,6 +315,10 @@ public class AuditLogPurgeActor : ReceiveActor
channel,
days,
sw.ElapsedMilliseconds);
// A per-channel DELETE failure is a purge failure too — surface it alongside the
// switch-out failures. MonthBoundary is DateTime.MinValue (this is not a
// whole-month switch), matching the enumeration-phase sentinel.
PublishPurgeFailure(eventStream, purgeFailureCounter, DateTime.MinValue, ex, sw.ElapsedMilliseconds);
}
}
}
@@ -0,0 +1,29 @@
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// <summary>
/// Symmetric failure twin of <see cref="AuditLogPurgedEvent"/>. Published on the
/// actor-system EventStream by <see cref="AuditLogPurgeActor"/> whenever a purge
/// step throws — a partition switch-out, a per-channel override DELETE, or the
/// boundary enumeration that precedes them. A silently failing retention job is a
/// health-surface condition, not just a log line, so this event lets the central
/// health collector + ops dashboards observe purge failures without coupling to
/// the actor (arch-review 04, S2).
/// </summary>
/// <param name="MonthBoundary">
/// The pf_AuditLog_Month lower-bound boundary whose switch-out failed. When the
/// failure occurs BEFORE any specific boundary is selected — i.e. the pre-purge
/// boundary enumeration threw — this is <see cref="System.DateTime.MinValue"/> as a
/// sentinel for "enumeration phase, no boundary in hand".
/// </param>
/// <param name="Error">
/// The failure's message (<see cref="System.Exception.Message"/>). The full
/// exception is logged at Error on the actor's logger; this carries the short form
/// for surfacing on health/ops without leaking a full stack trace onto the stream.
/// </param>
/// <param name="ElapsedMilliseconds">
/// Wall-clock time spent on the failed step before it threw, in milliseconds.
/// </param>
public sealed record AuditLogPurgeFailedEvent(
DateTime MonthBoundary,
string Error,
long ElapsedMilliseconds);
@@ -56,6 +56,30 @@ public sealed class AuditLogPurgeOptions
/// </summary>
public int ChannelPurgeBatchSize => ChannelPurgeBatchSizeConfigured < 1 ? 1 : ChannelPurgeBatchSizeConfigured;
/// <summary>
/// Per-command timeout (in minutes) for the maintenance SQL the purge tick issues —
/// both the partition switch-out drop-and-rebuild dance
/// (<see cref="ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories.IAuditLogRepository.SwitchOutPartitionAsync"/>)
/// and each per-channel <c>DELETE TOP</c> batch. Default 30 minutes.
/// </summary>
/// <remarks>
/// The ADO.NET default command timeout is ~30 seconds. On a large or contended partition the
/// SWITCH dance (which briefly drops <c>UX_AuditLog_EventId</c>) can exceed that and abort
/// mid-flight — leaving the live table without its idempotency-supporting unique index until a
/// later tick's CATCH branch rebuilds it (arch-review 04, S2). A generous maintenance timeout
/// lets the metadata-only switch complete rather than self-locking. Resolved via
/// <see cref="ResolvedMaintenanceCommandTimeout"/>, clamped to a 1-minute floor.
/// </remarks>
public int MaintenanceCommandTimeoutMinutes { get; set; } = 30;
/// <summary>
/// Resolves the effective maintenance command timeout, clamped to at least 1 minute so a
/// misconfigured <c>0</c>/negative value cannot yield a zero/negative timeout (which ADO.NET
/// interprets as "no timeout" for <c>0</c> and rejects for negatives).
/// </summary>
public TimeSpan ResolvedMaintenanceCommandTimeout =>
TimeSpan.FromMinutes(MaintenanceCommandTimeoutMinutes < 1 ? 1 : MaintenanceCommandTimeoutMinutes);
/// <summary>
/// Test-only override for finer control over the tick cadence than
/// whole-hour resolution allows. When non-null, takes precedence over
@@ -61,6 +61,16 @@ public interface IAuditCentralHealthSnapshot
/// </summary>
int AuditInboundCeilingHits { get; }
/// <summary>
/// Count of AuditLog partition-purge failures since process start. Incremented by
/// <see cref="AuditLogPurgeActor"/> every time a purge step throws — a partition
/// switch-out, a per-channel override DELETE, or the pre-purge boundary enumeration.
/// A sustained non-zero count means the retention job is silently not draining the
/// table; each failure also publishes an <see cref="AuditLogPurgeFailedEvent"/> on the
/// EventStream (arch-review 04, S2).
/// </summary>
int PurgeFailures { get; }
/// <summary>
/// Per-site latched stalled state: <c>true</c> when the
/// <see cref="SiteAuditReconciliationActor"/> has observed two
@@ -0,0 +1,21 @@
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// <summary>
/// Counter sink invoked by <see cref="AuditLogPurgeActor"/> every time a purge
/// step throws (partition switch-out, per-channel override DELETE, or the pre-purge
/// boundary enumeration). Mirrors <see cref="ICentralAuditWriteFailureCounter"/>
/// one-for-one — same one-method contract, same NoOp default, same
/// never-abort-the-tick invariant (the purge actor swallows the failure per
/// boundary and continues).
/// </summary>
/// <remarks>
/// A NoOp default is the safe fallback for site/test composition roots that do not
/// wire the central health surface; <see cref="AuditCentralHealthSnapshot"/> is the
/// production binding that routes increments into the aggregated central health
/// snapshot's <see cref="IAuditCentralHealthSnapshot.PurgeFailures"/> read surface.
/// </remarks>
public interface IAuditPurgeFailureCounter
{
/// <summary>Increment the central AuditLog purge-failure counter by one.</summary>
void Increment();
}
@@ -0,0 +1,16 @@
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
/// <summary>
/// Default <see cref="IAuditPurgeFailureCounter"/> binding used when the central
/// health surface (<see cref="AuditCentralHealthSnapshot"/>) has not been wired
/// (test composition roots, site-only hosts). Drops every increment on the floor.
/// Mirrors <see cref="NoOpCentralAuditWriteFailureCounter"/>.
/// </summary>
public sealed class NoOpAuditPurgeFailureCounter : IAuditPurgeFailureCounter
{
/// <inheritdoc/>
public void Increment()
{
// intentional no-op
}
}
@@ -101,6 +101,11 @@ public static class ServiceCollectionExtensions
.Bind(config.GetSection(SiteWriterSectionName));
services.AddOptions<SiteAuditTelemetryOptions>()
.Bind(config.GetSection(SiteTelemetrySectionName));
// Retention purge options (the SiteAuditRetentionService hosted job is
// registered site-only in AddAuditLogHealthMetricsBridge). Binding here is
// inert on central — no hosted service consumes it there.
services.AddOptions<SiteAuditRetentionOptions>()
.Bind(config.GetSection(SiteAuditRetentionOptions.SectionName));
// SqliteAuditWriter is a singleton with a single owned SqliteConnection
// and a background writer Task; multiple instances would race on the
@@ -207,6 +212,12 @@ public static class ServiceCollectionExtensions
// redaction-failure counters.
services.TryAddSingleton<IAuditInboundCeilingHitsCounter, NoOpAuditInboundCeilingHitsCounter>();
// AuditLog purge-failure counter — NoOp default for site/test
// composition roots. AddAuditLogCentralMaintenance below replaces this binding
// with the AuditCentralHealthSnapshot implementation so partition/channel purge
// failures surface on the central dashboard alongside the write-failure counter.
services.TryAddSingleton<IAuditPurgeFailureCounter, NoOpAuditPurgeFailureCounter>();
// Central direct-write audit writer used by
// NotificationOutboxActor and Inbound API to
// emit AuditLog rows that originate ON central, not via site telemetry.
@@ -316,6 +327,12 @@ public static class ServiceCollectionExtensions
// free of hosted-service registrations that would resolve a missing
// ISiteHealthCollector on central.
services.AddHostedService<SiteAuditBacklogReporter>();
// Site-only retention purge: runs ISiteAuditQueue.PurgeExpiredAsync on a
// daily timer so the ephemeral site SQLite store does not grow unbounded.
// Registered here (not in AddAuditLog, which also runs on central) so the
// hosted service only starts on site composition roots. Guarded by the same
// SiteAuditBacklogReporter sentinel above against double-registration.
services.AddHostedService<SiteAuditRetentionService>();
return services;
}
@@ -408,6 +425,11 @@ public static class ServiceCollectionExtensions
// ICentralAuditWriteFailureCounter above.
services.Replace(ServiceDescriptor.Singleton<IAuditInboundCeilingHitsCounter>(
sp => sp.GetRequiredService<AuditCentralHealthSnapshot>()));
// Replace the NoOp IAuditPurgeFailureCounter with the AuditCentralHealthSnapshot so
// partition/channel purge failures surface on the central dashboard. Same
// singleton-forward pattern as ICentralAuditWriteFailureCounter above (arch-review 04, S2).
services.Replace(ServiceDescriptor.Singleton<IAuditPurgeFailureCounter>(
sp => sp.GetRequiredService<AuditCentralHealthSnapshot>()));
return services;
}
@@ -0,0 +1,77 @@
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site;
/// <summary>
/// Configuration for the site-side SQLite retention purge (PLAN-04 Task 2, S1/U2).
/// The site audit store is ephemeral (~7-day retention); <see cref="SiteAuditRetentionService"/>
/// runs <see cref="ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services.ISiteAuditQueue.PurgeExpiredAsync"/>
/// on a timer using these knobs. Mirrors the resolve/clamp shape of
/// <c>SiteCallAuditOptions</c> so a misconfigured value can never make the timer spin
/// or the retention window collapse.
/// </summary>
public class SiteAuditRetentionOptions
{
/// <summary>
/// Configuration section path bound in the site host. Follows the component's
/// existing <c>AuditLog:*</c> convention (sibling of <c>AuditLog:SiteWriter</c>,
/// <c>AuditLog:Purge</c>, …), not an application-root-absolute path.
/// </summary>
public const string SectionName = "AuditLog:SiteRetention";
/// <summary>
/// Retention window in days. On each purge tick, Forwarded/Reconciled rows whose
/// <c>OccurredAtUtc</c> is older than <c>UtcNow - ResolvedRetentionDays</c> are
/// deleted (Pending rows are never purged on age alone). Default 7. Resolved value
/// is clamped to <see cref="MinRetentionDays"/>..<see cref="MaxRetentionDays"/>.
/// </summary>
public int RetentionDays { get; set; } = 7;
private const int MinRetentionDays = 1;
private const int MaxRetentionDays = 90;
/// <summary>
/// Effective retention window: <see cref="RetentionDays"/> clamped to
/// [<see cref="MinRetentionDays"/>, <see cref="MaxRetentionDays"/>] so a
/// misconfigured 0 (which would purge everything) or an unbounded value can
/// never take effect.
/// </summary>
public int ResolvedRetentionDays =>
RetentionDays < MinRetentionDays ? MinRetentionDays
: RetentionDays > MaxRetentionDays ? MaxRetentionDays
: RetentionDays;
/// <summary>
/// Period of the purge tick. Default 24 hours. Resolved value is clamped to at
/// least <see cref="MinPurgeInterval"/> so a zero/negative config value cannot
/// make the timer fire in a tight loop.
/// </summary>
public TimeSpan PurgeInterval { get; set; } = TimeSpan.FromHours(24);
/// <summary>
/// Test-only override for the purge cadence — bypasses the
/// <see cref="MinPurgeInterval"/> clamp so unit tests can drop the period to
/// milliseconds. Production config never sets this; leave null. Mirrors
/// <c>SiteCallAuditOptions.PurgeIntervalOverride</c>.
/// </summary>
public TimeSpan? PurgeIntervalOverride { get; set; }
private static readonly TimeSpan MinPurgeInterval = TimeSpan.FromMinutes(1);
/// <summary>
/// Effective purge period: the test override when set (bypassing the clamp),
/// otherwise <see cref="PurgeInterval"/> clamped to at least
/// <see cref="MinPurgeInterval"/> (the Timer/Akka zero-interval spin footgun).
/// </summary>
public TimeSpan ResolvedPurgeInterval =>
PurgeIntervalOverride is { } o
? o
: PurgeInterval < MinPurgeInterval
? MinPurgeInterval
: PurgeInterval;
/// <summary>
/// Delay before the first purge tick. Default 5 minutes — short so a node that
/// restarts daily (or more often) still runs the purge shortly after start
/// rather than waiting a full <see cref="PurgeInterval"/> it may never reach.
/// </summary>
public TimeSpan InitialDelay { get; set; } = TimeSpan.FromMinutes(5);
}
@@ -0,0 +1,133 @@
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site;
/// <summary>
/// Site-side hosted service that runs the SQLite retention purge on a timer
/// (PLAN-04 Task 3, S1/U2). Each tick calls
/// <see cref="ISiteAuditQueue.PurgeExpiredAsync"/> with a cutoff of
/// <c>UtcNow - <see cref="SiteAuditRetentionOptions.ResolvedRetentionDays"/></c>,
/// deleting Forwarded/Reconciled rows past the retention window and reclaiming the
/// freed pages. Without it the ephemeral site store grows unbounded until the next
/// deployment recreates it.
/// </summary>
/// <remarks>
/// <para>
/// <b>Cadence.</b> First tick after <see cref="SiteAuditRetentionOptions.InitialDelay"/>
/// (default 5 min — short so a node that recycles daily still purges shortly after
/// start), then every <see cref="SiteAuditRetentionOptions.ResolvedPurgeInterval"/>
/// (default 24 h). Mirrors the loop shape of <see cref="SiteAuditBacklogReporter"/>.
/// </para>
/// <para>
/// <b>Failure containment.</b> Each purge is wrapped in a try/catch so a transient
/// SQLite error never tears down the hosted service — the next tick retries.
/// </para>
/// </remarks>
public sealed class SiteAuditRetentionService : IHostedService, IDisposable
{
private readonly ISiteAuditQueue _queue;
private readonly SiteAuditRetentionOptions _options;
private readonly ILogger<SiteAuditRetentionService> _logger;
private CancellationTokenSource? _cts;
private Task? _loop;
/// <summary>Initializes a new instance of <see cref="SiteAuditRetentionService"/>.</summary>
/// <param name="queue">The site audit queue whose retention purge is driven.</param>
/// <param name="options">Retention cadence + window options.</param>
/// <param name="logger">Logger instance.</param>
public SiteAuditRetentionService(
ISiteAuditQueue queue,
IOptions<SiteAuditRetentionOptions> options,
ILogger<SiteAuditRetentionService> logger)
{
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
/// <summary>Starts the background purge loop.</summary>
/// <param name="ct">Cancellation token signalling host shutdown.</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public Task StartAsync(CancellationToken ct)
{
// Linked CTS so both StopAsync and the host shutdown token abort the loop.
_cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
_loop = Task.Run(() => RunLoopAsync(_cts.Token));
return Task.CompletedTask;
}
private async Task RunLoopAsync(CancellationToken ct)
{
// InitialDelay before the first purge — short so a daily-recycled node
// still purges soon after start rather than waiting a full interval it may
// never reach.
try
{
await Task.Delay(_options.InitialDelay, ct).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
return;
}
await SafePurgeAsync(ct).ConfigureAwait(false);
while (!ct.IsCancellationRequested)
{
try
{
await Task.Delay(_options.ResolvedPurgeInterval, ct).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
break;
}
await SafePurgeAsync(ct).ConfigureAwait(false);
}
}
private async Task SafePurgeAsync(CancellationToken ct)
{
try
{
var cutoff = DateTime.UtcNow.AddDays(-_options.ResolvedRetentionDays);
var purged = await _queue.PurgeExpiredAsync(cutoff, ct).ConfigureAwait(false);
if (purged > 0)
{
_logger.LogInformation(
"Site audit retention purge removed {Count} row(s) older than {Cutoff:o}.",
purged, cutoff);
}
}
catch (OperationCanceledException)
{
// Shutdown — let the outer loop exit cleanly.
throw;
}
catch (Exception ex)
{
// Catch-all is deliberate: a transient SQLite fault must not tear down
// the hosted service; the next tick retries.
_logger.LogWarning(ex, "SiteAuditRetentionService purge failed; next tick will retry.");
}
}
/// <summary>Signals the purge loop to stop and waits for it to complete.</summary>
/// <param name="ct">Cancellation token (not used; the internal CTS governs shutdown).</param>
/// <returns>A task that represents the asynchronous operation.</returns>
public Task StopAsync(CancellationToken ct)
{
_cts?.Cancel();
return _loop ?? Task.CompletedTask;
}
/// <summary>Releases the internal <see cref="CancellationTokenSource"/>.</summary>
public void Dispose()
{
_cts?.Dispose();
}
}
@@ -141,9 +141,9 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
private void InitializeSchema()
{
// auto_vacuum must be set before any table is created for it to take
// effect on a fresh database. INCREMENTAL lets a future
// `PRAGMA incremental_vacuum` shrink the file after the 7-day retention
// purge — see alog.md §10.
// effect on a fresh database. INCREMENTAL lets the `PRAGMA
// incremental_vacuum` run at the end of PurgeExpiredAsync shrink the file
// after the ~7-day retention purge — see alog.md §10.
using (var pragmaCmd = _connection.CreateCommand())
{
pragmaCmd.CommandText = "PRAGMA auto_vacuum = INCREMENTAL";
@@ -821,6 +821,117 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
OnDiskBytes: onDiskBytes));
}
/// <inheritdoc />
public Task<int> PurgeExpiredAsync(DateTime olderThanUtc, CancellationToken ct = default)
{
// Runs on the WRITE connection under _writeLock (a DELETE mutates the file),
// mirroring MarkForwardedAsync / MarkReconciledAsync. Serialising with the
// batched writer means the purge never races an in-flight FlushBatch.
lock (_writeLock)
{
ObjectDisposedException.ThrowIf(_disposed, this);
// Cutoff uses the same round-trip "o" encoding the writer stores
// OccurredAtUtc in, so string comparison stays monotonic (see FlushBatch).
var cutoff = EnsureUtc(olderThanUtc).ToString(
"o", System.Globalization.CultureInfo.InvariantCulture);
int purged;
using (var transaction = _connection.BeginTransaction())
{
try
{
// Capture the eligible EventIds from the sidecar FIRST: the
// Forwarded/Reconciled + age criteria live on
// audit_forward_state, and it is the FK child of audit_event, so
// it must be deleted before the parent. A temp table (rather than
// an IN (...) list) keeps an arbitrarily large purge set off the
// SQLite 999-parameter limit. The hard invariant is encoded here:
// only Forwarded/Reconciled rows qualify — Pending is never purged
// on age alone.
using (var createCmd = _connection.CreateCommand())
{
createCmd.Transaction = transaction;
createCmd.CommandText = """
DROP TABLE IF EXISTS purge_ids;
CREATE TEMP TABLE purge_ids AS
SELECT EventId FROM audit_forward_state
WHERE ForwardState IN ($forwarded, $reconciled)
AND OccurredAtUtc < $cutoff;
""";
createCmd.Parameters.AddWithValue("$forwarded", AuditForwardState.Forwarded.ToString());
createCmd.Parameters.AddWithValue("$reconciled", AuditForwardState.Reconciled.ToString());
createCmd.Parameters.AddWithValue("$cutoff", cutoff);
createCmd.ExecuteNonQuery();
}
// Delete the sidecar (FK child) first, then the canonical row.
using (var delSidecar = _connection.CreateCommand())
{
delSidecar.Transaction = transaction;
delSidecar.CommandText =
"DELETE FROM audit_forward_state WHERE EventId IN (SELECT EventId FROM purge_ids);";
delSidecar.ExecuteNonQuery();
}
using (var delEvent = _connection.CreateCommand())
{
delEvent.Transaction = transaction;
delEvent.CommandText =
"DELETE FROM audit_event WHERE EventId IN (SELECT EventId FROM purge_ids);";
delEvent.ExecuteNonQuery();
}
// changes() reports rows affected by the most recent DELETE on
// this connection (the audit_event delete above); read it before
// the DROP so no later statement can perturb the count.
using (var changesCmd = _connection.CreateCommand())
{
changesCmd.Transaction = transaction;
changesCmd.CommandText = "SELECT changes();";
purged = Convert.ToInt32(changesCmd.ExecuteScalar(),
System.Globalization.CultureInfo.InvariantCulture);
}
using (var dropCmd = _connection.CreateCommand())
{
dropCmd.Transaction = transaction;
dropCmd.CommandText = "DROP TABLE IF EXISTS purge_ids;";
dropCmd.ExecuteNonQuery();
}
transaction.Commit();
}
catch
{
transaction.Rollback();
throw;
}
}
// Reclaim the freed pages back to the OS. auto_vacuum=INCREMENTAL (set in
// InitializeSchema) makes this a bounded incremental step rather than a
// full VACUUM rewrite. Runs outside the transaction; a vacuum failure must
// not fault the purge itself, so it is best-effort.
if (purged > 0)
{
try
{
using var vacuumCmd = _connection.CreateCommand();
vacuumCmd.CommandText = "PRAGMA incremental_vacuum;";
vacuumCmd.ExecuteNonQuery();
}
catch (Exception ex)
{
_logger.LogDebug(ex,
"SqliteAuditWriter incremental_vacuum after retention purge failed (non-fatal).");
}
}
return Task.FromResult(purged);
}
}
private static DateTime EnsureUtc(DateTime value) =>
value.Kind == DateTimeKind.Utc
? value
@@ -83,9 +83,17 @@ public interface IAuditLogRepository
/// </para>
/// </remarks>
/// <param name="monthBoundary">Lower-bound datetime of the monthly partition to switch out.</param>
/// <param name="commandTimeout">
/// Optional per-command timeout for the maintenance SQL (the row-count sample plus the
/// drop-and-rebuild dance). When null the provider default applies. The purge actor passes
/// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout"/>
/// (default 30 min) because the ~30s ADO.NET default can abort the switch mid-dance on a large
/// or contended partition, leaving the table without <c>UX_AuditLog_EventId</c> until the next
/// tick recovers (arch-review 04, S2).
/// </param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A task that resolves to the approximate number of rows discarded by the partition switch.</returns>
Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default);
Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default);
/// <summary>
/// Per-channel retention override purge. Deletes <c>AuditLog</c> rows for a
@@ -115,12 +123,20 @@ public interface IAuditLogRepository
/// <param name="channel">Canonical channel name (the <c>Category</c> column value, e.g. <c>ApiOutbound</c>).</param>
/// <param name="threshold">Rows with <c>OccurredAtUtc</c> strictly older than this UTC datetime are deleted.</param>
/// <param name="batchSize">Maximum rows deleted per batch; must be &gt; 0.</param>
/// <param name="commandTimeout">
/// Optional per-batch <c>DELETE TOP</c> command timeout. When null the provider default applies.
/// The purge actor passes the same
/// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout"/>
/// it uses for the partition switch so a large single batch under lock contention is not aborted
/// by the ~30s ADO.NET default (arch-review 04, S2).
/// </param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A task that resolves to the total number of rows deleted across all batches.</returns>
Task<long> PurgeChannelOlderThanAsync(
string channel,
DateTime threshold,
int batchSize,
TimeSpan? commandTimeout = null,
CancellationToken ct = default);
/// <summary>
@@ -142,4 +142,25 @@ public interface ISiteAuditQueue
/// <param name="ct">Cancellation token.</param>
/// <returns>A task that resolves to a point-in-time snapshot of the site audit queue's pending count, oldest timestamp, and on-disk file size.</returns>
Task<SiteAuditBacklogSnapshot> GetBacklogStatsAsync(CancellationToken ct = default);
/// <summary>
/// Retention purge: permanently deletes rows that have already left the site
/// (<see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AuditForwardState.Forwarded"/> or
/// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AuditForwardState.Reconciled"/>)
/// whose <see cref="AuditEvent.OccurredAtUtc"/> is strictly older than
/// <paramref name="olderThanUtc"/>, and reclaims the freed pages.
/// </summary>
/// <remarks>
/// <b>Hard <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AuditForwardState"/> invariant:</b>
/// a row still in <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AuditForwardState.Pending"/>
/// is NEVER purged on age alone — it has not yet reached central, so dropping it
/// would silently lose audit data. Only Forwarded/Reconciled rows are eligible.
/// Idempotent: a second call with the same cutoff removes nothing further and
/// returns 0. The site store is ephemeral (~7-day retention), so this bounds its
/// unbounded growth (arch-review 04, S1/U2).
/// </remarks>
/// <param name="olderThanUtc">Exclusive upper bound on <see cref="AuditEvent.OccurredAtUtc"/>; rows at or after this instant are retained.</param>
/// <param name="ct">Cancellation token.</param>
/// <returns>A task that resolves to the number of canonical rows deleted.</returns>
Task<int> PurgeExpiredAsync(DateTime olderThanUtc, CancellationToken ct = default);
}
@@ -66,8 +66,9 @@ public class SiteCallEntityTypeConfiguration : IEntityTypeConfiguration<SiteCall
// SourceNode (SourceNode-stamping): node-local identifier of the
// cluster member that produced the call (e.g. "node-a", "central-a"). NULL is
// valid for rows that pre-date this feature and for reconciled rows from a
// retired node. ASCII — varchar(64). No index — Site Call Audit KPIs are
// per-site, not per-node, on this table.
// retired node. ASCII — varchar(64). Not a key column of any index, but it rides
// as an INCLUDE column on IX_SiteCalls_NonTerminal below so the per-node KPI
// breakdown reads it covered off the filtered live-queue index.
builder.Property(s => s.SourceNode)
.HasColumnType("varchar(64)")
.HasMaxLength(64)
@@ -85,5 +86,16 @@ public class SiteCallEntityTypeConfiguration : IEntityTypeConfiguration<SiteCall
builder.HasIndex(s => new { s.Status, s.UpdatedAtUtc })
.IsDescending(false, true)
.HasDatabaseName("IX_SiteCalls_Status_Updated");
// NonTerminal backs every "live queue" KPI predicate (TerminalAtUtc IS NULL):
// buffered/stuck/oldest counts run every 60 s from the KPI recorder plus every
// 10 s Health-dashboard poll. Filtered: the non-terminal population is the live
// queue, not the 365-day archive — each count becomes a small seek instead of a
// clustered scan (arch-review 04, P1). SourceSite/SourceNode/Status ride as
// INCLUDE columns so the per-site + per-node KPI breakdowns stay covered.
builder.HasIndex(s => s.CreatedAtUtc)
.HasDatabaseName("IX_SiteCalls_NonTerminal")
.HasFilter("[TerminalAtUtc] IS NULL")
.IncludeProperties(nameof(SiteCall.SourceSite), nameof(SiteCall.SourceNode), nameof(SiteCall.Status));
}
}
@@ -0,0 +1,47 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
{
/// <summary>
/// Grants the segregated maintenance principal <c>scadabridge_audit_purger</c> the two
/// permissions its purge path actually needs but the original role creation
/// (<c>CollapseAuditLogToCanonical</c>) omitted (arch-review 04, S3):
/// <list type="bullet">
/// <item><c>CREATE TABLE</c> — the switch-out dance CREATEs a staging table; the
/// role's existing <c>ALTER ON SCHEMA::dbo</c> does NOT confer the database-level
/// <c>CREATE TABLE</c> permission.</item>
/// <item><c>DELETE ON dbo.AuditLog</c> — the per-channel retention override
/// (<c>PurgeChannelOlderThanAsync</c>) is a bounded row DELETE on the maintenance path.</item>
/// </list>
/// Permission DDL only — no schema/data change. Idempotent and safe to run standalone.
/// </summary>
public partial class FixAuditPurgerRoleGrants : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.Sql(@"
IF DATABASE_PRINCIPAL_ID('scadabridge_audit_purger') IS NULL
EXEC sp_executesql N'CREATE ROLE scadabridge_audit_purger';
-- The switch-out dance CREATEs a staging table; ALTER ON SCHEMA::dbo alone does not
-- confer the database-level CREATE TABLE permission (review finding S3).
GRANT CREATE TABLE TO scadabridge_audit_purger;
-- The per-channel retention override (PurgeChannelOlderThanAsync) is a bounded row
-- DELETE on the maintenance path; a segregated purger principal needs this grant.
GRANT DELETE ON dbo.AuditLog TO scadabridge_audit_purger;"); // AUDIT-PURGE-ALLOWED: role grant for the maintenance principal
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.Sql(@"
IF DATABASE_PRINCIPAL_ID('scadabridge_audit_purger') IS NOT NULL
BEGIN
REVOKE CREATE TABLE FROM scadabridge_audit_purger;
REVOKE DELETE ON dbo.AuditLog FROM scadabridge_audit_purger;
END"); // AUDIT-PURGE-ALLOWED: role grant reversal for the maintenance principal
}
}
}
@@ -0,0 +1,29 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
{
/// <inheritdoc />
public partial class AddSiteCallsNonTerminalIndex : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateIndex(
name: "IX_SiteCalls_NonTerminal",
table: "SiteCalls",
column: "CreatedAtUtc",
filter: "[TerminalAtUtc] IS NULL")
.Annotation("SqlServer:Include", new[] { "SourceSite", "SourceNode", "Status" });
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_SiteCalls_NonTerminal",
table: "SiteCalls");
}
}
}
@@ -161,6 +161,12 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
b.HasKey("TrackedOperationId");
b.HasIndex("CreatedAtUtc")
.HasDatabaseName("IX_SiteCalls_NonTerminal")
.HasFilter("[TerminalAtUtc] IS NULL");
SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("CreatedAtUtc"), new[] { "SourceSite", "SourceNode", "Status" });
b.HasIndex("SourceSite", "CreatedAtUtc")
.IsDescending(false, true)
.HasDatabaseName("IX_SiteCalls_Source_Created");
@@ -227,8 +227,13 @@ VALUES
};
/// <inheritdoc />
public async Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default)
public async Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default)
{
// Maintenance timeout in whole seconds (ADO.NET CommandTimeout unit). Null leaves the
// provider default in place. See AuditLogPurgeOptions.MaintenanceCommandTimeoutMinutes /
// arch-review 04 S2 for why the ~30s default is unsafe for the switch-out dance.
var commandTimeoutSeconds = commandTimeout is { } ct2 ? (int)ct2.TotalSeconds : (int?)null;
// GUID-suffixed staging name: prevents collision with any concurrent
// purge attempt and avoids polluting the AuditLog object namespace with
// a predictable identifier.
@@ -351,6 +356,10 @@ VALUES
await using (var sampleCmd = conn.CreateCommand())
{
sampleCmd.CommandText = sampleSql;
if (commandTimeoutSeconds is { } sampleTimeout)
{
sampleCmd.CommandTimeout = sampleTimeout;
}
var sampleResult = await sampleCmd.ExecuteScalarAsync(ct).ConfigureAwait(false);
if (sampleResult is not null && sampleResult is not DBNull)
{
@@ -366,7 +375,27 @@ VALUES
}
}
await _context.Database.ExecuteSqlRawAsync(sql, ct);
// Apply the maintenance timeout to the drop-and-rebuild dance. The context is scoped per
// purge tick (see AuditLogPurgeActor.OnTickAsync's CreateAsyncScope), so a restore is not
// strictly required — but we restore the previous value in a finally for hygiene in case a
// future caller reuses the context for other work after the switch-out.
var previousTimeout = _context.Database.GetCommandTimeout();
if (commandTimeout is { } maintenanceTimeout)
{
_context.Database.SetCommandTimeout(maintenanceTimeout);
}
try
{
await _context.Database.ExecuteSqlRawAsync(sql, ct);
}
finally
{
if (commandTimeout is not null)
{
_context.Database.SetCommandTimeout(previousTimeout);
}
}
return rowsDeleted;
}
@@ -375,6 +404,7 @@ VALUES
string channel,
DateTime threshold,
int batchSize,
TimeSpan? commandTimeout = null,
CancellationToken ct = default)
{
if (string.IsNullOrWhiteSpace(channel))
@@ -389,6 +419,9 @@ VALUES
var thresholdUtc = DateTime.SpecifyKind(threshold.ToUniversalTime(), DateTimeKind.Utc);
// Maintenance timeout in whole seconds; null leaves the provider default (arch-review 04 S2).
var commandTimeoutSeconds = commandTimeout is { } cto ? (int)cto.TotalSeconds : (int?)null;
// Per-channel retention override purge. This is the ONLY DELETE
// against dbo.AuditLog in the codebase and it runs on the purge/maintenance
// path, NOT the append-only writer role (which has INSERT + SELECT only — see
@@ -427,6 +460,10 @@ VALUES
await using var cmd = conn.CreateCommand();
cmd.CommandText = deleteBatchSql;
if (commandTimeoutSeconds is { } batchTimeout)
{
cmd.CommandTimeout = batchTimeout;
}
var pBatch = cmd.CreateParameter();
pBatch.ParameterName = "@batch";
@@ -628,6 +665,16 @@ VALUES
// practice.
private const int ExecutionChainMaxDepth = 32;
// Execution trees span minutes, not years — an inbound request and every script it
// spawns all fire inside one operational burst. Anchoring the Edges CTE scan to a
// narrow window around the root's first event lets SQL Server eliminate every
// AuditLog partition outside it, turning a full-table DISTINCT scan into a seek on
// IX_AuditLog_Execution over one or two months (arch-review 04, P2). The slack absorbs
// minor clock skew before the root; the max span is the documented traversal bound —
// a genuine descendant stamped beyond it is excluded by design.
private static readonly TimeSpan ExecutionTreeWindowSlack = TimeSpan.FromHours(1);
private static readonly TimeSpan ExecutionTreeMaxSpan = TimeSpan.FromDays(7);
/// <inheritdoc />
public async Task<IReadOnlyList<ExecutionTreeNode>> GetExecutionTreeAsync(
Guid executionId,
@@ -676,6 +723,31 @@ VALUES
rootExecutionId = parent.Value;
}
// --- Between phases: bound the down-walk to a root-anchored window ---
// The root's first event timestamp anchors a [rootFirst - slack, rootFirst + maxSpan)
// window used to prune the Edges CTE scan to a handful of partitions. A row-less stub
// root (purged/no-action parent) has no MIN — leave the scan unbounded in that
// degenerate case (correctness over speed).
DateTime? rootFirstOccurred;
await using (var winCmd = conn.CreateCommand())
{
winCmd.CommandText =
"SELECT MIN(OccurredAtUtc) FROM dbo.AuditLog WHERE ExecutionId = @root;";
var pWinRoot = winCmd.CreateParameter();
pWinRoot.ParameterName = "@root";
pWinRoot.Value = rootExecutionId;
winCmd.Parameters.Add(pWinRoot);
var winResult = await winCmd.ExecuteScalarAsync(ct).ConfigureAwait(false);
rootFirstOccurred = winResult is null or DBNull ? null : (DateTime)winResult;
}
// Build the optional window predicate for the Edges CTE. When the root has rows the
// scan is confined to the window (partition elimination); otherwise it stays unbounded.
var edgesWindowPredicate = rootFirstOccurred is null
? string.Empty
: "AND OccurredAtUtc >= @winStart AND OccurredAtUtc < @winEnd";
// --- Phase 2: walk down from the root via a recursive CTE ---------
// Edges : a non-recursive, DISTINCT (ExecutionId, ParentExecutionId)
// edge set distilled from AuditLog. Recursing over edges
@@ -706,6 +778,7 @@ VALUES
SELECT DISTINCT ExecutionId, ParentExecutionId
FROM dbo.AuditLog
WHERE ExecutionId IS NOT NULL
{edgesWindowPredicate}
),
Chain AS (
-- Anchor: the root execution id, seeded as a literal so
@@ -760,6 +833,19 @@ VALUES
pRoot.Value = rootExecutionId;
downCmd.Parameters.Add(pRoot);
if (rootFirstOccurred is { } rootFirst)
{
var pWinStart = downCmd.CreateParameter();
pWinStart.ParameterName = "@winStart";
pWinStart.Value = rootFirst - ExecutionTreeWindowSlack;
downCmd.Parameters.Add(pWinStart);
var pWinEnd = downCmd.CreateParameter();
pWinEnd.ParameterName = "@winEnd";
pWinEnd.Value = rootFirst + ExecutionTreeMaxSpan;
downCmd.Parameters.Add(pWinEnd);
}
await using var reader = await downCmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
while (await reader.ReadAsync(ct).ConfigureAwait(false))
{
@@ -259,12 +259,12 @@ public class AuditLogIngestActorTests : TestKit, IClassFixture<MsSqlMigrationFix
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default) =>
_inner.QueryAsync(filter, paging, ct);
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default) =>
_inner.SwitchOutPartitionAsync(monthBoundary, ct);
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
_inner.SwitchOutPartitionAsync(monthBoundary, commandTimeout, ct);
public Task<long> PurgeChannelOlderThanAsync(
string channel, DateTime threshold, int batchSize, CancellationToken ct = default) =>
_inner.PurgeChannelOlderThanAsync(channel, threshold, batchSize, ct);
string channel, DateTime threshold, int batchSize, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
_inner.PurgeChannelOlderThanAsync(channel, threshold, batchSize, commandTimeout, ct);
public Task<long> BackfillSourceNodeAsync(
string sentinel, DateTime before, int batchSize, CancellationToken ct = default) =>
@@ -51,6 +51,11 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
public DateTime? ThrowOnBoundary { get; set; }
public Exception? BoundaryException { get; set; }
// Task 4: captures the maintenance command timeout the actor threads through to the
// switch-out + per-channel purge, so tests can assert the options value reaches the repo.
public List<TimeSpan?> SwitchTimeouts { get; } = new();
public List<TimeSpan?> ChannelTimeouts { get; } = new();
// M5.5 (T3): records every per-channel purge call as
// (channel, threshold, batchSize) so tests can assert which channels the
// actor chose to purge and with what window.
@@ -69,8 +74,9 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default) =>
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>());
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default)
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default)
{
SwitchTimeouts.Add(commandTimeout);
if (ThrowOnBoundary.HasValue && monthBoundary == ThrowOnBoundary.Value)
{
throw BoundaryException ?? new InvalidOperationException("simulated switch failure");
@@ -87,9 +93,10 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
}
public Task<long> PurgeChannelOlderThanAsync(
string channel, DateTime threshold, int batchSize, CancellationToken ct = default)
string channel, DateTime threshold, int batchSize, TimeSpan? commandTimeout = null, CancellationToken ct = default)
{
ChannelPurges.Add((channel, threshold, batchSize));
ChannelTimeouts.Add(commandTimeout);
return Task.FromResult(RowsPerChannel(channel));
}
@@ -491,4 +498,95 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
Assert.Empty(repo.ChannelPurges);
}
// ---------------------------------------------------------------------
// 10. PurgeTick_Passes_MaintenanceCommandTimeout_To_Repository (Task 4)
// ---------------------------------------------------------------------
[Fact]
public void PurgeTick_Passes_MaintenanceCommandTimeout_To_Repository()
{
// The actor must thread AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout through to
// BOTH the partition switch-out and each per-channel purge, so the ~30s ADO.NET default
// cannot self-lock the drop-and-rebuild dance (arch-review 04, S2).
var boundary = new DateTime(2025, 5, 1, 0, 0, 0, DateTimeKind.Utc);
var repo = new RecordingRepo
{
Boundaries = new List<DateTime> { boundary },
RowsPerBoundary = _ => 7L,
};
var purgeOptions = FastTickOptions();
purgeOptions.MaintenanceCommandTimeoutMinutes = 45;
// A per-channel override shorter than the global window so the channel purge also fires.
var auditOptions = Options.Create(new AuditLogOptions
{
RetentionDays = 365,
PerChannelRetentionDays = new Dictionary<string, int> { ["ApiOutbound"] = 30 },
});
var sp = BuildScopedProvider(repo);
Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
sp,
Options.Create(purgeOptions),
auditOptions,
NullLogger<AuditLogPurgeActor>.Instance)));
var expected = TimeSpan.FromMinutes(45);
AwaitAssert(
() =>
{
Assert.Contains(expected, repo.SwitchTimeouts);
Assert.Contains(expected, repo.ChannelTimeouts);
},
duration: TimeSpan.FromSeconds(3),
interval: TimeSpan.FromMilliseconds(50));
}
// ---------------------------------------------------------------------
// 11. SwitchThrows_PublishesPurgeFailedEvent_AndIncrementsHealthCounter (Task 5)
// ---------------------------------------------------------------------
[Fact]
public void SwitchThrows_PublishesPurgeFailedEvent_AndIncrementsHealthCounter()
{
// A silently failing retention job is a health-surface condition, not just a log line:
// the per-boundary catch must publish AuditLogPurgeFailedEvent AND bump the
// AuditCentralHealthSnapshot.PurgeFailures counter (arch-review 04, S2).
var boundary = new DateTime(2025, 9, 1, 0, 0, 0, DateTimeKind.Utc);
var repo = new RecordingRepo
{
Boundaries = new List<DateTime> { boundary },
ThrowOnBoundary = boundary,
BoundaryException = new InvalidOperationException("simulated switch timeout"),
};
var snapshot = new AuditCentralHealthSnapshot();
var services = new ServiceCollection();
services.AddScoped<IAuditLogRepository>(_ => repo);
services.AddSingleton(snapshot);
services.AddSingleton<IAuditPurgeFailureCounter>(snapshot);
var sp = services.BuildServiceProvider();
var probe = CreateTestProbe();
Sys.EventStream.Subscribe(probe.Ref, typeof(AuditLogPurgeFailedEvent));
Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
sp,
Options.Create(FastTickOptions()),
Options.Create(new AuditLogOptions()),
NullLogger<AuditLogPurgeActor>.Instance)));
var evt = probe.ExpectMsg<AuditLogPurgeFailedEvent>(TimeSpan.FromSeconds(5));
Assert.Equal(boundary, evt.MonthBoundary);
Assert.Contains("simulated switch timeout", evt.Error);
Assert.True(evt.ElapsedMilliseconds >= 0);
AwaitAssert(
() => Assert.True(snapshot.PurgeFailures >= 1,
$"expected PurgeFailures >= 1, got {snapshot.PurgeFailures}"),
duration: TimeSpan.FromSeconds(3),
interval: TimeSpan.FromMilliseconds(50));
}
}
@@ -0,0 +1,31 @@
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Central;
/// <summary>
/// Task 4 tests for <see cref="AuditLogPurgeOptions.MaintenanceCommandTimeoutMinutes"/> /
/// <see cref="AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout"/>: the maintenance timeout
/// defaults to 30 minutes and is clamped to a 1-minute floor so a misconfigured 0/negative value
/// cannot yield a zero/negative ADO.NET command timeout (arch-review 04, S2).
/// </summary>
public class AuditLogPurgeOptionsTests
{
[Fact]
public void MaintenanceCommandTimeout_Defaults_To_30_Minutes()
{
var options = new AuditLogPurgeOptions();
Assert.Equal(30, options.MaintenanceCommandTimeoutMinutes);
Assert.Equal(TimeSpan.FromMinutes(30), options.ResolvedMaintenanceCommandTimeout);
}
[Theory]
[InlineData(0, 1)] // clamp floor — a zero config value cannot become a zero timeout
[InlineData(-5, 1)] // clamp floor — negatives are rejected by ADO.NET
[InlineData(1, 1)] // boundary passthrough
[InlineData(45, 45)] // ordinary passthrough
public void ResolvedMaintenanceCommandTimeout_Clamps_To_MinimumOneMinute(int configured, int expectedMinutes)
{
var options = new AuditLogPurgeOptions { MaintenanceCommandTimeoutMinutes = configured };
Assert.Equal(TimeSpan.FromMinutes(expectedMinutes), options.ResolvedMaintenanceCommandTimeout);
}
}
@@ -42,10 +42,10 @@ public class CentralAuditWriteFailuresTests : TestKit
public Task<IReadOnlyList<AuditEvent>> QueryAsync(
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default) =>
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>());
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default) =>
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
Task.FromResult(0L);
public Task<long> PurgeChannelOlderThanAsync(
string channel, DateTime threshold, int batchSize, CancellationToken ct = default) =>
string channel, DateTime threshold, int batchSize, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
Task.FromResult(0L);
public Task<long> BackfillSourceNodeAsync(
string sentinel, DateTime before, int batchSize, CancellationToken ct = default) =>
@@ -86,11 +86,11 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default) =>
Task.FromResult<IReadOnlyList<AuditEvent>>(Inserted);
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default) =>
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
Task.FromResult(0L);
public Task<long> PurgeChannelOlderThanAsync(
string channel, DateTime threshold, int batchSize, CancellationToken ct = default) =>
string channel, DateTime threshold, int batchSize, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
Task.FromResult(0L);
public Task<long> BackfillSourceNodeAsync(
@@ -474,4 +474,54 @@ WHERE name = 'UX_AuditLog_EventId'
Assert.Contains(rows, r => r.EventId == dbOldId);
Assert.Contains(rows, r => r.EventId == dbRecentId);
}
// ---------------------------------------------------------------------
// 5. SwitchOut_WithExplicitCommandTimeout_Succeeds (Task 4)
// ---------------------------------------------------------------------
/// <summary>
/// Task 4 (arch-review 04, S2): proves the explicit-<see cref="TimeSpan"/> maintenance-timeout
/// overload of <see cref="IAuditLogRepository.SwitchOutPartitionAsync"/> runs the real
/// drop-and-rebuild dance to completion against SQL Server — the old row is purged, the kept row
/// survives, and the returned sampled row-count reflects the switched partition. Exercising the
/// timeout path end-to-end guards against a regression that only sets the timeout on the sample
/// command and forgets the DDL batch (or vice versa).
/// </summary>
[SkippableFact]
public async Task SwitchOut_WithExplicitCommandTimeout_Succeeds()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
var siteId = "switch-timeout-" + Guid.NewGuid().ToString("N").Substring(0, 8);
var oldEventId = Guid.NewGuid();
var keptEventId = Guid.NewGuid();
var (oldOccurred, keptOccurred, _, _) = SeedOccurredAt();
await using (var seedConn = _fixture.OpenConnection())
{
await DirectInsertAsync(seedConn, oldEventId, oldOccurred, siteId);
await DirectInsertAsync(seedConn, keptEventId, keptOccurred, siteId);
}
var janBoundary = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
await using (var ctx = CreateContext())
{
var repo = new AuditLogRepository(ctx);
var rowsDeleted = await repo.SwitchOutPartitionAsync(janBoundary, TimeSpan.FromMinutes(30));
Assert.True(rowsDeleted >= 1,
$"Expected the Jan-2026 partition switch-out to report >= 1 sampled row; got {rowsDeleted}.");
}
await using var verify = CreateContext();
var rows = await verify.Set<AuditLogRow>()
.Where(e => e.SourceSiteId == siteId)
.ToListAsync();
Assert.DoesNotContain(rows, r => r.EventId == oldEventId);
Assert.Contains(rows, r => r.EventId == keptEventId);
// The dance must leave the idempotency-supporting unique index rebuilt.
await using var check = _fixture.OpenConnection();
await AssertUxIndexExistsAsync(check);
}
}
@@ -0,0 +1,33 @@
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
/// <summary>
/// Tests for <see cref="SiteAuditRetentionOptions"/> clamping (PLAN-04 Task 2).
/// The resolved knobs guard the daily site retention job against misconfiguration:
/// retention days are clamped to [1, 90]; the purge interval is clamped away from
/// TimeSpan.Zero (an Akka/Timer spin footgun).
/// </summary>
public class SiteAuditRetentionOptionsTests
{
[Theory]
[InlineData(0, 1)] // clamp floor
[InlineData(7, 7)] // default passthrough
[InlineData(365, 90)] // clamp ceiling (spec: min 1, max 90)
public void ResolvedRetentionDays_Clamps(int configured, int expected)
=> Assert.Equal(expected, new SiteAuditRetentionOptions { RetentionDays = configured }.ResolvedRetentionDays);
[Fact]
public void Defaults_Are_SevenDays_DailyPurge_FiveMinuteInitialDelay()
{
var o = new SiteAuditRetentionOptions();
Assert.Equal(7, o.ResolvedRetentionDays);
Assert.Equal(TimeSpan.FromHours(24), o.ResolvedPurgeInterval);
Assert.Equal(TimeSpan.FromMinutes(5), o.InitialDelay);
}
[Fact]
public void ResolvedPurgeInterval_ClampsZeroToOneMinute()
=> Assert.Equal(TimeSpan.FromMinutes(1),
new SiteAuditRetentionOptions { PurgeInterval = TimeSpan.Zero }.ResolvedPurgeInterval);
}
@@ -0,0 +1,114 @@
using System.Collections.Concurrent;
using System.Diagnostics;
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
using ZB.MOM.WW.ScadaBridge.Commons.Types;
using AuditEvent = ZB.MOM.WW.Audit.AuditEvent;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
/// <summary>
/// Tests for <see cref="SiteAuditRetentionService"/> (PLAN-04 Task 3). The hosted
/// service runs the site SQLite retention purge on a timer: first tick after
/// <see cref="SiteAuditRetentionOptions.InitialDelay"/>, then every
/// <see cref="SiteAuditRetentionOptions.ResolvedPurgeInterval"/>, passing a cutoff of
/// <c>UtcNow - ResolvedRetentionDays</c>. Per-tick failures are swallowed so a
/// transient SQLite error never tears the service down.
/// </summary>
public class SiteAuditRetentionServiceTests
{
[Fact]
public async Task Tick_Purges_With_RetentionCutoff()
{
var queue = new RecordingSiteAuditQueue();
var options = Options.Create(new SiteAuditRetentionOptions
{
RetentionDays = 7,
// Override bypasses the 1-min production clamp so the loop ticks in ms.
PurgeIntervalOverride = TimeSpan.FromMilliseconds(50),
InitialDelay = TimeSpan.Zero,
});
using var svc = new SiteAuditRetentionService(queue, options, NullLogger<SiteAuditRetentionService>.Instance);
await svc.StartAsync(CancellationToken.None);
await WaitUntilAsync(() => queue.PurgeCalls.Count >= 1, TimeSpan.FromSeconds(5));
await svc.StopAsync(CancellationToken.None);
var cutoff = queue.PurgeCalls.First();
Assert.InRange(cutoff,
DateTime.UtcNow.AddDays(-7).AddMinutes(-1),
DateTime.UtcNow.AddDays(-7).AddMinutes(1));
}
[Fact]
public async Task Tick_SwallowsExceptions_AndKeepsTicking()
{
var queue = new RecordingSiteAuditQueue { ThrowOnFirstCall = true };
var options = Options.Create(new SiteAuditRetentionOptions
{
RetentionDays = 7,
// Override bypasses the 1-min production clamp so the loop ticks in ms.
PurgeIntervalOverride = TimeSpan.FromMilliseconds(50),
InitialDelay = TimeSpan.Zero,
});
using var svc = new SiteAuditRetentionService(queue, options, NullLogger<SiteAuditRetentionService>.Instance);
await svc.StartAsync(CancellationToken.None);
// The first tick throws; a second tick must still arrive (service survives).
await WaitUntilAsync(() => queue.PurgeCalls.Count >= 2, TimeSpan.FromSeconds(5));
await svc.StopAsync(CancellationToken.None);
Assert.True(queue.PurgeCalls.Count >= 2);
}
private static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
{
var sw = Stopwatch.StartNew();
while (!condition())
{
if (sw.Elapsed > timeout)
{
throw new TimeoutException("Condition was not met within the timeout.");
}
await Task.Delay(10);
}
}
/// <summary>Stub queue that records purge cutoffs and can throw on the first tick.</summary>
private sealed class RecordingSiteAuditQueue : ISiteAuditQueue
{
private readonly ConcurrentQueue<DateTime> _purgeCalls = new();
private int _calls;
public IReadOnlyCollection<DateTime> PurgeCalls => _purgeCalls;
public bool ThrowOnFirstCall { get; init; }
public Task<int> PurgeExpiredAsync(DateTime olderThanUtc, CancellationToken ct = default)
{
var n = Interlocked.Increment(ref _calls);
_purgeCalls.Enqueue(olderThanUtc);
if (ThrowOnFirstCall && n == 1)
{
throw new InvalidOperationException("Simulated transient purge failure.");
}
return Task.FromResult(0);
}
public Task<IReadOnlyList<AuditEvent>> ReadPendingAsync(int limit, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task<IReadOnlyList<AuditEvent>> ReadPendingCachedTelemetryAsync(int limit, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task MarkForwardedAsync(IReadOnlyList<Guid> eventIds, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task<IReadOnlyList<AuditEvent>> ReadPendingSinceAsync(DateTime sinceUtc, int batchSize, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task MarkReconciledAsync(IReadOnlyList<Guid> eventIds, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task<SiteAuditBacklogSnapshot> GetBacklogStatsAsync(CancellationToken ct = default)
=> throw new NotSupportedException();
}
}
@@ -0,0 +1,92 @@
using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
using ZB.MOM.WW.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
/// <summary>
/// Tests for <see cref="SqliteAuditWriter.PurgeExpiredAsync"/> (arch-review 04, S1/U2).
/// The site SQLite store is ephemeral (~7-day retention); the retention purge deletes
/// rows that have already left the site (Forwarded/Reconciled) and are older than the
/// cutoff, while honoring the hard <see cref="AuditForwardState"/> invariant: a
/// <see cref="AuditForwardState.Pending"/> row is NEVER purged on age alone — dropping
/// an un-forwarded row would silently lose audit data that never reached central.
/// Fixture mirrors <see cref="SqliteAuditWriterBacklogStatsTests"/> (file-backed so the
/// post-purge incremental_vacuum has a real file to shrink).
/// </summary>
public class SqliteAuditWriterRetentionTests : IDisposable
{
private readonly string _dbPath;
public SqliteAuditWriterRetentionTests()
{
_dbPath = Path.Combine(Path.GetTempPath(),
$"audit-retention-{Guid.NewGuid():N}.db");
}
public void Dispose()
{
if (File.Exists(_dbPath))
{
try { File.Delete(_dbPath); } catch { /* test cleanup best-effort */ }
}
}
private SqliteAuditWriter CreateWriter()
{
var options = new SqliteAuditWriterOptions { DatabasePath = _dbPath };
return new SqliteAuditWriter(
Options.Create(options),
NullLogger<SqliteAuditWriter>.Instance,
new FakeNodeIdentityProvider());
}
private static async Task<Guid> WriteEventAsync(SqliteAuditWriter writer, DateTime occurredAtUtc)
{
var id = Guid.NewGuid();
var evt = ScadaBridgeAuditEventFactory.Create(
eventId: id,
occurredAtUtc: occurredAtUtc,
channel: AuditChannel.ApiOutbound,
kind: AuditKind.ApiCall,
status: AuditStatus.Delivered);
await writer.WriteAsync(evt);
return id;
}
[Fact]
public async Task PurgeExpired_DeletesForwardedAndReconciled_OlderThanCutoff()
{
await using var writer = CreateWriter();
var oldForwarded = await WriteEventAsync(writer, DateTime.UtcNow.AddDays(-10));
var oldReconciled = await WriteEventAsync(writer, DateTime.UtcNow.AddDays(-10));
var oldPending = await WriteEventAsync(writer, DateTime.UtcNow.AddDays(-10));
var fresh = await WriteEventAsync(writer, DateTime.UtcNow);
await writer.MarkForwardedAsync(new[] { oldForwarded, oldReconciled, fresh });
await writer.MarkReconciledAsync(new[] { oldReconciled });
var purged = await writer.PurgeExpiredAsync(DateTime.UtcNow.AddDays(-7));
Assert.Equal(2, purged); // oldForwarded + oldReconciled
// Hard ForwardState invariant: the old *Pending* row survives on age alone.
var pending = await writer.ReadPendingAsync(10);
Assert.Contains(pending, e => e.EventId == oldPending);
// Fresh forwarded row inside the window survives; only oldPending remains Pending.
var stats = await writer.GetBacklogStatsAsync();
Assert.Equal(1, stats.PendingCount);
}
[Fact]
public async Task PurgeExpired_IsIdempotent()
{
await using var writer = CreateWriter();
var id = await WriteEventAsync(writer, DateTime.UtcNow.AddDays(-10));
await writer.MarkForwardedAsync(new[] { id });
Assert.Equal(1, await writer.PurgeExpiredAsync(DateTime.UtcNow.AddDays(-7)));
Assert.Equal(0, await writer.PurgeExpiredAsync(DateTime.UtcNow.AddDays(-7)));
}
}
@@ -0,0 +1,74 @@
using Xunit;
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
/// <summary>
/// Content guard for the <c>FixAuditPurgerRoleGrants</c> migration (arch-review 04, S3).
/// The switch-out dance CREATEs a staging table and the per-channel override runs a bounded
/// row DELETE — but the shipped <c>scadabridge_audit_purger</c> role only carried
/// <c>SELECT</c> + <c>ALTER ON SCHEMA::dbo</c>, which does NOT confer the database-level
/// <c>CREATE TABLE</c> permission nor row-level <c>DELETE</c>. This test asserts the migration
/// source grants both, so a segregated maintenance principal can genuinely execute the purge.
/// A pure source-content scan (no MSSQL required) — the schema-application sanity check lives
/// in the MSSQL migration suite.
/// </summary>
public class FixAuditPurgerRoleGrantsTests
{
private static string ReadMigrationSource()
{
var migrationsDir = Path.Combine(
GetConfigurationDatabaseSourceDirectory(), "Migrations");
// Timestamp-prefixed filename; match the non-Designer partial only.
var file = Directory.GetFiles(migrationsDir, "*_FixAuditPurgerRoleGrants.cs")
.FirstOrDefault(f => !f.EndsWith(".Designer.cs", StringComparison.OrdinalIgnoreCase));
Assert.True(file is not null,
"Expected a *_FixAuditPurgerRoleGrants.cs migration under " + migrationsDir +
" — scaffold it with `dotnet ef migrations add FixAuditPurgerRoleGrants` (build first, NO --no-build).");
return File.ReadAllText(file!);
}
[Fact]
public void Migration_Grants_CreateTable_To_Purger()
{
var src = ReadMigrationSource();
Assert.Contains("GRANT CREATE TABLE TO scadabridge_audit_purger", src);
}
[Fact]
public void Migration_Grants_Delete_On_AuditLog_To_Purger()
{
var src = ReadMigrationSource();
Assert.Contains("GRANT DELETE ON dbo.AuditLog TO scadabridge_audit_purger", src);
}
[Fact]
public void Down_Revokes_Both_Grants()
{
var src = ReadMigrationSource();
// Down must reverse both grants so the migration is cleanly reversible.
Assert.Contains("REVOKE CREATE TABLE", src);
Assert.Contains("REVOKE DELETE ON dbo.AuditLog", src);
}
// Same walk-up anchor pattern the append-only guard uses.
private static string GetConfigurationDatabaseSourceDirectory()
{
var dir = new DirectoryInfo(AppContext.BaseDirectory);
while (dir != null)
{
var candidate = Path.Combine(
dir.FullName, "src",
"ZB.MOM.WW.ScadaBridge.ConfigurationDatabase",
"ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.csproj");
if (File.Exists(candidate))
{
return Path.GetDirectoryName(candidate)!;
}
dir = dir.Parent;
}
throw new InvalidOperationException(
"Could not locate ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.csproj by walking up from the test output directory.");
}
}
@@ -1008,6 +1008,49 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
}
}
[SkippableFact]
public async Task ExecutionTree_Bounds_EdgeScan_To_Root_Window()
{
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
// arch-review 04 (P2): the down-walk Edges CTE is anchored to a
// [rootFirst - 1h, rootFirst + 7d) window around the root's first event, enabling
// partition elimination instead of a full-table DISTINCT scan. A genuine descendant
// stamped BEYOND that 7-day span is excluded by design; a normal in-window child is
// still returned.
var siteId = NewSiteId();
await using var context = CreateContext();
var repo = new AuditLogRepository(context);
var rootExec = Guid.NewGuid();
var childExec = Guid.NewGuid();
var farDescendantExec = Guid.NewGuid();
var unrelatedExec = Guid.NewGuid();
var t0 = new DateTime(2026, 10, 5, 9, 0, 0, DateTimeKind.Utc);
// Root + an in-window child (T0 + 5 min) — both must be returned.
await repo.InsertIfNotExistsAsync(NewEvent(siteId, occurredAtUtc: t0, executionId: rootExec));
await repo.InsertIfNotExistsAsync(NewEvent(siteId, occurredAtUtc: t0.AddMinutes(5), executionId: childExec, parentExecutionId: rootExec));
// A genuine descendant of the child, but stamped 30 days out — beyond the 7-day
// traversal bound, so it is documented-excluded from the tree.
await repo.InsertIfNotExistsAsync(NewEvent(siteId, occurredAtUtc: t0.AddDays(30), executionId: farDescendantExec, parentExecutionId: childExec));
// An unrelated execution far in time sharing no ancestry — never part of the chain
// (asserts the window bound does not accidentally pull in strangers).
await repo.InsertIfNotExistsAsync(NewEvent(siteId, occurredAtUtc: t0.AddDays(30), executionId: unrelatedExec));
var tree = await repo.GetExecutionTreeAsync(rootExec);
var ids = tree.Select(n => n.ExecutionId).ToHashSet();
Assert.Contains(rootExec, ids);
Assert.Contains(childExec, ids);
Assert.DoesNotContain(farDescendantExec, ids);
Assert.DoesNotContain(unrelatedExec, ids);
Assert.Equal(2, tree.Count);
}
[SkippableFact]
public async Task GetExecutionTree_StubParentNode()
{
@@ -1,4 +1,7 @@
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Metadata;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.ExternalSystems;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications;
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
@@ -70,6 +73,27 @@ public class SchemaConfigurationTests : IDisposable
Assert.IsType<EncryptedStringConverter>(converter);
}
// arch-review 04 (P1): the SiteCalls "live queue" KPI predicates all filter on
// TerminalAtUtc IS NULL and run every 60 s (KPI recorder) + every 10 s (Health poll).
// A filtered index over just the non-terminal rows turns each count from a clustered
// scan of the 365-day archive into a small seek. Locked shape:
// key = CreatedAtUtc, filter = [TerminalAtUtc] IS NULL, INCLUDE SourceSite/SourceNode/Status.
[Fact]
public void SiteCalls_Has_Filtered_NonTerminal_Index()
{
// IncludeProperties is a SQL-Server-only annotation stripped from the runtime
// read-optimized model, so read the design-time model where it is preserved.
var model = _context.GetService<IDesignTimeModel>().Model;
var entity = model.FindEntityType(typeof(SiteCall))!;
var index = entity.GetIndexes().Single(i => i.GetDatabaseName() == "IX_SiteCalls_NonTerminal");
Assert.Equal(new[] { nameof(SiteCall.CreatedAtUtc) }, index.Properties.Select(p => p.Name));
Assert.Equal("[TerminalAtUtc] IS NULL", index.GetFilter());
Assert.Equal(
new[] { nameof(SiteCall.SourceSite), nameof(SiteCall.SourceNode), nameof(SiteCall.Status) },
index.GetIncludeProperties());
}
}
public class SplitQueryBehaviourTests : IDisposable
@@ -86,11 +86,11 @@ public class SiteAuditPushFlowTests : TestKit
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default)
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task<long> PurgeChannelOlderThanAsync(
string channel, DateTime threshold, int batchSize, CancellationToken ct = default)
string channel, DateTime threshold, int batchSize, TimeSpan? commandTimeout = null, CancellationToken ct = default)
=> throw new NotSupportedException();
public Task<long> BackfillSourceNodeAsync(