Compare commits
12 Commits
1586e791f7
...
1dd993ec2f
| Author | SHA1 | Date | |
|---|---|---|---|
| 1dd993ec2f | |||
| 5d0fcd4e66 | |||
| f1957606ff | |||
| eb69b93947 | |||
| 2a080a3e12 | |||
| 7555e65746 | |||
| 14c4df54f9 | |||
| b59aa2d717 | |||
| 1b6b0aab2a | |||
| 74aa872c07 | |||
| b5271da5c8 | |||
| 731ee69797 |
@@ -1,15 +1,15 @@
|
||||
{
|
||||
"planPath": "archreview/plans/PLAN-04-data-audit-backbone.md",
|
||||
"tasks": [
|
||||
{ "id": 1, "subject": "Task 1: Site SQLite retention purge — PurgeExpiredAsync on the writer", "status": "pending", "blockedBy": [] },
|
||||
{ "id": 2, "subject": "Task 2: Site retention options", "status": "pending", "blockedBy": [] },
|
||||
{ "id": 3, "subject": "Task 3: Site retention hosted job + registration + design doc", "status": "pending", "blockedBy": [1, 2] },
|
||||
{ "id": 4, "subject": "Task 4: Maintenance command timeout on partition switch + per-channel purge", "status": "pending", "blockedBy": [] },
|
||||
{ "id": 5, "subject": "Task 5: Purge-failure health event (not just a log line)", "status": "pending", "blockedBy": [4] },
|
||||
{ "id": 6, "subject": "Task 6: Fix purger role grants (migration)", "status": "pending", "blockedBy": [] },
|
||||
{ "id": 7, "subject": "Task 7: Honest append-only enforcement story (docs)", "status": "pending", "blockedBy": [6] },
|
||||
{ "id": 8, "subject": "Task 8: SiteCalls filtered non-terminal index (migration)", "status": "pending", "blockedBy": [6] },
|
||||
{ "id": 9, "subject": "Task 9: Bound the execution-tree edge scan with a root time window", "status": "pending", "blockedBy": [] },
|
||||
{ "id": 1, "subject": "Task 1: Site SQLite retention purge — PurgeExpiredAsync on the writer", "status": "completed", "blockedBy": [] },
|
||||
{ "id": 2, "subject": "Task 2: Site retention options", "status": "completed", "blockedBy": [] },
|
||||
{ "id": 3, "subject": "Task 3: Site retention hosted job + registration + design doc", "status": "completed", "blockedBy": [1, 2] },
|
||||
{ "id": 4, "subject": "Task 4: Maintenance command timeout on partition switch + per-channel purge", "status": "completed", "blockedBy": [] },
|
||||
{ "id": 5, "subject": "Task 5: Purge-failure health event (not just a log line)", "status": "completed", "blockedBy": [4] },
|
||||
{ "id": 6, "subject": "Task 6: Fix purger role grants (migration)", "status": "completed", "blockedBy": [] },
|
||||
{ "id": 7, "subject": "Task 7: Honest append-only enforcement story (docs)", "status": "completed", "blockedBy": [6] },
|
||||
{ "id": 8, "subject": "Task 8: SiteCalls filtered non-terminal index (migration)", "status": "completed", "blockedBy": [6] },
|
||||
{ "id": 9, "subject": "Task 9: Bound the execution-tree edge scan with a root time window", "status": "completed", "blockedBy": [] },
|
||||
{ "id": 10, "subject": "Task 10: Fix the hardcoded-zero backlogTotal KPI history", "status": "pending", "blockedBy": [] },
|
||||
{ "id": 11, "subject": "Task 11: SiteCalls same-rank freshness upsert (unfreeze RetryCount/LastError)", "status": "pending", "blockedBy": [] },
|
||||
{ "id": 12, "subject": "Task 12: EF Core connection resiliency (EnableRetryOnFailure)", "status": "pending", "blockedBy": [] },
|
||||
@@ -28,5 +28,5 @@
|
||||
{ "id": 25, "subject": "Task 25: Bounded parallel notification delivery", "status": "pending", "blockedBy": [13] },
|
||||
{ "id": 26, "subject": "Final verification: full solution build + test sweep (infra up for MSSQL suites)", "status": "pending", "blockedBy": [3, 8, 9, 10, 12, 14, 18, 19, 20, 21, 22, 23, 24, 25] }
|
||||
],
|
||||
"lastUpdated": "2026-07-08"
|
||||
"lastUpdated": "2026-07-09 (Batch 3: Tasks 7-9 completed)"
|
||||
}
|
||||
|
||||
@@ -42,14 +42,14 @@ A `SecretAwareModelCacheKeyFactory` folds `HasSecretEncryptionProvider` into the
|
||||
|
||||
### Append-only AuditLog and DB-role enforcement
|
||||
|
||||
The central `dbo.AuditLog` table has two dedicated SQL Server roles:
|
||||
The central `dbo.AuditLog` table has two dedicated SQL Server roles — **optional DBA hardening** on top of the default application-level enforcement (CI grep guard + code review; the default deployment runs one connection principal for both paths):
|
||||
|
||||
| Role | Grants |
|
||||
|------|--------|
|
||||
| `scadabridge_audit_writer` | `INSERT`, `SELECT` on `AuditLog` only — no `UPDATE`, no `DELETE` |
|
||||
| `scadabridge_audit_purger` | `ALTER ON SCHEMA::dbo` (required for `SPLIT RANGE` and partition switch-out) |
|
||||
| `scadabridge_audit_writer` | `INSERT`, `SELECT` on `AuditLog` only; `DENY UPDATE` + `DENY DELETE` |
|
||||
| `scadabridge_audit_purger` | `SELECT` + `ALTER ON SCHEMA::dbo` (partition switch-out), **plus** `CREATE TABLE` (switch-out CREATEs a staging table) and scoped `DELETE ON dbo.AuditLog` (per-channel retention override) — the latter two added by the `FixAuditPurgerRoleGrants` migration |
|
||||
|
||||
Row-level `DELETE` on `AuditLog` is not granted even to the purge role; retention is always a partition switch, never a row delete.
|
||||
The primary retention purge is a monthly partition switch. The one sanctioned row-level `DELETE` against `AuditLog` is the per-channel retention override on the maintenance path (guarded by the `// AUDIT-PURGE-ALLOWED` marker); every other `UPDATE`/`DELETE` trips the append-only CI guard.
|
||||
|
||||
## Architecture
|
||||
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
BEGIN TRANSACTION;
|
||||
IF NOT EXISTS (
|
||||
SELECT * FROM [__EFMigrationsHistory]
|
||||
WHERE [MigrationId] = N'20260709110614_AddSiteCallsNonTerminalIndex'
|
||||
)
|
||||
BEGIN
|
||||
EXEC(N'CREATE INDEX [IX_SiteCalls_NonTerminal] ON [SiteCalls] ([CreatedAtUtc]) INCLUDE ([SourceSite], [SourceNode], [Status]) WHERE [TerminalAtUtc] IS NULL');
|
||||
END;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT * FROM [__EFMigrationsHistory]
|
||||
WHERE [MigrationId] = N'20260709110614_AddSiteCallsNonTerminalIndex'
|
||||
)
|
||||
BEGIN
|
||||
INSERT INTO [__EFMigrationsHistory] ([MigrationId], [ProductVersion])
|
||||
VALUES (N'20260709110614_AddSiteCallsNonTerminalIndex', N'10.0.7');
|
||||
END;
|
||||
|
||||
COMMIT;
|
||||
GO
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
BEGIN TRANSACTION;
|
||||
IF NOT EXISTS (
|
||||
SELECT * FROM [__EFMigrationsHistory]
|
||||
WHERE [MigrationId] = N'20260709105112_FixAuditPurgerRoleGrants'
|
||||
)
|
||||
BEGIN
|
||||
|
||||
IF DATABASE_PRINCIPAL_ID('scadabridge_audit_purger') IS NULL
|
||||
EXEC sp_executesql N'CREATE ROLE scadabridge_audit_purger';
|
||||
-- The switch-out dance CREATEs a staging table; ALTER ON SCHEMA::dbo alone does not
|
||||
-- confer the database-level CREATE TABLE permission (review finding S3).
|
||||
GRANT CREATE TABLE TO scadabridge_audit_purger;
|
||||
-- The per-channel retention override (PurgeChannelOlderThanAsync) is a bounded row
|
||||
-- DELETE on the maintenance path; a segregated purger principal needs this grant.
|
||||
GRANT DELETE ON dbo.AuditLog TO scadabridge_audit_purger;
|
||||
END;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT * FROM [__EFMigrationsHistory]
|
||||
WHERE [MigrationId] = N'20260709105112_FixAuditPurgerRoleGrants'
|
||||
)
|
||||
BEGIN
|
||||
INSERT INTO [__EFMigrationsHistory] ([MigrationId], [ProductVersion])
|
||||
VALUES (N'20260709105112_FixAuditPurgerRoleGrants', N'10.0.7');
|
||||
END;
|
||||
|
||||
COMMIT;
|
||||
GO
|
||||
|
||||
@@ -209,6 +209,17 @@ tag subscription) are also wired: trigger-driven runs carry `ParentExecutionId =
|
||||
NULL` (top-level roots), and any nested `CallScript`/`CallShared` they perform
|
||||
chains as above. The schema is unchanged — no further tag-cascade work is deferred.
|
||||
|
||||
**Execution-tree traversal bound.** `GetExecutionTreeAsync` first walks up
|
||||
`ParentExecutionId` to the chain root, then walks down via a recursive CTE. The
|
||||
down-walk's edge scan is bounded to a window anchored at the root's **first event**:
|
||||
`[rootFirst − 1 h, rootFirst + 7 d)` (configurable constants). Execution trees span
|
||||
minutes, not years — an inbound request and everything it spawns fire inside one
|
||||
operational burst — so the window lets SQL Server eliminate every `AuditLog`
|
||||
partition outside it, turning a full-table `DISTINCT` scan into a seek on
|
||||
`IX_AuditLog_Execution` (arch-review 04, P2). A genuine descendant stamped **beyond
|
||||
the 7-day span is excluded by design**; a row-less stub root (its first event purged)
|
||||
has no anchor and falls back to the unbounded scan (correctness over speed).
|
||||
|
||||
## The Site-Local `AuditLog` (SQLite)
|
||||
|
||||
A SQLite database file on each site node, alongside the Store-and-Forward
|
||||
@@ -390,9 +401,25 @@ MS SQL for direct-write events). Unredacted secrets never persist.
|
||||
- **Purge actor:** `AuditLogPurgeActor` singleton on the active central node
|
||||
runs daily, switches out any partition whose latest `OccurredAtUtc` is older
|
||||
than the retention window, then applies any per-channel overrides (see below),
|
||||
and emits an `AuditLog:Purged` event (partition range, rowcount, duration) per
|
||||
and emits an `AuditLogPurgedEvent` (partition range, rowcount, duration) per
|
||||
switched partition. A partition-maintenance step rolls forward each month,
|
||||
creating the next month's partition ahead of time.
|
||||
- **Purge failure is a health metric, not just a log line:** every purge step that
|
||||
throws — a partition switch-out, a per-channel override DELETE, or the pre-purge
|
||||
boundary enumeration — publishes an `AuditLogPurgeFailedEvent` (month boundary,
|
||||
error message, elapsed ms; `DateTime.MinValue` boundary for the enumeration/channel
|
||||
phases that have no single month in hand) and increments the `PurgeFailures` central
|
||||
health counter on `AuditCentralHealthSnapshot` (alongside `CentralAuditWriteFailures`).
|
||||
This is the symmetric failure twin of the success-path `AuditLogPurgedEvent`: a
|
||||
silently failing retention job would otherwise be invisible until the table grew
|
||||
unbounded. Per-boundary/per-channel error isolation is unchanged — a single failure
|
||||
still never abandons the rest of the tick.
|
||||
- **Maintenance command timeout:** the switch-out drop-and-rebuild dance and each
|
||||
per-channel `DELETE TOP` batch run with an explicit command timeout
|
||||
(`AuditLog:Purge:MaintenanceCommandTimeoutMinutes`, default 30, floor 1 min) rather
|
||||
than the ~30 s ADO.NET default, which could abort the metadata-only SWITCH mid-dance
|
||||
on a large or contended partition and leave the live table without
|
||||
`UX_AuditLog_EventId` until a later tick's CATCH branch rebuilt it.
|
||||
- **Per-channel retention overrides (M5.5 T3):** `AuditLog:PerChannelRetentionDays`
|
||||
is a dictionary keyed by canonical channel name (`ApiOutbound`, `DbOutbound`,
|
||||
`Notification`, `ApiInbound`) whose value is a retention window in days that
|
||||
@@ -409,19 +436,49 @@ MS SQL for direct-write events). Unredacted secrets never persist.
|
||||
partition switch-out loop. Values are validated to be in
|
||||
`[30, RetentionDays]`; keys that are not a recognized `AuditChannel` enum name
|
||||
are rejected at startup.
|
||||
- **Sites:** daily site job; default 7-day retention (configurable, min 1,
|
||||
max 90). Respects the hard `ForwardState` invariant — `Pending` rows are
|
||||
never purged on age alone.
|
||||
- **Sites:** `SiteAuditRetentionService` (site-only `IHostedService`) runs the
|
||||
site SQLite retention purge on a timer — first tick after
|
||||
`AuditLog:SiteRetention:InitialDelay` (default 5 min, short so a daily-recycled
|
||||
node still purges soon after start), then every
|
||||
`AuditLog:SiteRetention:PurgeInterval` (default 24 h, clamped to ≥ 1 min).
|
||||
Each tick calls `ISiteAuditQueue.PurgeExpiredAsync(UtcNow − RetentionDays)`,
|
||||
which deletes eligible rows in one transaction (sidecar first, then the
|
||||
canonical row) and then runs `PRAGMA incremental_vacuum` to return the freed
|
||||
pages to the OS. `AuditLog:SiteRetention:RetentionDays` defaults to 7 and is
|
||||
clamped to `[1, 90]`. Respects the hard `ForwardState` invariant — a `Pending`
|
||||
row is never purged on age alone; only `Forwarded`/`Reconciled` rows older than
|
||||
the cutoff are removed. Per-tick failures are swallowed and logged so a transient
|
||||
SQLite fault never tears the service down.
|
||||
|
||||
## Security & Tamper-Evidence
|
||||
|
||||
- **Append-only enforcement.** The application accesses `AuditLog` via a
|
||||
dedicated DB role `scadabridge_audit_writer` granted `INSERT` + `SELECT` only —
|
||||
no `UPDATE`, no `DELETE`. Purge runs under a separate role
|
||||
`scadabridge_audit_purger` whose permissions are limited to the partition-switch
|
||||
operation; row-level `DELETE` is not granted even to purge.
|
||||
- **CI grep guard.** The build greps the data layer for any
|
||||
`UPDATE … AuditLog` or `DELETE … AuditLog` text and fails on a hit.
|
||||
- **Append-only enforcement — what actually holds in the default deployment.**
|
||||
ScadaBridge runs with **one** connection principal for both the writer and the
|
||||
maintenance/purge paths (the runtime does not open a second, lower-privilege
|
||||
connection for purge). So in the default deployment the append-only invariant is
|
||||
enforced by two application-level controls, not by database permissions:
|
||||
1. **CI grep guard** — `AuditLogAppendOnlyGuardTests` scans the ConfigurationDatabase
|
||||
source for any `UPDATE`/`DELETE` DML targeting `dbo.AuditLog` and fails the build
|
||||
on a hit. Exactly two maintenance-path mutations are allow-listed by an explicit
|
||||
`// AUDIT-PURGE-ALLOWED` marker (the per-channel retention `DELETE TOP` and the
|
||||
one-time `SourceNode` sentinel backfill `UPDATE`); every other UPDATE/DELETE trips
|
||||
the guard.
|
||||
2. **Code review** — the marker is deliberately specific so an unrelated mutation
|
||||
cannot inherit the exemption without a reviewer noticing.
|
||||
- **Optional DBA hardening (two DB roles).** For a deployment that wants
|
||||
database-level enforcement on top of the application controls, migrations provision
|
||||
two roles: `scadabridge_audit_writer` (`INSERT` + `SELECT` only; `DENY UPDATE` +
|
||||
`DENY DELETE` so a later `db_datawriter` membership cannot silently re-enable
|
||||
mutation) and `scadabridge_audit_purger` (the maintenance principal). To use them,
|
||||
a DBA provisions **two logins** — the runtime connection mapped to the writer role,
|
||||
and a separate maintenance job/connection mapped to the purger role. The purger role
|
||||
now carries the permissions its purge path genuinely needs: `SELECT` + `ALTER ON
|
||||
SCHEMA::dbo` **plus** `CREATE TABLE` (the switch-out dance CREATEs a staging table)
|
||||
and a scoped `DELETE ON dbo.AuditLog` (the per-channel retention override is a bounded
|
||||
row DELETE) — granted by the `FixAuditPurgerRoleGrants` migration (arch-review 04, S3).
|
||||
Without those two grants the purger role could not actually execute the switch-out or
|
||||
the per-channel purge; the earlier claim that "row-level `DELETE` is not granted even
|
||||
to purge" was stale once `PerChannelRetentionDays` shipped.
|
||||
- **Authorization.** Reading the Audit Log requires the existing **Audit** role
|
||||
extended with a new **OperationalAudit** permission. Per-site row scoping
|
||||
reuses the existing site-permission model; bulk export requires an additional
|
||||
|
||||
@@ -100,7 +100,7 @@ The configuration database stores all central system data, organized by domain a
|
||||
- `IX_AuditLog_ParentExecution (ParentExecutionId)` — unfiltered
|
||||
- `IX_AuditLog_Node_Occurred (SourceNode, OccurredAtUtc)` (per-node queries)
|
||||
|
||||
**Monthly partitioning** on `OccurredAtUtc` via partition function `pf_AuditLog_Month` and partition scheme `ps_AuditLog_Month`; retention purge is a partition switch, never a row-level delete. The table is populated only by Audit Log writers (site telemetry, central direct-write, reconciliation pulls); central ingest is **insert-if-not-exists** keyed on `EventId`. See Component-AuditLog.md for the full lifecycle, payload-capture policy, and ingestion paths.
|
||||
**Monthly partitioning** on `OccurredAtUtc` via partition function `pf_AuditLog_Month` and partition scheme `ps_AuditLog_Month`; the primary retention purge is a partition switch (per-channel retention overrides add one sanctioned bounded row DELETE on the maintenance path — see AuditLog Table Purge). The table is populated only by Audit Log writers (site telemetry, central direct-write, reconciliation pulls); central ingest is **insert-if-not-exists** keyed on `EventId`. See Component-AuditLog.md for the full lifecycle, payload-capture policy, and ingestion paths.
|
||||
|
||||
### Inbound API
|
||||
- **API Keys**: The `ApiKeys` SQL Server table was **retired** by migration `RetireInboundApiKeyStore`. Inbound API keys now live in the shared `ZB.MOM.WW.Auth.ApiKeys` SQLite store, not the configuration MS SQL database. The `IInboundApiRepository` no longer exposes any `ApiKey` operations; key management goes through `IInboundApiKeyAdmin` (Commons, `Interfaces/Security/`). The `ApprovedApiKeyIds` column on `ApiMethods` was removed at the same time — key-to-method scoping is now managed by the auth library.
|
||||
@@ -327,12 +327,12 @@ Results are returned in reverse chronological order (most recent first) with pag
|
||||
|
||||
## Database Roles
|
||||
|
||||
The configuration database defines dedicated SQL Server roles for the append-only `AuditLog` table so that the application can never accidentally mutate audit history:
|
||||
The configuration database defines dedicated SQL Server roles for the append-only `AuditLog` table. These roles are **optional DBA hardening**: the default deployment runs with a single connection principal for both writer and maintenance paths, so append-only is enforced application-side (CI grep guard + code review — see Component-AuditLog.md). A deployment that wants database-level enforcement provisions two logins mapped to these roles:
|
||||
|
||||
- **`scadabridge_audit_writer`** — the role used by application code that ingests audit events (the `AuditLogIngestActor`, central direct-write paths, and the Notification Outbox dispatcher). Granted `INSERT` and `SELECT` on `AuditLog` only — explicitly **no** `UPDATE` and **no** `DELETE`. Audit ingest is `INSERT … WHERE NOT EXISTS` keyed on `EventId`, which this grant set fully supports.
|
||||
- **`scadabridge_audit_purger`** — the role used by the `AuditLogPurgeActor`. Granted only the permissions required to execute the monthly partition-switch operation (switch out a partition to a staging table and drop the staging table). Row-level `DELETE` on `AuditLog` is **not** granted even to the purge role; retention is a partition switch, never a row-by-row delete.
|
||||
- **`scadabridge_audit_writer`** — the role for application code that ingests audit events (the `AuditLogIngestActor`, central direct-write paths, and the Notification Outbox dispatcher). Granted `INSERT` and `SELECT` on `AuditLog` only, with `DENY UPDATE` + `DENY DELETE` so a later `db_datawriter` membership cannot silently re-enable mutation (`DENY` > `GRANT`). Audit ingest is `INSERT … WHERE NOT EXISTS` keyed on `EventId`, which this grant set fully supports.
|
||||
- **`scadabridge_audit_purger`** — the maintenance role for the `AuditLogPurgeActor`. Granted `SELECT` + `ALTER ON SCHEMA::dbo` for the monthly partition-switch dance, **plus** `CREATE TABLE` (the switch-out CREATEs a staging table — `ALTER ON SCHEMA::dbo` alone does not confer the database-level `CREATE TABLE` permission) and a scoped `DELETE ON dbo.AuditLog` (the per-channel retention override is a bounded row DELETE on the maintenance path). The `CREATE TABLE` + `DELETE` grants were added by the `FixAuditPurgerRoleGrants` migration (arch-review 04, S3); the earlier claim that row-level `DELETE` is "not granted even to the purge role" was stale once per-channel retention overrides shipped.
|
||||
|
||||
A CI grep guard fails the build on any occurrence of `UPDATE … AuditLog` or `DELETE … AuditLog` in the data-access layer source, backstopping the DB-grant enforcement at code-review time. See Component-AuditLog.md (Security & Tamper-Evidence) for the full enforcement contract.
|
||||
A CI grep guard (`AuditLogAppendOnlyGuardTests`) fails the build on any `UPDATE`/`DELETE` DML targeting `dbo.AuditLog` in the data-access layer source, save the two `// AUDIT-PURGE-ALLOWED`-marked maintenance-path mutations. In the default single-principal deployment this guard (with code review) IS the enforcement; the DB roles above harden it further when a DBA opts in. See Component-AuditLog.md (Security & Tamper-Evidence) for the full contract.
|
||||
|
||||
---
|
||||
|
||||
@@ -396,7 +396,7 @@ The `SiteCalls` table grows one row per cached site call and is never trimmed by
|
||||
|
||||
### AuditLog Table Purge
|
||||
|
||||
The `AuditLog` table is append-only and grows by every script-trust-boundary event across all channels. Unlike `Notifications` and `SiteCalls`, purge is **never a row-level `DELETE`** — it is a **monthly partition switch** against the `ps_AuditLog_Month` scheme. A daily job switches out any partition whose latest `OccurredAtUtc` is older than the global retention window (default 365 days, configurable via the `AuditLog:RetentionDays` Audit Log option — single global value in v1, no per-channel overrides) and drops the resulting staging table. The job is owned and scheduled by the Audit Log component (`AuditLogPurgeActor` — see Component-AuditLog.md), which is also the consumer of the `AuditLog:RetentionDays` option. The Configuration Database component contributes only the table, the partition function/scheme, the indexes, and the DB roles that constrain the purge to a partition switch.
|
||||
The `AuditLog` table is append-only and grows by every script-trust-boundary event across all channels. Unlike `Notifications` and `SiteCalls`, the primary purge is a **monthly partition switch** against the `ps_AuditLog_Month` scheme, not a row-level `DELETE`. A daily job switches out any partition whose latest `OccurredAtUtc` is older than the global retention window (default 365 days, configurable via `AuditLog:RetentionDays`) and drops the resulting staging table. On top of the global window, **per-channel retention overrides** (`AuditLog:PerChannelRetentionDays`, a dictionary keyed by canonical channel name with windows strictly shorter than the global one) expire a single channel's rows earlier via a bounded, batched `DELETE TOP` on the maintenance path — the one sanctioned row-level DELETE against `AuditLog` (see Component-AuditLog.md). The job is owned and scheduled by the Audit Log component (`AuditLogPurgeActor` — see Component-AuditLog.md), which is also the consumer of both options. The Configuration Database component contributes only the table, the partition function/scheme, the indexes, and the optional DB roles.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -40,11 +40,13 @@ public sealed class AuditCentralHealthSnapshot
|
||||
: IAuditCentralHealthSnapshot,
|
||||
ICentralAuditWriteFailureCounter,
|
||||
IAuditRedactionFailureCounter,
|
||||
IAuditInboundCeilingHitsCounter
|
||||
IAuditInboundCeilingHitsCounter,
|
||||
IAuditPurgeFailureCounter
|
||||
{
|
||||
private int _centralAuditWriteFailures;
|
||||
private int _auditRedactionFailure;
|
||||
private int _auditInboundCeilingHits;
|
||||
private int _auditPurgeFailures;
|
||||
private readonly ConcurrentDictionary<string, bool> _stalled = new();
|
||||
|
||||
/// <inheritdoc/>
|
||||
@@ -59,6 +61,10 @@ public sealed class AuditCentralHealthSnapshot
|
||||
public int AuditInboundCeilingHits =>
|
||||
Interlocked.CompareExchange(ref _auditInboundCeilingHits, 0, 0);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public int PurgeFailures =>
|
||||
Interlocked.CompareExchange(ref _auditPurgeFailures, 0, 0);
|
||||
|
||||
/// <inheritdoc/>
|
||||
public IReadOnlyDictionary<string, bool> SiteAuditTelemetryStalled =>
|
||||
new Dictionary<string, bool>(_stalled);
|
||||
@@ -88,4 +94,8 @@ public sealed class AuditCentralHealthSnapshot
|
||||
/// <inheritdoc/>
|
||||
void IAuditInboundCeilingHitsCounter.Increment() =>
|
||||
Interlocked.Increment(ref _auditInboundCeilingHits);
|
||||
|
||||
/// <inheritdoc/>
|
||||
void IAuditPurgeFailureCounter.Increment() =>
|
||||
Interlocked.Increment(ref _auditPurgeFailures);
|
||||
}
|
||||
|
||||
@@ -149,7 +149,14 @@ public class AuditLogPurgeActor : ReceiveActor
|
||||
return;
|
||||
}
|
||||
|
||||
// Optional purge-failure counter — resolved from the same per-tick scope the repository
|
||||
// comes from. NoOp default on site/test roots; the central composition root binds the
|
||||
// AuditCentralHealthSnapshot so failures surface on the central health dashboard. Mirrors
|
||||
// AuditLogIngestActor's ICentralAuditWriteFailureCounter resolution shape.
|
||||
var purgeFailureCounter = scope.ServiceProvider.GetService<IAuditPurgeFailureCounter>();
|
||||
|
||||
IReadOnlyList<DateTime> boundaries;
|
||||
var enumSw = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
boundaries = await repository
|
||||
@@ -158,10 +165,14 @@ public class AuditLogPurgeActor : ReceiveActor
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
enumSw.Stop();
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Failed to enumerate eligible AuditLog partition boundaries (threshold {ThresholdUtc:o}); skipping purge tick.",
|
||||
threshold);
|
||||
// Enumeration threw before any boundary was selected — publish with the
|
||||
// DateTime.MinValue sentinel so the failure is still observable on the health surface.
|
||||
PublishPurgeFailure(eventStream, purgeFailureCounter, DateTime.MinValue, ex, enumSw.ElapsedMilliseconds);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -169,7 +180,7 @@ public class AuditLogPurgeActor : ReceiveActor
|
||||
{
|
||||
// No whole-month partitions are eligible, but per-channel overrides may
|
||||
// still expire rows earlier than the global window — run them below.
|
||||
await RunPerChannelOverridesAsync(repository).ConfigureAwait(false);
|
||||
await RunPerChannelOverridesAsync(repository, eventStream, purgeFailureCounter).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -182,7 +193,7 @@ public class AuditLogPurgeActor : ReceiveActor
|
||||
try
|
||||
{
|
||||
var rowsDeleted = await repository
|
||||
.SwitchOutPartitionAsync(boundary)
|
||||
.SwitchOutPartitionAsync(boundary, _purgeOptions.ResolvedMaintenanceCommandTimeout)
|
||||
.ConfigureAwait(false);
|
||||
sw.Stop();
|
||||
|
||||
@@ -203,6 +214,7 @@ public class AuditLogPurgeActor : ReceiveActor
|
||||
"Failed to purge AuditLog partition {MonthBoundary:yyyy-MM-dd}; other partitions continue. Elapsed {DurationMs} ms.",
|
||||
boundary,
|
||||
sw.ElapsedMilliseconds);
|
||||
PublishPurgeFailure(eventStream, purgeFailureCounter, boundary, ex, sw.ElapsedMilliseconds);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,7 +224,24 @@ public class AuditLogPurgeActor : ReceiveActor
|
||||
// switch-out has already dropped whole months older than RetentionDays; these
|
||||
// deletes only ever expire rows EARLIER than that, so they run last and are a
|
||||
// strict tightening.
|
||||
await RunPerChannelOverridesAsync(repository).ConfigureAwait(false);
|
||||
await RunPerChannelOverridesAsync(repository, eventStream, purgeFailureCounter).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Publishes an <see cref="AuditLogPurgeFailedEvent"/> on the EventStream and increments the
|
||||
/// central purge-failure health counter (when wired). Best-effort — the symmetric failure twin
|
||||
/// of the success-path <see cref="AuditLogPurgedEvent"/> publish, so a silently failing
|
||||
/// retention job is a health-surface condition, not just a log line (arch-review 04, S2).
|
||||
/// </summary>
|
||||
private static void PublishPurgeFailure(
|
||||
Akka.Event.EventStream eventStream,
|
||||
IAuditPurgeFailureCounter? counter,
|
||||
DateTime monthBoundary,
|
||||
Exception ex,
|
||||
long elapsedMilliseconds)
|
||||
{
|
||||
eventStream.Publish(new AuditLogPurgeFailedEvent(monthBoundary, ex.Message, elapsedMilliseconds));
|
||||
counter?.Increment();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -224,7 +253,12 @@ public class AuditLogPurgeActor : ReceiveActor
|
||||
/// per-boundary error isolation of the partition switch-out loop.
|
||||
/// </summary>
|
||||
/// <param name="repository">The repository resolved for this tick's DI scope.</param>
|
||||
private async Task RunPerChannelOverridesAsync(IAuditLogRepository repository)
|
||||
/// <param name="eventStream">The actor-system EventStream captured before the first await.</param>
|
||||
/// <param name="purgeFailureCounter">Optional central purge-failure health counter (NoOp on site/test roots).</param>
|
||||
private async Task RunPerChannelOverridesAsync(
|
||||
IAuditLogRepository repository,
|
||||
Akka.Event.EventStream eventStream,
|
||||
IAuditPurgeFailureCounter? purgeFailureCounter)
|
||||
{
|
||||
var overrides = _auditOptions.PerChannelRetentionDays;
|
||||
if (overrides is null || overrides.Count == 0)
|
||||
@@ -250,7 +284,11 @@ public class AuditLogPurgeActor : ReceiveActor
|
||||
try
|
||||
{
|
||||
var rowsDeleted = await repository
|
||||
.PurgeChannelOlderThanAsync(channel, channelThreshold, _purgeOptions.ChannelPurgeBatchSize)
|
||||
.PurgeChannelOlderThanAsync(
|
||||
channel,
|
||||
channelThreshold,
|
||||
_purgeOptions.ChannelPurgeBatchSize,
|
||||
_purgeOptions.ResolvedMaintenanceCommandTimeout)
|
||||
.ConfigureAwait(false);
|
||||
sw.Stop();
|
||||
|
||||
@@ -277,6 +315,10 @@ public class AuditLogPurgeActor : ReceiveActor
|
||||
channel,
|
||||
days,
|
||||
sw.ElapsedMilliseconds);
|
||||
// A per-channel DELETE failure is a purge failure too — surface it alongside the
|
||||
// switch-out failures. MonthBoundary is DateTime.MinValue (this is not a
|
||||
// whole-month switch), matching the enumeration-phase sentinel.
|
||||
PublishPurgeFailure(eventStream, purgeFailureCounter, DateTime.MinValue, ex, sw.ElapsedMilliseconds);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
|
||||
/// <summary>
|
||||
/// Symmetric failure twin of <see cref="AuditLogPurgedEvent"/>. Published on the
|
||||
/// actor-system EventStream by <see cref="AuditLogPurgeActor"/> whenever a purge
|
||||
/// step throws — a partition switch-out, a per-channel override DELETE, or the
|
||||
/// boundary enumeration that precedes them. A silently failing retention job is a
|
||||
/// health-surface condition, not just a log line, so this event lets the central
|
||||
/// health collector + ops dashboards observe purge failures without coupling to
|
||||
/// the actor (arch-review 04, S2).
|
||||
/// </summary>
|
||||
/// <param name="MonthBoundary">
|
||||
/// The pf_AuditLog_Month lower-bound boundary whose switch-out failed. When the
|
||||
/// failure occurs BEFORE any specific boundary is selected — i.e. the pre-purge
|
||||
/// boundary enumeration threw — this is <see cref="System.DateTime.MinValue"/> as a
|
||||
/// sentinel for "enumeration phase, no boundary in hand".
|
||||
/// </param>
|
||||
/// <param name="Error">
|
||||
/// The failure's message (<see cref="System.Exception.Message"/>). The full
|
||||
/// exception is logged at Error on the actor's logger; this carries the short form
|
||||
/// for surfacing on health/ops without leaking a full stack trace onto the stream.
|
||||
/// </param>
|
||||
/// <param name="ElapsedMilliseconds">
|
||||
/// Wall-clock time spent on the failed step before it threw, in milliseconds.
|
||||
/// </param>
|
||||
public sealed record AuditLogPurgeFailedEvent(
|
||||
DateTime MonthBoundary,
|
||||
string Error,
|
||||
long ElapsedMilliseconds);
|
||||
@@ -56,6 +56,30 @@ public sealed class AuditLogPurgeOptions
|
||||
/// </summary>
|
||||
public int ChannelPurgeBatchSize => ChannelPurgeBatchSizeConfigured < 1 ? 1 : ChannelPurgeBatchSizeConfigured;
|
||||
|
||||
/// <summary>
|
||||
/// Per-command timeout (in minutes) for the maintenance SQL the purge tick issues —
|
||||
/// both the partition switch-out drop-and-rebuild dance
|
||||
/// (<see cref="ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories.IAuditLogRepository.SwitchOutPartitionAsync"/>)
|
||||
/// and each per-channel <c>DELETE TOP</c> batch. Default 30 minutes.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The ADO.NET default command timeout is ~30 seconds. On a large or contended partition the
|
||||
/// SWITCH dance (which briefly drops <c>UX_AuditLog_EventId</c>) can exceed that and abort
|
||||
/// mid-flight — leaving the live table without its idempotency-supporting unique index until a
|
||||
/// later tick's CATCH branch rebuilds it (arch-review 04, S2). A generous maintenance timeout
|
||||
/// lets the metadata-only switch complete rather than self-locking. Resolved via
|
||||
/// <see cref="ResolvedMaintenanceCommandTimeout"/>, clamped to a 1-minute floor.
|
||||
/// </remarks>
|
||||
public int MaintenanceCommandTimeoutMinutes { get; set; } = 30;
|
||||
|
||||
/// <summary>
|
||||
/// Resolves the effective maintenance command timeout, clamped to at least 1 minute so a
|
||||
/// misconfigured <c>0</c>/negative value cannot yield a zero/negative timeout (which ADO.NET
|
||||
/// interprets as "no timeout" for <c>0</c> and rejects for negatives).
|
||||
/// </summary>
|
||||
public TimeSpan ResolvedMaintenanceCommandTimeout =>
|
||||
TimeSpan.FromMinutes(MaintenanceCommandTimeoutMinutes < 1 ? 1 : MaintenanceCommandTimeoutMinutes);
|
||||
|
||||
/// <summary>
|
||||
/// Test-only override for finer control over the tick cadence than
|
||||
/// whole-hour resolution allows. When non-null, takes precedence over
|
||||
|
||||
@@ -61,6 +61,16 @@ public interface IAuditCentralHealthSnapshot
|
||||
/// </summary>
|
||||
int AuditInboundCeilingHits { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Count of AuditLog partition-purge failures since process start. Incremented by
|
||||
/// <see cref="AuditLogPurgeActor"/> every time a purge step throws — a partition
|
||||
/// switch-out, a per-channel override DELETE, or the pre-purge boundary enumeration.
|
||||
/// A sustained non-zero count means the retention job is silently not draining the
|
||||
/// table; each failure also publishes an <see cref="AuditLogPurgeFailedEvent"/> on the
|
||||
/// EventStream (arch-review 04, S2).
|
||||
/// </summary>
|
||||
int PurgeFailures { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Per-site latched stalled state: <c>true</c> when the
|
||||
/// <see cref="SiteAuditReconciliationActor"/> has observed two
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
|
||||
/// <summary>
|
||||
/// Counter sink invoked by <see cref="AuditLogPurgeActor"/> every time a purge
|
||||
/// step throws (partition switch-out, per-channel override DELETE, or the pre-purge
|
||||
/// boundary enumeration). Mirrors <see cref="ICentralAuditWriteFailureCounter"/>
|
||||
/// one-for-one — same one-method contract, same NoOp default, same
|
||||
/// never-abort-the-tick invariant (the purge actor swallows the failure per
|
||||
/// boundary and continues).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A NoOp default is the safe fallback for site/test composition roots that do not
|
||||
/// wire the central health surface; <see cref="AuditCentralHealthSnapshot"/> is the
|
||||
/// production binding that routes increments into the aggregated central health
|
||||
/// snapshot's <see cref="IAuditCentralHealthSnapshot.PurgeFailures"/> read surface.
|
||||
/// </remarks>
|
||||
public interface IAuditPurgeFailureCounter
|
||||
{
|
||||
/// <summary>Increment the central AuditLog purge-failure counter by one.</summary>
|
||||
void Increment();
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
|
||||
/// <summary>
|
||||
/// Default <see cref="IAuditPurgeFailureCounter"/> binding used when the central
|
||||
/// health surface (<see cref="AuditCentralHealthSnapshot"/>) has not been wired
|
||||
/// (test composition roots, site-only hosts). Drops every increment on the floor.
|
||||
/// Mirrors <see cref="NoOpCentralAuditWriteFailureCounter"/>.
|
||||
/// </summary>
|
||||
public sealed class NoOpAuditPurgeFailureCounter : IAuditPurgeFailureCounter
|
||||
{
|
||||
/// <inheritdoc/>
|
||||
public void Increment()
|
||||
{
|
||||
// intentional no-op
|
||||
}
|
||||
}
|
||||
@@ -101,6 +101,11 @@ public static class ServiceCollectionExtensions
|
||||
.Bind(config.GetSection(SiteWriterSectionName));
|
||||
services.AddOptions<SiteAuditTelemetryOptions>()
|
||||
.Bind(config.GetSection(SiteTelemetrySectionName));
|
||||
// Retention purge options (the SiteAuditRetentionService hosted job is
|
||||
// registered site-only in AddAuditLogHealthMetricsBridge). Binding here is
|
||||
// inert on central — no hosted service consumes it there.
|
||||
services.AddOptions<SiteAuditRetentionOptions>()
|
||||
.Bind(config.GetSection(SiteAuditRetentionOptions.SectionName));
|
||||
|
||||
// SqliteAuditWriter is a singleton with a single owned SqliteConnection
|
||||
// and a background writer Task; multiple instances would race on the
|
||||
@@ -207,6 +212,12 @@ public static class ServiceCollectionExtensions
|
||||
// redaction-failure counters.
|
||||
services.TryAddSingleton<IAuditInboundCeilingHitsCounter, NoOpAuditInboundCeilingHitsCounter>();
|
||||
|
||||
// AuditLog purge-failure counter — NoOp default for site/test
|
||||
// composition roots. AddAuditLogCentralMaintenance below replaces this binding
|
||||
// with the AuditCentralHealthSnapshot implementation so partition/channel purge
|
||||
// failures surface on the central dashboard alongside the write-failure counter.
|
||||
services.TryAddSingleton<IAuditPurgeFailureCounter, NoOpAuditPurgeFailureCounter>();
|
||||
|
||||
// Central direct-write audit writer used by
|
||||
// NotificationOutboxActor and Inbound API to
|
||||
// emit AuditLog rows that originate ON central, not via site telemetry.
|
||||
@@ -316,6 +327,12 @@ public static class ServiceCollectionExtensions
|
||||
// free of hosted-service registrations that would resolve a missing
|
||||
// ISiteHealthCollector on central.
|
||||
services.AddHostedService<SiteAuditBacklogReporter>();
|
||||
// Site-only retention purge: runs ISiteAuditQueue.PurgeExpiredAsync on a
|
||||
// daily timer so the ephemeral site SQLite store does not grow unbounded.
|
||||
// Registered here (not in AddAuditLog, which also runs on central) so the
|
||||
// hosted service only starts on site composition roots. Guarded by the same
|
||||
// SiteAuditBacklogReporter sentinel above against double-registration.
|
||||
services.AddHostedService<SiteAuditRetentionService>();
|
||||
return services;
|
||||
}
|
||||
|
||||
@@ -408,6 +425,11 @@ public static class ServiceCollectionExtensions
|
||||
// ICentralAuditWriteFailureCounter above.
|
||||
services.Replace(ServiceDescriptor.Singleton<IAuditInboundCeilingHitsCounter>(
|
||||
sp => sp.GetRequiredService<AuditCentralHealthSnapshot>()));
|
||||
// Replace the NoOp IAuditPurgeFailureCounter with the AuditCentralHealthSnapshot so
|
||||
// partition/channel purge failures surface on the central dashboard. Same
|
||||
// singleton-forward pattern as ICentralAuditWriteFailureCounter above (arch-review 04, S2).
|
||||
services.Replace(ServiceDescriptor.Singleton<IAuditPurgeFailureCounter>(
|
||||
sp => sp.GetRequiredService<AuditCentralHealthSnapshot>()));
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for the site-side SQLite retention purge (PLAN-04 Task 2, S1/U2).
|
||||
/// The site audit store is ephemeral (~7-day retention); <see cref="SiteAuditRetentionService"/>
|
||||
/// runs <see cref="ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services.ISiteAuditQueue.PurgeExpiredAsync"/>
|
||||
/// on a timer using these knobs. Mirrors the resolve/clamp shape of
|
||||
/// <c>SiteCallAuditOptions</c> so a misconfigured value can never make the timer spin
|
||||
/// or the retention window collapse.
|
||||
/// </summary>
|
||||
public class SiteAuditRetentionOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Configuration section path bound in the site host. Follows the component's
|
||||
/// existing <c>AuditLog:*</c> convention (sibling of <c>AuditLog:SiteWriter</c>,
|
||||
/// <c>AuditLog:Purge</c>, …), not an application-root-absolute path.
|
||||
/// </summary>
|
||||
public const string SectionName = "AuditLog:SiteRetention";
|
||||
|
||||
/// <summary>
|
||||
/// Retention window in days. On each purge tick, Forwarded/Reconciled rows whose
|
||||
/// <c>OccurredAtUtc</c> is older than <c>UtcNow - ResolvedRetentionDays</c> are
|
||||
/// deleted (Pending rows are never purged on age alone). Default 7. Resolved value
|
||||
/// is clamped to <see cref="MinRetentionDays"/>..<see cref="MaxRetentionDays"/>.
|
||||
/// </summary>
|
||||
public int RetentionDays { get; set; } = 7;
|
||||
|
||||
private const int MinRetentionDays = 1;
|
||||
private const int MaxRetentionDays = 90;
|
||||
|
||||
/// <summary>
|
||||
/// Effective retention window: <see cref="RetentionDays"/> clamped to
|
||||
/// [<see cref="MinRetentionDays"/>, <see cref="MaxRetentionDays"/>] so a
|
||||
/// misconfigured 0 (which would purge everything) or an unbounded value can
|
||||
/// never take effect.
|
||||
/// </summary>
|
||||
public int ResolvedRetentionDays =>
|
||||
RetentionDays < MinRetentionDays ? MinRetentionDays
|
||||
: RetentionDays > MaxRetentionDays ? MaxRetentionDays
|
||||
: RetentionDays;
|
||||
|
||||
/// <summary>
|
||||
/// Period of the purge tick. Default 24 hours. Resolved value is clamped to at
|
||||
/// least <see cref="MinPurgeInterval"/> so a zero/negative config value cannot
|
||||
/// make the timer fire in a tight loop.
|
||||
/// </summary>
|
||||
public TimeSpan PurgeInterval { get; set; } = TimeSpan.FromHours(24);
|
||||
|
||||
/// <summary>
|
||||
/// Test-only override for the purge cadence — bypasses the
|
||||
/// <see cref="MinPurgeInterval"/> clamp so unit tests can drop the period to
|
||||
/// milliseconds. Production config never sets this; leave null. Mirrors
|
||||
/// <c>SiteCallAuditOptions.PurgeIntervalOverride</c>.
|
||||
/// </summary>
|
||||
public TimeSpan? PurgeIntervalOverride { get; set; }
|
||||
|
||||
private static readonly TimeSpan MinPurgeInterval = TimeSpan.FromMinutes(1);
|
||||
|
||||
/// <summary>
|
||||
/// Effective purge period: the test override when set (bypassing the clamp),
|
||||
/// otherwise <see cref="PurgeInterval"/> clamped to at least
|
||||
/// <see cref="MinPurgeInterval"/> (the Timer/Akka zero-interval spin footgun).
|
||||
/// </summary>
|
||||
public TimeSpan ResolvedPurgeInterval =>
|
||||
PurgeIntervalOverride is { } o
|
||||
? o
|
||||
: PurgeInterval < MinPurgeInterval
|
||||
? MinPurgeInterval
|
||||
: PurgeInterval;
|
||||
|
||||
/// <summary>
|
||||
/// Delay before the first purge tick. Default 5 minutes — short so a node that
|
||||
/// restarts daily (or more often) still runs the purge shortly after start
|
||||
/// rather than waiting a full <see cref="PurgeInterval"/> it may never reach.
|
||||
/// </summary>
|
||||
public TimeSpan InitialDelay { get; set; } = TimeSpan.FromMinutes(5);
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
|
||||
/// <summary>
|
||||
/// Site-side hosted service that runs the SQLite retention purge on a timer
|
||||
/// (PLAN-04 Task 3, S1/U2). Each tick calls
|
||||
/// <see cref="ISiteAuditQueue.PurgeExpiredAsync"/> with a cutoff of
|
||||
/// <c>UtcNow - <see cref="SiteAuditRetentionOptions.ResolvedRetentionDays"/></c>,
|
||||
/// deleting Forwarded/Reconciled rows past the retention window and reclaiming the
|
||||
/// freed pages. Without it the ephemeral site store grows unbounded until the next
|
||||
/// deployment recreates it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Cadence.</b> First tick after <see cref="SiteAuditRetentionOptions.InitialDelay"/>
|
||||
/// (default 5 min — short so a node that recycles daily still purges shortly after
|
||||
/// start), then every <see cref="SiteAuditRetentionOptions.ResolvedPurgeInterval"/>
|
||||
/// (default 24 h). Mirrors the loop shape of <see cref="SiteAuditBacklogReporter"/>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Failure containment.</b> Each purge is wrapped in a try/catch so a transient
|
||||
/// SQLite error never tears down the hosted service — the next tick retries.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class SiteAuditRetentionService : IHostedService, IDisposable
|
||||
{
|
||||
private readonly ISiteAuditQueue _queue;
|
||||
private readonly SiteAuditRetentionOptions _options;
|
||||
private readonly ILogger<SiteAuditRetentionService> _logger;
|
||||
private CancellationTokenSource? _cts;
|
||||
private Task? _loop;
|
||||
|
||||
/// <summary>Initializes a new instance of <see cref="SiteAuditRetentionService"/>.</summary>
|
||||
/// <param name="queue">The site audit queue whose retention purge is driven.</param>
|
||||
/// <param name="options">Retention cadence + window options.</param>
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
public SiteAuditRetentionService(
|
||||
ISiteAuditQueue queue,
|
||||
IOptions<SiteAuditRetentionOptions> options,
|
||||
ILogger<SiteAuditRetentionService> logger)
|
||||
{
|
||||
_queue = queue ?? throw new ArgumentNullException(nameof(queue));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
/// <summary>Starts the background purge loop.</summary>
|
||||
/// <param name="ct">Cancellation token signalling host shutdown.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public Task StartAsync(CancellationToken ct)
|
||||
{
|
||||
// Linked CTS so both StopAsync and the host shutdown token abort the loop.
|
||||
_cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
_loop = Task.Run(() => RunLoopAsync(_cts.Token));
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private async Task RunLoopAsync(CancellationToken ct)
|
||||
{
|
||||
// InitialDelay before the first purge — short so a daily-recycled node
|
||||
// still purges soon after start rather than waiting a full interval it may
|
||||
// never reach.
|
||||
try
|
||||
{
|
||||
await Task.Delay(_options.InitialDelay, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await SafePurgeAsync(ct).ConfigureAwait(false);
|
||||
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(_options.ResolvedPurgeInterval, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
await SafePurgeAsync(ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task SafePurgeAsync(CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
var cutoff = DateTime.UtcNow.AddDays(-_options.ResolvedRetentionDays);
|
||||
var purged = await _queue.PurgeExpiredAsync(cutoff, ct).ConfigureAwait(false);
|
||||
if (purged > 0)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Site audit retention purge removed {Count} row(s) older than {Cutoff:o}.",
|
||||
purged, cutoff);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Shutdown — let the outer loop exit cleanly.
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Catch-all is deliberate: a transient SQLite fault must not tear down
|
||||
// the hosted service; the next tick retries.
|
||||
_logger.LogWarning(ex, "SiteAuditRetentionService purge failed; next tick will retry.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Signals the purge loop to stop and waits for it to complete.</summary>
|
||||
/// <param name="ct">Cancellation token (not used; the internal CTS governs shutdown).</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public Task StopAsync(CancellationToken ct)
|
||||
{
|
||||
_cts?.Cancel();
|
||||
return _loop ?? Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>Releases the internal <see cref="CancellationTokenSource"/>.</summary>
|
||||
public void Dispose()
|
||||
{
|
||||
_cts?.Dispose();
|
||||
}
|
||||
}
|
||||
@@ -141,9 +141,9 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
private void InitializeSchema()
|
||||
{
|
||||
// auto_vacuum must be set before any table is created for it to take
|
||||
// effect on a fresh database. INCREMENTAL lets a future
|
||||
// `PRAGMA incremental_vacuum` shrink the file after the 7-day retention
|
||||
// purge — see alog.md §10.
|
||||
// effect on a fresh database. INCREMENTAL lets the `PRAGMA
|
||||
// incremental_vacuum` run at the end of PurgeExpiredAsync shrink the file
|
||||
// after the ~7-day retention purge — see alog.md §10.
|
||||
using (var pragmaCmd = _connection.CreateCommand())
|
||||
{
|
||||
pragmaCmd.CommandText = "PRAGMA auto_vacuum = INCREMENTAL";
|
||||
@@ -821,6 +821,117 @@ public class SqliteAuditWriter : IAuditWriter, ISiteAuditQueue, IAsyncDisposable
|
||||
OnDiskBytes: onDiskBytes));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<int> PurgeExpiredAsync(DateTime olderThanUtc, CancellationToken ct = default)
|
||||
{
|
||||
// Runs on the WRITE connection under _writeLock (a DELETE mutates the file),
|
||||
// mirroring MarkForwardedAsync / MarkReconciledAsync. Serialising with the
|
||||
// batched writer means the purge never races an in-flight FlushBatch.
|
||||
lock (_writeLock)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
|
||||
// Cutoff uses the same round-trip "o" encoding the writer stores
|
||||
// OccurredAtUtc in, so string comparison stays monotonic (see FlushBatch).
|
||||
var cutoff = EnsureUtc(olderThanUtc).ToString(
|
||||
"o", System.Globalization.CultureInfo.InvariantCulture);
|
||||
|
||||
int purged;
|
||||
using (var transaction = _connection.BeginTransaction())
|
||||
{
|
||||
try
|
||||
{
|
||||
// Capture the eligible EventIds from the sidecar FIRST: the
|
||||
// Forwarded/Reconciled + age criteria live on
|
||||
// audit_forward_state, and it is the FK child of audit_event, so
|
||||
// it must be deleted before the parent. A temp table (rather than
|
||||
// an IN (...) list) keeps an arbitrarily large purge set off the
|
||||
// SQLite 999-parameter limit. The hard invariant is encoded here:
|
||||
// only Forwarded/Reconciled rows qualify — Pending is never purged
|
||||
// on age alone.
|
||||
using (var createCmd = _connection.CreateCommand())
|
||||
{
|
||||
createCmd.Transaction = transaction;
|
||||
createCmd.CommandText = """
|
||||
DROP TABLE IF EXISTS purge_ids;
|
||||
CREATE TEMP TABLE purge_ids AS
|
||||
SELECT EventId FROM audit_forward_state
|
||||
WHERE ForwardState IN ($forwarded, $reconciled)
|
||||
AND OccurredAtUtc < $cutoff;
|
||||
""";
|
||||
createCmd.Parameters.AddWithValue("$forwarded", AuditForwardState.Forwarded.ToString());
|
||||
createCmd.Parameters.AddWithValue("$reconciled", AuditForwardState.Reconciled.ToString());
|
||||
createCmd.Parameters.AddWithValue("$cutoff", cutoff);
|
||||
createCmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
// Delete the sidecar (FK child) first, then the canonical row.
|
||||
using (var delSidecar = _connection.CreateCommand())
|
||||
{
|
||||
delSidecar.Transaction = transaction;
|
||||
delSidecar.CommandText =
|
||||
"DELETE FROM audit_forward_state WHERE EventId IN (SELECT EventId FROM purge_ids);";
|
||||
delSidecar.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
using (var delEvent = _connection.CreateCommand())
|
||||
{
|
||||
delEvent.Transaction = transaction;
|
||||
delEvent.CommandText =
|
||||
"DELETE FROM audit_event WHERE EventId IN (SELECT EventId FROM purge_ids);";
|
||||
delEvent.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
// changes() reports rows affected by the most recent DELETE on
|
||||
// this connection (the audit_event delete above); read it before
|
||||
// the DROP so no later statement can perturb the count.
|
||||
using (var changesCmd = _connection.CreateCommand())
|
||||
{
|
||||
changesCmd.Transaction = transaction;
|
||||
changesCmd.CommandText = "SELECT changes();";
|
||||
purged = Convert.ToInt32(changesCmd.ExecuteScalar(),
|
||||
System.Globalization.CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
using (var dropCmd = _connection.CreateCommand())
|
||||
{
|
||||
dropCmd.Transaction = transaction;
|
||||
dropCmd.CommandText = "DROP TABLE IF EXISTS purge_ids;";
|
||||
dropCmd.ExecuteNonQuery();
|
||||
}
|
||||
|
||||
transaction.Commit();
|
||||
}
|
||||
catch
|
||||
{
|
||||
transaction.Rollback();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
// Reclaim the freed pages back to the OS. auto_vacuum=INCREMENTAL (set in
|
||||
// InitializeSchema) makes this a bounded incremental step rather than a
|
||||
// full VACUUM rewrite. Runs outside the transaction; a vacuum failure must
|
||||
// not fault the purge itself, so it is best-effort.
|
||||
if (purged > 0)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var vacuumCmd = _connection.CreateCommand();
|
||||
vacuumCmd.CommandText = "PRAGMA incremental_vacuum;";
|
||||
vacuumCmd.ExecuteNonQuery();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex,
|
||||
"SqliteAuditWriter incremental_vacuum after retention purge failed (non-fatal).");
|
||||
}
|
||||
}
|
||||
|
||||
return Task.FromResult(purged);
|
||||
}
|
||||
}
|
||||
|
||||
private static DateTime EnsureUtc(DateTime value) =>
|
||||
value.Kind == DateTimeKind.Utc
|
||||
? value
|
||||
|
||||
@@ -83,9 +83,17 @@ public interface IAuditLogRepository
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="monthBoundary">Lower-bound datetime of the monthly partition to switch out.</param>
|
||||
/// <param name="commandTimeout">
|
||||
/// Optional per-command timeout for the maintenance SQL (the row-count sample plus the
|
||||
/// drop-and-rebuild dance). When null the provider default applies. The purge actor passes
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout"/>
|
||||
/// (default 30 min) because the ~30s ADO.NET default can abort the switch mid-dance on a large
|
||||
/// or contended partition, leaving the table without <c>UX_AuditLog_EventId</c> until the next
|
||||
/// tick recovers (arch-review 04, S2).
|
||||
/// </param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that resolves to the approximate number of rows discarded by the partition switch.</returns>
|
||||
Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default);
|
||||
Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Per-channel retention override purge. Deletes <c>AuditLog</c> rows for a
|
||||
@@ -115,12 +123,20 @@ public interface IAuditLogRepository
|
||||
/// <param name="channel">Canonical channel name (the <c>Category</c> column value, e.g. <c>ApiOutbound</c>).</param>
|
||||
/// <param name="threshold">Rows with <c>OccurredAtUtc</c> strictly older than this UTC datetime are deleted.</param>
|
||||
/// <param name="batchSize">Maximum rows deleted per batch; must be > 0.</param>
|
||||
/// <param name="commandTimeout">
|
||||
/// Optional per-batch <c>DELETE TOP</c> command timeout. When null the provider default applies.
|
||||
/// The purge actor passes the same
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout"/>
|
||||
/// it uses for the partition switch so a large single batch under lock contention is not aborted
|
||||
/// by the ~30s ADO.NET default (arch-review 04, S2).
|
||||
/// </param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that resolves to the total number of rows deleted across all batches.</returns>
|
||||
Task<long> PurgeChannelOlderThanAsync(
|
||||
string channel,
|
||||
DateTime threshold,
|
||||
int batchSize,
|
||||
TimeSpan? commandTimeout = null,
|
||||
CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -142,4 +142,25 @@ public interface ISiteAuditQueue
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that resolves to a point-in-time snapshot of the site audit queue's pending count, oldest timestamp, and on-disk file size.</returns>
|
||||
Task<SiteAuditBacklogSnapshot> GetBacklogStatsAsync(CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Retention purge: permanently deletes rows that have already left the site
|
||||
/// (<see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AuditForwardState.Forwarded"/> or
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AuditForwardState.Reconciled"/>)
|
||||
/// whose <see cref="AuditEvent.OccurredAtUtc"/> is strictly older than
|
||||
/// <paramref name="olderThanUtc"/>, and reclaims the freed pages.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <b>Hard <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AuditForwardState"/> invariant:</b>
|
||||
/// a row still in <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Enums.AuditForwardState.Pending"/>
|
||||
/// is NEVER purged on age alone — it has not yet reached central, so dropping it
|
||||
/// would silently lose audit data. Only Forwarded/Reconciled rows are eligible.
|
||||
/// Idempotent: a second call with the same cutoff removes nothing further and
|
||||
/// returns 0. The site store is ephemeral (~7-day retention), so this bounds its
|
||||
/// unbounded growth (arch-review 04, S1/U2).
|
||||
/// </remarks>
|
||||
/// <param name="olderThanUtc">Exclusive upper bound on <see cref="AuditEvent.OccurredAtUtc"/>; rows at or after this instant are retained.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that resolves to the number of canonical rows deleted.</returns>
|
||||
Task<int> PurgeExpiredAsync(DateTime olderThanUtc, CancellationToken ct = default);
|
||||
}
|
||||
|
||||
+14
-2
@@ -66,8 +66,9 @@ public class SiteCallEntityTypeConfiguration : IEntityTypeConfiguration<SiteCall
|
||||
// SourceNode (SourceNode-stamping): node-local identifier of the
|
||||
// cluster member that produced the call (e.g. "node-a", "central-a"). NULL is
|
||||
// valid for rows that pre-date this feature and for reconciled rows from a
|
||||
// retired node. ASCII — varchar(64). No index — Site Call Audit KPIs are
|
||||
// per-site, not per-node, on this table.
|
||||
// retired node. ASCII — varchar(64). Not a key column of any index, but it rides
|
||||
// as an INCLUDE column on IX_SiteCalls_NonTerminal below so the per-node KPI
|
||||
// breakdown reads it covered off the filtered live-queue index.
|
||||
builder.Property(s => s.SourceNode)
|
||||
.HasColumnType("varchar(64)")
|
||||
.HasMaxLength(64)
|
||||
@@ -85,5 +86,16 @@ public class SiteCallEntityTypeConfiguration : IEntityTypeConfiguration<SiteCall
|
||||
builder.HasIndex(s => new { s.Status, s.UpdatedAtUtc })
|
||||
.IsDescending(false, true)
|
||||
.HasDatabaseName("IX_SiteCalls_Status_Updated");
|
||||
|
||||
// NonTerminal backs every "live queue" KPI predicate (TerminalAtUtc IS NULL):
|
||||
// buffered/stuck/oldest counts run every 60 s from the KPI recorder plus every
|
||||
// 10 s Health-dashboard poll. Filtered: the non-terminal population is the live
|
||||
// queue, not the 365-day archive — each count becomes a small seek instead of a
|
||||
// clustered scan (arch-review 04, P1). SourceSite/SourceNode/Status ride as
|
||||
// INCLUDE columns so the per-site + per-node KPI breakdowns stay covered.
|
||||
builder.HasIndex(s => s.CreatedAtUtc)
|
||||
.HasDatabaseName("IX_SiteCalls_NonTerminal")
|
||||
.HasFilter("[TerminalAtUtc] IS NULL")
|
||||
.IncludeProperties(nameof(SiteCall.SourceSite), nameof(SiteCall.SourceNode), nameof(SiteCall.Status));
|
||||
}
|
||||
}
|
||||
|
||||
+2009
File diff suppressed because it is too large
Load Diff
+47
@@ -0,0 +1,47 @@
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
|
||||
{
|
||||
/// <summary>
|
||||
/// Grants the segregated maintenance principal <c>scadabridge_audit_purger</c> the two
|
||||
/// permissions its purge path actually needs but the original role creation
|
||||
/// (<c>CollapseAuditLogToCanonical</c>) omitted (arch-review 04, S3):
|
||||
/// <list type="bullet">
|
||||
/// <item><c>CREATE TABLE</c> — the switch-out dance CREATEs a staging table; the
|
||||
/// role's existing <c>ALTER ON SCHEMA::dbo</c> does NOT confer the database-level
|
||||
/// <c>CREATE TABLE</c> permission.</item>
|
||||
/// <item><c>DELETE ON dbo.AuditLog</c> — the per-channel retention override
|
||||
/// (<c>PurgeChannelOlderThanAsync</c>) is a bounded row DELETE on the maintenance path.</item>
|
||||
/// </list>
|
||||
/// Permission DDL only — no schema/data change. Idempotent and safe to run standalone.
|
||||
/// </summary>
|
||||
public partial class FixAuditPurgerRoleGrants : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.Sql(@"
|
||||
IF DATABASE_PRINCIPAL_ID('scadabridge_audit_purger') IS NULL
|
||||
EXEC sp_executesql N'CREATE ROLE scadabridge_audit_purger';
|
||||
-- The switch-out dance CREATEs a staging table; ALTER ON SCHEMA::dbo alone does not
|
||||
-- confer the database-level CREATE TABLE permission (review finding S3).
|
||||
GRANT CREATE TABLE TO scadabridge_audit_purger;
|
||||
-- The per-channel retention override (PurgeChannelOlderThanAsync) is a bounded row
|
||||
-- DELETE on the maintenance path; a segregated purger principal needs this grant.
|
||||
GRANT DELETE ON dbo.AuditLog TO scadabridge_audit_purger;"); // AUDIT-PURGE-ALLOWED: role grant for the maintenance principal
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.Sql(@"
|
||||
IF DATABASE_PRINCIPAL_ID('scadabridge_audit_purger') IS NOT NULL
|
||||
BEGIN
|
||||
REVOKE CREATE TABLE FROM scadabridge_audit_purger;
|
||||
REVOKE DELETE ON dbo.AuditLog FROM scadabridge_audit_purger;
|
||||
END"); // AUDIT-PURGE-ALLOWED: role grant reversal for the maintenance principal
|
||||
}
|
||||
}
|
||||
}
|
||||
+2015
File diff suppressed because it is too large
Load Diff
+29
@@ -0,0 +1,29 @@
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class AddSiteCallsNonTerminalIndex : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "IX_SiteCalls_NonTerminal",
|
||||
table: "SiteCalls",
|
||||
column: "CreatedAtUtc",
|
||||
filter: "[TerminalAtUtc] IS NULL")
|
||||
.Annotation("SqlServer:Include", new[] { "SourceSite", "SourceNode", "Status" });
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropIndex(
|
||||
name: "IX_SiteCalls_NonTerminal",
|
||||
table: "SiteCalls");
|
||||
}
|
||||
}
|
||||
}
|
||||
+6
@@ -161,6 +161,12 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
|
||||
|
||||
b.HasKey("TrackedOperationId");
|
||||
|
||||
b.HasIndex("CreatedAtUtc")
|
||||
.HasDatabaseName("IX_SiteCalls_NonTerminal")
|
||||
.HasFilter("[TerminalAtUtc] IS NULL");
|
||||
|
||||
SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("CreatedAtUtc"), new[] { "SourceSite", "SourceNode", "Status" });
|
||||
|
||||
b.HasIndex("SourceSite", "CreatedAtUtc")
|
||||
.IsDescending(false, true)
|
||||
.HasDatabaseName("IX_SiteCalls_Source_Created");
|
||||
|
||||
@@ -227,8 +227,13 @@ VALUES
|
||||
};
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default)
|
||||
public async Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default)
|
||||
{
|
||||
// Maintenance timeout in whole seconds (ADO.NET CommandTimeout unit). Null leaves the
|
||||
// provider default in place. See AuditLogPurgeOptions.MaintenanceCommandTimeoutMinutes /
|
||||
// arch-review 04 S2 for why the ~30s default is unsafe for the switch-out dance.
|
||||
var commandTimeoutSeconds = commandTimeout is { } ct2 ? (int)ct2.TotalSeconds : (int?)null;
|
||||
|
||||
// GUID-suffixed staging name: prevents collision with any concurrent
|
||||
// purge attempt and avoids polluting the AuditLog object namespace with
|
||||
// a predictable identifier.
|
||||
@@ -351,6 +356,10 @@ VALUES
|
||||
await using (var sampleCmd = conn.CreateCommand())
|
||||
{
|
||||
sampleCmd.CommandText = sampleSql;
|
||||
if (commandTimeoutSeconds is { } sampleTimeout)
|
||||
{
|
||||
sampleCmd.CommandTimeout = sampleTimeout;
|
||||
}
|
||||
var sampleResult = await sampleCmd.ExecuteScalarAsync(ct).ConfigureAwait(false);
|
||||
if (sampleResult is not null && sampleResult is not DBNull)
|
||||
{
|
||||
@@ -366,7 +375,27 @@ VALUES
|
||||
}
|
||||
}
|
||||
|
||||
await _context.Database.ExecuteSqlRawAsync(sql, ct);
|
||||
// Apply the maintenance timeout to the drop-and-rebuild dance. The context is scoped per
|
||||
// purge tick (see AuditLogPurgeActor.OnTickAsync's CreateAsyncScope), so a restore is not
|
||||
// strictly required — but we restore the previous value in a finally for hygiene in case a
|
||||
// future caller reuses the context for other work after the switch-out.
|
||||
var previousTimeout = _context.Database.GetCommandTimeout();
|
||||
if (commandTimeout is { } maintenanceTimeout)
|
||||
{
|
||||
_context.Database.SetCommandTimeout(maintenanceTimeout);
|
||||
}
|
||||
try
|
||||
{
|
||||
await _context.Database.ExecuteSqlRawAsync(sql, ct);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (commandTimeout is not null)
|
||||
{
|
||||
_context.Database.SetCommandTimeout(previousTimeout);
|
||||
}
|
||||
}
|
||||
|
||||
return rowsDeleted;
|
||||
}
|
||||
|
||||
@@ -375,6 +404,7 @@ VALUES
|
||||
string channel,
|
||||
DateTime threshold,
|
||||
int batchSize,
|
||||
TimeSpan? commandTimeout = null,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(channel))
|
||||
@@ -389,6 +419,9 @@ VALUES
|
||||
|
||||
var thresholdUtc = DateTime.SpecifyKind(threshold.ToUniversalTime(), DateTimeKind.Utc);
|
||||
|
||||
// Maintenance timeout in whole seconds; null leaves the provider default (arch-review 04 S2).
|
||||
var commandTimeoutSeconds = commandTimeout is { } cto ? (int)cto.TotalSeconds : (int?)null;
|
||||
|
||||
// Per-channel retention override purge. This is the ONLY DELETE
|
||||
// against dbo.AuditLog in the codebase and it runs on the purge/maintenance
|
||||
// path, NOT the append-only writer role (which has INSERT + SELECT only — see
|
||||
@@ -427,6 +460,10 @@ VALUES
|
||||
|
||||
await using var cmd = conn.CreateCommand();
|
||||
cmd.CommandText = deleteBatchSql;
|
||||
if (commandTimeoutSeconds is { } batchTimeout)
|
||||
{
|
||||
cmd.CommandTimeout = batchTimeout;
|
||||
}
|
||||
|
||||
var pBatch = cmd.CreateParameter();
|
||||
pBatch.ParameterName = "@batch";
|
||||
@@ -628,6 +665,16 @@ VALUES
|
||||
// practice.
|
||||
private const int ExecutionChainMaxDepth = 32;
|
||||
|
||||
// Execution trees span minutes, not years — an inbound request and every script it
|
||||
// spawns all fire inside one operational burst. Anchoring the Edges CTE scan to a
|
||||
// narrow window around the root's first event lets SQL Server eliminate every
|
||||
// AuditLog partition outside it, turning a full-table DISTINCT scan into a seek on
|
||||
// IX_AuditLog_Execution over one or two months (arch-review 04, P2). The slack absorbs
|
||||
// minor clock skew before the root; the max span is the documented traversal bound —
|
||||
// a genuine descendant stamped beyond it is excluded by design.
|
||||
private static readonly TimeSpan ExecutionTreeWindowSlack = TimeSpan.FromHours(1);
|
||||
private static readonly TimeSpan ExecutionTreeMaxSpan = TimeSpan.FromDays(7);
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<ExecutionTreeNode>> GetExecutionTreeAsync(
|
||||
Guid executionId,
|
||||
@@ -676,6 +723,31 @@ VALUES
|
||||
rootExecutionId = parent.Value;
|
||||
}
|
||||
|
||||
// --- Between phases: bound the down-walk to a root-anchored window ---
|
||||
// The root's first event timestamp anchors a [rootFirst - slack, rootFirst + maxSpan)
|
||||
// window used to prune the Edges CTE scan to a handful of partitions. A row-less stub
|
||||
// root (purged/no-action parent) has no MIN — leave the scan unbounded in that
|
||||
// degenerate case (correctness over speed).
|
||||
DateTime? rootFirstOccurred;
|
||||
await using (var winCmd = conn.CreateCommand())
|
||||
{
|
||||
winCmd.CommandText =
|
||||
"SELECT MIN(OccurredAtUtc) FROM dbo.AuditLog WHERE ExecutionId = @root;";
|
||||
var pWinRoot = winCmd.CreateParameter();
|
||||
pWinRoot.ParameterName = "@root";
|
||||
pWinRoot.Value = rootExecutionId;
|
||||
winCmd.Parameters.Add(pWinRoot);
|
||||
|
||||
var winResult = await winCmd.ExecuteScalarAsync(ct).ConfigureAwait(false);
|
||||
rootFirstOccurred = winResult is null or DBNull ? null : (DateTime)winResult;
|
||||
}
|
||||
|
||||
// Build the optional window predicate for the Edges CTE. When the root has rows the
|
||||
// scan is confined to the window (partition elimination); otherwise it stays unbounded.
|
||||
var edgesWindowPredicate = rootFirstOccurred is null
|
||||
? string.Empty
|
||||
: "AND OccurredAtUtc >= @winStart AND OccurredAtUtc < @winEnd";
|
||||
|
||||
// --- Phase 2: walk down from the root via a recursive CTE ---------
|
||||
// Edges : a non-recursive, DISTINCT (ExecutionId, ParentExecutionId)
|
||||
// edge set distilled from AuditLog. Recursing over edges
|
||||
@@ -706,6 +778,7 @@ VALUES
|
||||
SELECT DISTINCT ExecutionId, ParentExecutionId
|
||||
FROM dbo.AuditLog
|
||||
WHERE ExecutionId IS NOT NULL
|
||||
{edgesWindowPredicate}
|
||||
),
|
||||
Chain AS (
|
||||
-- Anchor: the root execution id, seeded as a literal so
|
||||
@@ -760,6 +833,19 @@ VALUES
|
||||
pRoot.Value = rootExecutionId;
|
||||
downCmd.Parameters.Add(pRoot);
|
||||
|
||||
if (rootFirstOccurred is { } rootFirst)
|
||||
{
|
||||
var pWinStart = downCmd.CreateParameter();
|
||||
pWinStart.ParameterName = "@winStart";
|
||||
pWinStart.Value = rootFirst - ExecutionTreeWindowSlack;
|
||||
downCmd.Parameters.Add(pWinStart);
|
||||
|
||||
var pWinEnd = downCmd.CreateParameter();
|
||||
pWinEnd.ParameterName = "@winEnd";
|
||||
pWinEnd.Value = rootFirst + ExecutionTreeMaxSpan;
|
||||
downCmd.Parameters.Add(pWinEnd);
|
||||
}
|
||||
|
||||
await using var reader = await downCmd.ExecuteReaderAsync(ct).ConfigureAwait(false);
|
||||
while (await reader.ReadAsync(ct).ConfigureAwait(false))
|
||||
{
|
||||
|
||||
@@ -259,12 +259,12 @@ public class AuditLogIngestActorTests : TestKit, IClassFixture<MsSqlMigrationFix
|
||||
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default) =>
|
||||
_inner.QueryAsync(filter, paging, ct);
|
||||
|
||||
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default) =>
|
||||
_inner.SwitchOutPartitionAsync(monthBoundary, ct);
|
||||
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
|
||||
_inner.SwitchOutPartitionAsync(monthBoundary, commandTimeout, ct);
|
||||
|
||||
public Task<long> PurgeChannelOlderThanAsync(
|
||||
string channel, DateTime threshold, int batchSize, CancellationToken ct = default) =>
|
||||
_inner.PurgeChannelOlderThanAsync(channel, threshold, batchSize, ct);
|
||||
string channel, DateTime threshold, int batchSize, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
|
||||
_inner.PurgeChannelOlderThanAsync(channel, threshold, batchSize, commandTimeout, ct);
|
||||
|
||||
public Task<long> BackfillSourceNodeAsync(
|
||||
string sentinel, DateTime before, int batchSize, CancellationToken ct = default) =>
|
||||
|
||||
@@ -51,6 +51,11 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
|
||||
public DateTime? ThrowOnBoundary { get; set; }
|
||||
public Exception? BoundaryException { get; set; }
|
||||
|
||||
// Task 4: captures the maintenance command timeout the actor threads through to the
|
||||
// switch-out + per-channel purge, so tests can assert the options value reaches the repo.
|
||||
public List<TimeSpan?> SwitchTimeouts { get; } = new();
|
||||
public List<TimeSpan?> ChannelTimeouts { get; } = new();
|
||||
|
||||
// M5.5 (T3): records every per-channel purge call as
|
||||
// (channel, threshold, batchSize) so tests can assert which channels the
|
||||
// actor chose to purge and with what window.
|
||||
@@ -69,8 +74,9 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
|
||||
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default) =>
|
||||
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>());
|
||||
|
||||
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default)
|
||||
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default)
|
||||
{
|
||||
SwitchTimeouts.Add(commandTimeout);
|
||||
if (ThrowOnBoundary.HasValue && monthBoundary == ThrowOnBoundary.Value)
|
||||
{
|
||||
throw BoundaryException ?? new InvalidOperationException("simulated switch failure");
|
||||
@@ -87,9 +93,10 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
|
||||
}
|
||||
|
||||
public Task<long> PurgeChannelOlderThanAsync(
|
||||
string channel, DateTime threshold, int batchSize, CancellationToken ct = default)
|
||||
string channel, DateTime threshold, int batchSize, TimeSpan? commandTimeout = null, CancellationToken ct = default)
|
||||
{
|
||||
ChannelPurges.Add((channel, threshold, batchSize));
|
||||
ChannelTimeouts.Add(commandTimeout);
|
||||
return Task.FromResult(RowsPerChannel(channel));
|
||||
}
|
||||
|
||||
@@ -491,4 +498,95 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture<MsSqlMigrationFixt
|
||||
|
||||
Assert.Empty(repo.ChannelPurges);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 10. PurgeTick_Passes_MaintenanceCommandTimeout_To_Repository (Task 4)
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void PurgeTick_Passes_MaintenanceCommandTimeout_To_Repository()
|
||||
{
|
||||
// The actor must thread AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout through to
|
||||
// BOTH the partition switch-out and each per-channel purge, so the ~30s ADO.NET default
|
||||
// cannot self-lock the drop-and-rebuild dance (arch-review 04, S2).
|
||||
var boundary = new DateTime(2025, 5, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
var repo = new RecordingRepo
|
||||
{
|
||||
Boundaries = new List<DateTime> { boundary },
|
||||
RowsPerBoundary = _ => 7L,
|
||||
};
|
||||
|
||||
var purgeOptions = FastTickOptions();
|
||||
purgeOptions.MaintenanceCommandTimeoutMinutes = 45;
|
||||
|
||||
// A per-channel override shorter than the global window so the channel purge also fires.
|
||||
var auditOptions = Options.Create(new AuditLogOptions
|
||||
{
|
||||
RetentionDays = 365,
|
||||
PerChannelRetentionDays = new Dictionary<string, int> { ["ApiOutbound"] = 30 },
|
||||
});
|
||||
|
||||
var sp = BuildScopedProvider(repo);
|
||||
Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
|
||||
sp,
|
||||
Options.Create(purgeOptions),
|
||||
auditOptions,
|
||||
NullLogger<AuditLogPurgeActor>.Instance)));
|
||||
|
||||
var expected = TimeSpan.FromMinutes(45);
|
||||
AwaitAssert(
|
||||
() =>
|
||||
{
|
||||
Assert.Contains(expected, repo.SwitchTimeouts);
|
||||
Assert.Contains(expected, repo.ChannelTimeouts);
|
||||
},
|
||||
duration: TimeSpan.FromSeconds(3),
|
||||
interval: TimeSpan.FromMilliseconds(50));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 11. SwitchThrows_PublishesPurgeFailedEvent_AndIncrementsHealthCounter (Task 5)
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void SwitchThrows_PublishesPurgeFailedEvent_AndIncrementsHealthCounter()
|
||||
{
|
||||
// A silently failing retention job is a health-surface condition, not just a log line:
|
||||
// the per-boundary catch must publish AuditLogPurgeFailedEvent AND bump the
|
||||
// AuditCentralHealthSnapshot.PurgeFailures counter (arch-review 04, S2).
|
||||
var boundary = new DateTime(2025, 9, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
var repo = new RecordingRepo
|
||||
{
|
||||
Boundaries = new List<DateTime> { boundary },
|
||||
ThrowOnBoundary = boundary,
|
||||
BoundaryException = new InvalidOperationException("simulated switch timeout"),
|
||||
};
|
||||
|
||||
var snapshot = new AuditCentralHealthSnapshot();
|
||||
var services = new ServiceCollection();
|
||||
services.AddScoped<IAuditLogRepository>(_ => repo);
|
||||
services.AddSingleton(snapshot);
|
||||
services.AddSingleton<IAuditPurgeFailureCounter>(snapshot);
|
||||
var sp = services.BuildServiceProvider();
|
||||
|
||||
var probe = CreateTestProbe();
|
||||
Sys.EventStream.Subscribe(probe.Ref, typeof(AuditLogPurgeFailedEvent));
|
||||
|
||||
Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor(
|
||||
sp,
|
||||
Options.Create(FastTickOptions()),
|
||||
Options.Create(new AuditLogOptions()),
|
||||
NullLogger<AuditLogPurgeActor>.Instance)));
|
||||
|
||||
var evt = probe.ExpectMsg<AuditLogPurgeFailedEvent>(TimeSpan.FromSeconds(5));
|
||||
Assert.Equal(boundary, evt.MonthBoundary);
|
||||
Assert.Contains("simulated switch timeout", evt.Error);
|
||||
Assert.True(evt.ElapsedMilliseconds >= 0);
|
||||
|
||||
AwaitAssert(
|
||||
() => Assert.True(snapshot.PurgeFailures >= 1,
|
||||
$"expected PurgeFailures >= 1, got {snapshot.PurgeFailures}"),
|
||||
duration: TimeSpan.FromSeconds(3),
|
||||
interval: TimeSpan.FromMilliseconds(50));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Central;
|
||||
|
||||
/// <summary>
|
||||
/// Task 4 tests for <see cref="AuditLogPurgeOptions.MaintenanceCommandTimeoutMinutes"/> /
|
||||
/// <see cref="AuditLogPurgeOptions.ResolvedMaintenanceCommandTimeout"/>: the maintenance timeout
|
||||
/// defaults to 30 minutes and is clamped to a 1-minute floor so a misconfigured 0/negative value
|
||||
/// cannot yield a zero/negative ADO.NET command timeout (arch-review 04, S2).
|
||||
/// </summary>
|
||||
public class AuditLogPurgeOptionsTests
|
||||
{
|
||||
[Fact]
|
||||
public void MaintenanceCommandTimeout_Defaults_To_30_Minutes()
|
||||
{
|
||||
var options = new AuditLogPurgeOptions();
|
||||
Assert.Equal(30, options.MaintenanceCommandTimeoutMinutes);
|
||||
Assert.Equal(TimeSpan.FromMinutes(30), options.ResolvedMaintenanceCommandTimeout);
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(0, 1)] // clamp floor — a zero config value cannot become a zero timeout
|
||||
[InlineData(-5, 1)] // clamp floor — negatives are rejected by ADO.NET
|
||||
[InlineData(1, 1)] // boundary passthrough
|
||||
[InlineData(45, 45)] // ordinary passthrough
|
||||
public void ResolvedMaintenanceCommandTimeout_Clamps_To_MinimumOneMinute(int configured, int expectedMinutes)
|
||||
{
|
||||
var options = new AuditLogPurgeOptions { MaintenanceCommandTimeoutMinutes = configured };
|
||||
Assert.Equal(TimeSpan.FromMinutes(expectedMinutes), options.ResolvedMaintenanceCommandTimeout);
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -42,10 +42,10 @@ public class CentralAuditWriteFailuresTests : TestKit
|
||||
public Task<IReadOnlyList<AuditEvent>> QueryAsync(
|
||||
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default) =>
|
||||
Task.FromResult<IReadOnlyList<AuditEvent>>(Array.Empty<AuditEvent>());
|
||||
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default) =>
|
||||
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
|
||||
Task.FromResult(0L);
|
||||
public Task<long> PurgeChannelOlderThanAsync(
|
||||
string channel, DateTime threshold, int batchSize, CancellationToken ct = default) =>
|
||||
string channel, DateTime threshold, int batchSize, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
|
||||
Task.FromResult(0L);
|
||||
public Task<long> BackfillSourceNodeAsync(
|
||||
string sentinel, DateTime before, int batchSize, CancellationToken ct = default) =>
|
||||
|
||||
+2
-2
@@ -86,11 +86,11 @@ public class SiteAuditReconciliationActorTests : TestKit, IClassFixture<MsSqlMig
|
||||
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default) =>
|
||||
Task.FromResult<IReadOnlyList<AuditEvent>>(Inserted);
|
||||
|
||||
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default) =>
|
||||
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
|
||||
Task.FromResult(0L);
|
||||
|
||||
public Task<long> PurgeChannelOlderThanAsync(
|
||||
string channel, DateTime threshold, int batchSize, CancellationToken ct = default) =>
|
||||
string channel, DateTime threshold, int batchSize, TimeSpan? commandTimeout = null, CancellationToken ct = default) =>
|
||||
Task.FromResult(0L);
|
||||
|
||||
public Task<long> BackfillSourceNodeAsync(
|
||||
|
||||
@@ -474,4 +474,54 @@ WHERE name = 'UX_AuditLog_EventId'
|
||||
Assert.Contains(rows, r => r.EventId == dbOldId);
|
||||
Assert.Contains(rows, r => r.EventId == dbRecentId);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// 5. SwitchOut_WithExplicitCommandTimeout_Succeeds (Task 4)
|
||||
// ---------------------------------------------------------------------
|
||||
|
||||
/// <summary>
|
||||
/// Task 4 (arch-review 04, S2): proves the explicit-<see cref="TimeSpan"/> maintenance-timeout
|
||||
/// overload of <see cref="IAuditLogRepository.SwitchOutPartitionAsync"/> runs the real
|
||||
/// drop-and-rebuild dance to completion against SQL Server — the old row is purged, the kept row
|
||||
/// survives, and the returned sampled row-count reflects the switched partition. Exercising the
|
||||
/// timeout path end-to-end guards against a regression that only sets the timeout on the sample
|
||||
/// command and forgets the DDL batch (or vice versa).
|
||||
/// </summary>
|
||||
[SkippableFact]
|
||||
public async Task SwitchOut_WithExplicitCommandTimeout_Succeeds()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
var siteId = "switch-timeout-" + Guid.NewGuid().ToString("N").Substring(0, 8);
|
||||
var oldEventId = Guid.NewGuid();
|
||||
var keptEventId = Guid.NewGuid();
|
||||
var (oldOccurred, keptOccurred, _, _) = SeedOccurredAt();
|
||||
|
||||
await using (var seedConn = _fixture.OpenConnection())
|
||||
{
|
||||
await DirectInsertAsync(seedConn, oldEventId, oldOccurred, siteId);
|
||||
await DirectInsertAsync(seedConn, keptEventId, keptOccurred, siteId);
|
||||
}
|
||||
|
||||
var janBoundary = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
await using (var ctx = CreateContext())
|
||||
{
|
||||
var repo = new AuditLogRepository(ctx);
|
||||
var rowsDeleted = await repo.SwitchOutPartitionAsync(janBoundary, TimeSpan.FromMinutes(30));
|
||||
Assert.True(rowsDeleted >= 1,
|
||||
$"Expected the Jan-2026 partition switch-out to report >= 1 sampled row; got {rowsDeleted}.");
|
||||
}
|
||||
|
||||
await using var verify = CreateContext();
|
||||
var rows = await verify.Set<AuditLogRow>()
|
||||
.Where(e => e.SourceSiteId == siteId)
|
||||
.ToListAsync();
|
||||
Assert.DoesNotContain(rows, r => r.EventId == oldEventId);
|
||||
Assert.Contains(rows, r => r.EventId == keptEventId);
|
||||
|
||||
// The dance must leave the idempotency-supporting unique index rebuilt.
|
||||
await using var check = _fixture.OpenConnection();
|
||||
await AssertUxIndexExistsAsync(check);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="SiteAuditRetentionOptions"/> clamping (PLAN-04 Task 2).
|
||||
/// The resolved knobs guard the daily site retention job against misconfiguration:
|
||||
/// retention days are clamped to [1, 90]; the purge interval is clamped away from
|
||||
/// TimeSpan.Zero (an Akka/Timer spin footgun).
|
||||
/// </summary>
|
||||
public class SiteAuditRetentionOptionsTests
|
||||
{
|
||||
[Theory]
|
||||
[InlineData(0, 1)] // clamp floor
|
||||
[InlineData(7, 7)] // default passthrough
|
||||
[InlineData(365, 90)] // clamp ceiling (spec: min 1, max 90)
|
||||
public void ResolvedRetentionDays_Clamps(int configured, int expected)
|
||||
=> Assert.Equal(expected, new SiteAuditRetentionOptions { RetentionDays = configured }.ResolvedRetentionDays);
|
||||
|
||||
[Fact]
|
||||
public void Defaults_Are_SevenDays_DailyPurge_FiveMinuteInitialDelay()
|
||||
{
|
||||
var o = new SiteAuditRetentionOptions();
|
||||
Assert.Equal(7, o.ResolvedRetentionDays);
|
||||
Assert.Equal(TimeSpan.FromHours(24), o.ResolvedPurgeInterval);
|
||||
Assert.Equal(TimeSpan.FromMinutes(5), o.InitialDelay);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void ResolvedPurgeInterval_ClampsZeroToOneMinute()
|
||||
=> Assert.Equal(TimeSpan.FromMinutes(1),
|
||||
new SiteAuditRetentionOptions { PurgeInterval = TimeSpan.Zero }.ResolvedPurgeInterval);
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Diagnostics;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
using AuditEvent = ZB.MOM.WW.Audit.AuditEvent;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="SiteAuditRetentionService"/> (PLAN-04 Task 3). The hosted
|
||||
/// service runs the site SQLite retention purge on a timer: first tick after
|
||||
/// <see cref="SiteAuditRetentionOptions.InitialDelay"/>, then every
|
||||
/// <see cref="SiteAuditRetentionOptions.ResolvedPurgeInterval"/>, passing a cutoff of
|
||||
/// <c>UtcNow - ResolvedRetentionDays</c>. Per-tick failures are swallowed so a
|
||||
/// transient SQLite error never tears the service down.
|
||||
/// </summary>
|
||||
public class SiteAuditRetentionServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Tick_Purges_With_RetentionCutoff()
|
||||
{
|
||||
var queue = new RecordingSiteAuditQueue();
|
||||
var options = Options.Create(new SiteAuditRetentionOptions
|
||||
{
|
||||
RetentionDays = 7,
|
||||
// Override bypasses the 1-min production clamp so the loop ticks in ms.
|
||||
PurgeIntervalOverride = TimeSpan.FromMilliseconds(50),
|
||||
InitialDelay = TimeSpan.Zero,
|
||||
});
|
||||
using var svc = new SiteAuditRetentionService(queue, options, NullLogger<SiteAuditRetentionService>.Instance);
|
||||
|
||||
await svc.StartAsync(CancellationToken.None);
|
||||
await WaitUntilAsync(() => queue.PurgeCalls.Count >= 1, TimeSpan.FromSeconds(5));
|
||||
await svc.StopAsync(CancellationToken.None);
|
||||
|
||||
var cutoff = queue.PurgeCalls.First();
|
||||
Assert.InRange(cutoff,
|
||||
DateTime.UtcNow.AddDays(-7).AddMinutes(-1),
|
||||
DateTime.UtcNow.AddDays(-7).AddMinutes(1));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Tick_SwallowsExceptions_AndKeepsTicking()
|
||||
{
|
||||
var queue = new RecordingSiteAuditQueue { ThrowOnFirstCall = true };
|
||||
var options = Options.Create(new SiteAuditRetentionOptions
|
||||
{
|
||||
RetentionDays = 7,
|
||||
// Override bypasses the 1-min production clamp so the loop ticks in ms.
|
||||
PurgeIntervalOverride = TimeSpan.FromMilliseconds(50),
|
||||
InitialDelay = TimeSpan.Zero,
|
||||
});
|
||||
using var svc = new SiteAuditRetentionService(queue, options, NullLogger<SiteAuditRetentionService>.Instance);
|
||||
|
||||
await svc.StartAsync(CancellationToken.None);
|
||||
// The first tick throws; a second tick must still arrive (service survives).
|
||||
await WaitUntilAsync(() => queue.PurgeCalls.Count >= 2, TimeSpan.FromSeconds(5));
|
||||
await svc.StopAsync(CancellationToken.None);
|
||||
|
||||
Assert.True(queue.PurgeCalls.Count >= 2);
|
||||
}
|
||||
|
||||
private static async Task WaitUntilAsync(Func<bool> condition, TimeSpan timeout)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
while (!condition())
|
||||
{
|
||||
if (sw.Elapsed > timeout)
|
||||
{
|
||||
throw new TimeoutException("Condition was not met within the timeout.");
|
||||
}
|
||||
|
||||
await Task.Delay(10);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Stub queue that records purge cutoffs and can throw on the first tick.</summary>
|
||||
private sealed class RecordingSiteAuditQueue : ISiteAuditQueue
|
||||
{
|
||||
private readonly ConcurrentQueue<DateTime> _purgeCalls = new();
|
||||
private int _calls;
|
||||
|
||||
public IReadOnlyCollection<DateTime> PurgeCalls => _purgeCalls;
|
||||
public bool ThrowOnFirstCall { get; init; }
|
||||
|
||||
public Task<int> PurgeExpiredAsync(DateTime olderThanUtc, CancellationToken ct = default)
|
||||
{
|
||||
var n = Interlocked.Increment(ref _calls);
|
||||
_purgeCalls.Enqueue(olderThanUtc);
|
||||
if (ThrowOnFirstCall && n == 1)
|
||||
{
|
||||
throw new InvalidOperationException("Simulated transient purge failure.");
|
||||
}
|
||||
|
||||
return Task.FromResult(0);
|
||||
}
|
||||
|
||||
public Task<IReadOnlyList<AuditEvent>> ReadPendingAsync(int limit, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
public Task<IReadOnlyList<AuditEvent>> ReadPendingCachedTelemetryAsync(int limit, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
public Task MarkForwardedAsync(IReadOnlyList<Guid> eventIds, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
public Task<IReadOnlyList<AuditEvent>> ReadPendingSinceAsync(DateTime sinceUtc, int batchSize, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
public Task MarkReconciledAsync(IReadOnlyList<Guid> eventIds, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
public Task<SiteAuditBacklogSnapshot> GetBacklogStatsAsync(CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
using ZB.MOM.WW.ScadaBridge.AuditLog.Tests.TestSupport;
|
||||
using ZB.MOM.WW.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Site;
|
||||
|
||||
/// <summary>
|
||||
/// Tests for <see cref="SqliteAuditWriter.PurgeExpiredAsync"/> (arch-review 04, S1/U2).
|
||||
/// The site SQLite store is ephemeral (~7-day retention); the retention purge deletes
|
||||
/// rows that have already left the site (Forwarded/Reconciled) and are older than the
|
||||
/// cutoff, while honoring the hard <see cref="AuditForwardState"/> invariant: a
|
||||
/// <see cref="AuditForwardState.Pending"/> row is NEVER purged on age alone — dropping
|
||||
/// an un-forwarded row would silently lose audit data that never reached central.
|
||||
/// Fixture mirrors <see cref="SqliteAuditWriterBacklogStatsTests"/> (file-backed so the
|
||||
/// post-purge incremental_vacuum has a real file to shrink).
|
||||
/// </summary>
|
||||
public class SqliteAuditWriterRetentionTests : IDisposable
|
||||
{
|
||||
private readonly string _dbPath;
|
||||
|
||||
public SqliteAuditWriterRetentionTests()
|
||||
{
|
||||
_dbPath = Path.Combine(Path.GetTempPath(),
|
||||
$"audit-retention-{Guid.NewGuid():N}.db");
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
if (File.Exists(_dbPath))
|
||||
{
|
||||
try { File.Delete(_dbPath); } catch { /* test cleanup best-effort */ }
|
||||
}
|
||||
}
|
||||
|
||||
private SqliteAuditWriter CreateWriter()
|
||||
{
|
||||
var options = new SqliteAuditWriterOptions { DatabasePath = _dbPath };
|
||||
return new SqliteAuditWriter(
|
||||
Options.Create(options),
|
||||
NullLogger<SqliteAuditWriter>.Instance,
|
||||
new FakeNodeIdentityProvider());
|
||||
}
|
||||
|
||||
private static async Task<Guid> WriteEventAsync(SqliteAuditWriter writer, DateTime occurredAtUtc)
|
||||
{
|
||||
var id = Guid.NewGuid();
|
||||
var evt = ScadaBridgeAuditEventFactory.Create(
|
||||
eventId: id,
|
||||
occurredAtUtc: occurredAtUtc,
|
||||
channel: AuditChannel.ApiOutbound,
|
||||
kind: AuditKind.ApiCall,
|
||||
status: AuditStatus.Delivered);
|
||||
await writer.WriteAsync(evt);
|
||||
return id;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PurgeExpired_DeletesForwardedAndReconciled_OlderThanCutoff()
|
||||
{
|
||||
await using var writer = CreateWriter();
|
||||
var oldForwarded = await WriteEventAsync(writer, DateTime.UtcNow.AddDays(-10));
|
||||
var oldReconciled = await WriteEventAsync(writer, DateTime.UtcNow.AddDays(-10));
|
||||
var oldPending = await WriteEventAsync(writer, DateTime.UtcNow.AddDays(-10));
|
||||
var fresh = await WriteEventAsync(writer, DateTime.UtcNow);
|
||||
await writer.MarkForwardedAsync(new[] { oldForwarded, oldReconciled, fresh });
|
||||
await writer.MarkReconciledAsync(new[] { oldReconciled });
|
||||
|
||||
var purged = await writer.PurgeExpiredAsync(DateTime.UtcNow.AddDays(-7));
|
||||
|
||||
Assert.Equal(2, purged); // oldForwarded + oldReconciled
|
||||
// Hard ForwardState invariant: the old *Pending* row survives on age alone.
|
||||
var pending = await writer.ReadPendingAsync(10);
|
||||
Assert.Contains(pending, e => e.EventId == oldPending);
|
||||
// Fresh forwarded row inside the window survives; only oldPending remains Pending.
|
||||
var stats = await writer.GetBacklogStatsAsync();
|
||||
Assert.Equal(1, stats.PendingCount);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task PurgeExpired_IsIdempotent()
|
||||
{
|
||||
await using var writer = CreateWriter();
|
||||
var id = await WriteEventAsync(writer, DateTime.UtcNow.AddDays(-10));
|
||||
await writer.MarkForwardedAsync(new[] { id });
|
||||
Assert.Equal(1, await writer.PurgeExpiredAsync(DateTime.UtcNow.AddDays(-7)));
|
||||
Assert.Equal(0, await writer.PurgeExpiredAsync(DateTime.UtcNow.AddDays(-7)));
|
||||
}
|
||||
}
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
using Xunit;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations;
|
||||
|
||||
/// <summary>
|
||||
/// Content guard for the <c>FixAuditPurgerRoleGrants</c> migration (arch-review 04, S3).
|
||||
/// The switch-out dance CREATEs a staging table and the per-channel override runs a bounded
|
||||
/// row DELETE — but the shipped <c>scadabridge_audit_purger</c> role only carried
|
||||
/// <c>SELECT</c> + <c>ALTER ON SCHEMA::dbo</c>, which does NOT confer the database-level
|
||||
/// <c>CREATE TABLE</c> permission nor row-level <c>DELETE</c>. This test asserts the migration
|
||||
/// source grants both, so a segregated maintenance principal can genuinely execute the purge.
|
||||
/// A pure source-content scan (no MSSQL required) — the schema-application sanity check lives
|
||||
/// in the MSSQL migration suite.
|
||||
/// </summary>
|
||||
public class FixAuditPurgerRoleGrantsTests
|
||||
{
|
||||
private static string ReadMigrationSource()
|
||||
{
|
||||
var migrationsDir = Path.Combine(
|
||||
GetConfigurationDatabaseSourceDirectory(), "Migrations");
|
||||
|
||||
// Timestamp-prefixed filename; match the non-Designer partial only.
|
||||
var file = Directory.GetFiles(migrationsDir, "*_FixAuditPurgerRoleGrants.cs")
|
||||
.FirstOrDefault(f => !f.EndsWith(".Designer.cs", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
Assert.True(file is not null,
|
||||
"Expected a *_FixAuditPurgerRoleGrants.cs migration under " + migrationsDir +
|
||||
" — scaffold it with `dotnet ef migrations add FixAuditPurgerRoleGrants` (build first, NO --no-build).");
|
||||
return File.ReadAllText(file!);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Migration_Grants_CreateTable_To_Purger()
|
||||
{
|
||||
var src = ReadMigrationSource();
|
||||
Assert.Contains("GRANT CREATE TABLE TO scadabridge_audit_purger", src);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Migration_Grants_Delete_On_AuditLog_To_Purger()
|
||||
{
|
||||
var src = ReadMigrationSource();
|
||||
Assert.Contains("GRANT DELETE ON dbo.AuditLog TO scadabridge_audit_purger", src);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Down_Revokes_Both_Grants()
|
||||
{
|
||||
var src = ReadMigrationSource();
|
||||
// Down must reverse both grants so the migration is cleanly reversible.
|
||||
Assert.Contains("REVOKE CREATE TABLE", src);
|
||||
Assert.Contains("REVOKE DELETE ON dbo.AuditLog", src);
|
||||
}
|
||||
|
||||
// Same walk-up anchor pattern the append-only guard uses.
|
||||
private static string GetConfigurationDatabaseSourceDirectory()
|
||||
{
|
||||
var dir = new DirectoryInfo(AppContext.BaseDirectory);
|
||||
while (dir != null)
|
||||
{
|
||||
var candidate = Path.Combine(
|
||||
dir.FullName, "src",
|
||||
"ZB.MOM.WW.ScadaBridge.ConfigurationDatabase",
|
||||
"ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.csproj");
|
||||
if (File.Exists(candidate))
|
||||
{
|
||||
return Path.GetDirectoryName(candidate)!;
|
||||
}
|
||||
dir = dir.Parent;
|
||||
}
|
||||
throw new InvalidOperationException(
|
||||
"Could not locate ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.csproj by walking up from the test output directory.");
|
||||
}
|
||||
}
|
||||
+43
@@ -1008,6 +1008,49 @@ public class AuditLogRepositoryTests : IClassFixture<MsSqlMigrationFixture>
|
||||
}
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task ExecutionTree_Bounds_EdgeScan_To_Root_Window()
|
||||
{
|
||||
Skip.IfNot(_fixture.Available, _fixture.SkipReason);
|
||||
|
||||
// arch-review 04 (P2): the down-walk Edges CTE is anchored to a
|
||||
// [rootFirst - 1h, rootFirst + 7d) window around the root's first event, enabling
|
||||
// partition elimination instead of a full-table DISTINCT scan. A genuine descendant
|
||||
// stamped BEYOND that 7-day span is excluded by design; a normal in-window child is
|
||||
// still returned.
|
||||
var siteId = NewSiteId();
|
||||
await using var context = CreateContext();
|
||||
var repo = new AuditLogRepository(context);
|
||||
|
||||
var rootExec = Guid.NewGuid();
|
||||
var childExec = Guid.NewGuid();
|
||||
var farDescendantExec = Guid.NewGuid();
|
||||
var unrelatedExec = Guid.NewGuid();
|
||||
|
||||
var t0 = new DateTime(2026, 10, 5, 9, 0, 0, DateTimeKind.Utc);
|
||||
|
||||
// Root + an in-window child (T0 + 5 min) — both must be returned.
|
||||
await repo.InsertIfNotExistsAsync(NewEvent(siteId, occurredAtUtc: t0, executionId: rootExec));
|
||||
await repo.InsertIfNotExistsAsync(NewEvent(siteId, occurredAtUtc: t0.AddMinutes(5), executionId: childExec, parentExecutionId: rootExec));
|
||||
|
||||
// A genuine descendant of the child, but stamped 30 days out — beyond the 7-day
|
||||
// traversal bound, so it is documented-excluded from the tree.
|
||||
await repo.InsertIfNotExistsAsync(NewEvent(siteId, occurredAtUtc: t0.AddDays(30), executionId: farDescendantExec, parentExecutionId: childExec));
|
||||
|
||||
// An unrelated execution far in time sharing no ancestry — never part of the chain
|
||||
// (asserts the window bound does not accidentally pull in strangers).
|
||||
await repo.InsertIfNotExistsAsync(NewEvent(siteId, occurredAtUtc: t0.AddDays(30), executionId: unrelatedExec));
|
||||
|
||||
var tree = await repo.GetExecutionTreeAsync(rootExec);
|
||||
var ids = tree.Select(n => n.ExecutionId).ToHashSet();
|
||||
|
||||
Assert.Contains(rootExec, ids);
|
||||
Assert.Contains(childExec, ids);
|
||||
Assert.DoesNotContain(farDescendantExec, ids);
|
||||
Assert.DoesNotContain(unrelatedExec, ids);
|
||||
Assert.Equal(2, tree.Count);
|
||||
}
|
||||
|
||||
[SkippableFact]
|
||||
public async Task GetExecutionTree_StubParentNode()
|
||||
{
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Metadata;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.ExternalSystems;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Notifications;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Sites;
|
||||
@@ -70,6 +73,27 @@ public class SchemaConfigurationTests : IDisposable
|
||||
|
||||
Assert.IsType<EncryptedStringConverter>(converter);
|
||||
}
|
||||
|
||||
// arch-review 04 (P1): the SiteCalls "live queue" KPI predicates all filter on
|
||||
// TerminalAtUtc IS NULL and run every 60 s (KPI recorder) + every 10 s (Health poll).
|
||||
// A filtered index over just the non-terminal rows turns each count from a clustered
|
||||
// scan of the 365-day archive into a small seek. Locked shape:
|
||||
// key = CreatedAtUtc, filter = [TerminalAtUtc] IS NULL, INCLUDE SourceSite/SourceNode/Status.
|
||||
[Fact]
|
||||
public void SiteCalls_Has_Filtered_NonTerminal_Index()
|
||||
{
|
||||
// IncludeProperties is a SQL-Server-only annotation stripped from the runtime
|
||||
// read-optimized model, so read the design-time model where it is preserved.
|
||||
var model = _context.GetService<IDesignTimeModel>().Model;
|
||||
var entity = model.FindEntityType(typeof(SiteCall))!;
|
||||
var index = entity.GetIndexes().Single(i => i.GetDatabaseName() == "IX_SiteCalls_NonTerminal");
|
||||
|
||||
Assert.Equal(new[] { nameof(SiteCall.CreatedAtUtc) }, index.Properties.Select(p => p.Name));
|
||||
Assert.Equal("[TerminalAtUtc] IS NULL", index.GetFilter());
|
||||
Assert.Equal(
|
||||
new[] { nameof(SiteCall.SourceSite), nameof(SiteCall.SourceNode), nameof(SiteCall.Status) },
|
||||
index.GetIncludeProperties());
|
||||
}
|
||||
}
|
||||
|
||||
public class SplitQueryBehaviourTests : IDisposable
|
||||
|
||||
@@ -86,11 +86,11 @@ public class SiteAuditPushFlowTests : TestKit
|
||||
AuditLogQueryFilter filter, AuditLogPaging paging, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, CancellationToken ct = default)
|
||||
public Task<long> SwitchOutPartitionAsync(DateTime monthBoundary, TimeSpan? commandTimeout = null, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<long> PurgeChannelOlderThanAsync(
|
||||
string channel, DateTime threshold, int batchSize, CancellationToken ct = default)
|
||||
string channel, DateTime threshold, int batchSize, TimeSpan? commandTimeout = null, CancellationToken ct = default)
|
||||
=> throw new NotSupportedException();
|
||||
|
||||
public Task<long> BackfillSourceNodeAsync(
|
||||
|
||||
Reference in New Issue
Block a user