From c4d7ec9b96f83a95975730f1192d959bd9b1cc48 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Mon, 9 Nov 2026 23:16:05 -0500 Subject: [PATCH] feat(audit): off-box chain-head export, verify-heads, maintenance login (ScadaBridge#107) Item 1 (owner decision: no signing). AuditLog:HashChain:HeadExport (default off): both centrals periodically read every month's chain head (LastSeq, LastHash, purge anchor, chained rows, tombstones; one snapshot under the chain lock SHARED) and append the changed heads to a sink the DB login cannot write - File (append-only JSONL per node per day, local or UNC) or Syslog (RFC 5424 over UDP / TCP octet counting). `audit verify-heads --path` reads those records back (file or syslog-collector lines) and the new VerifyAuditChainHeads command checks each against the database: Matches / Extended / Purged, or Mismatch / Regressed / MissingHead / MissingLink / CountMismatch (exit 3). AuditChainHeadStoreTests forges a chain with the production hasher: verify-chain says Intact, verify-heads says Mismatch. Item 3. AuditLog:MaintenanceConnectionString (${secret:}-capable, unset = today's single login) runs the chain job, chain verify / head export, the partition switch-out + archive and the per-channel purge on AuditMaintenanceContext (keyed audit-maintenance repository/archive, purge actor prefers them). AuditMaintenanceLoginTests runs the hardened setup on SQL Server: writer (datareader/datawriter/audit_writer) ingests and is refused chaining and deleting; the chainer+purger login does the full cycle. Two-login grants documented in Component-AuditLog.md. Claude-Session: https://claude.ai/code/session_01GT67AEHd2Acv1YPwGfXEoV --- CLAUDE.md | 2 +- alog.md | 7 +- docs/components/CLI.md | 5 +- docs/requirements/Component-AuditLog.md | 107 ++++-- docs/requirements/Component-CLI.md | 15 +- .../Component-ManagementService.md | 1 + docs/requirements/HighLevelReqs.md | 2 +- .../Central/AuditLogHashChainOptions.cs | 35 ++ .../Central/AuditLogPurgeActor.cs | 13 +- .../HeadExport/AuditChainHeadExportOptions.cs | 87 +++++ .../HeadExport/AuditChainHeadExportService.cs | 197 +++++++++++ .../Central/HeadExport/AuditChainHeadJson.cs | 21 ++ .../HeadExport/FileAuditChainHeadSink.cs | 79 +++++ .../Central/HeadExport/IAuditChainHeadSink.cs | 16 + .../HeadExport/SyslogAuditChainHeadSink.cs | 108 ++++++ .../ServiceCollectionExtensions.cs | 26 ++ .../Commands/AuditCommands.cs | 55 ++- .../Commands/AuditVerifyHeadsHelpers.cs | 160 +++++++++ src/ZB.MOM.WW.ScadaBridge.CLI/README.md | 35 ++ .../Repositories/IAuditChainHeadStore.cs | 28 ++ .../Messages/Management/AuditLogCommands.cs | 10 + .../Types/Audit/AuditChainHeads.cs | 126 +++++++ .../Maintenance/AuditChainHeadComparer.cs | 139 ++++++++ .../Maintenance/AuditLogHashChain.cs | 147 ++++++++- .../Maintenance/AuditMaintenanceContext.cs | 82 +++++ .../ServiceCollectionExtensions.cs | 40 ++- src/ZB.MOM.WW.ScadaBridge.Host/Program.cs | 7 +- .../ManagementActor.cs | 45 +++ .../ManagementEndpoints.cs | 3 + .../Central/AuditLogPurgeActorTests.cs | 30 ++ .../AuditChainHeadExportServiceTests.cs | 164 +++++++++ .../HeadExport/AuditChainHeadSinkTests.cs | 135 ++++++++ .../Commands/AuditCommandsScaffoldTests.cs | 8 +- .../Commands/AuditVerifyHeadsCommandTests.cs | 158 +++++++++ .../ManagementSurfaceParityTests.cs | 14 +- .../AuditChainHeadComparerTests.cs | 81 +++++ .../Maintenance/AuditChainHeadStoreTests.cs | 312 ++++++++++++++++++ .../Maintenance/AuditMaintenanceLoginTests.cs | 200 +++++++++++ .../RequiredRoleMatrixTests.cs | 2 + .../VerifyAuditChainHeadsCommandTests.cs | 110 ++++++ 40 files changed, 2769 insertions(+), 43 deletions(-) create mode 100644 src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/AuditChainHeadExportOptions.cs create mode 100644 src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/AuditChainHeadExportService.cs create mode 100644 src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/AuditChainHeadJson.cs create mode 100644 src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/FileAuditChainHeadSink.cs create mode 100644 src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/IAuditChainHeadSink.cs create mode 100644 src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/SyslogAuditChainHeadSink.cs create mode 100644 src/ZB.MOM.WW.ScadaBridge.CLI/Commands/AuditVerifyHeadsHelpers.cs create mode 100644 src/ZB.MOM.WW.ScadaBridge.Commons/Interfaces/Repositories/IAuditChainHeadStore.cs create mode 100644 src/ZB.MOM.WW.ScadaBridge.Commons/Types/Audit/AuditChainHeads.cs create mode 100644 src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditChainHeadComparer.cs create mode 100644 src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditMaintenanceContext.cs create mode 100644 tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/HeadExport/AuditChainHeadExportServiceTests.cs create mode 100644 tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/HeadExport/AuditChainHeadSinkTests.cs create mode 100644 tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/Commands/AuditVerifyHeadsCommandTests.cs create mode 100644 tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditChainHeadComparerTests.cs create mode 100644 tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditChainHeadStoreTests.cs create mode 100644 tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditMaintenanceLoginTests.cs create mode 100644 tests/ZB.MOM.WW.ScadaBridge.ManagementService.Tests/VerifyAuditChainHeadsCommandTests.cs diff --git a/CLAUDE.md b/CLAUDE.md index 6910bd68..1d0dcaad 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -184,7 +184,7 @@ spec for each is `docs/requirements/Component-.md`, and `README.md` carrie - Append-only enforced via DB roles (writer role has INSERT only, no UPDATE/DELETE). - **Clustered key `(OccurredAtUtc, EventId)` + aligned unique `UX_AuditLog_EventId_Occurred (EventId, OccurredAtUtc)` (migration `FlipAuditLogClusteredKey`, #111, B2 of the #78 benchmark).** The Audit Log page's newest-first page/keyset now reads an ordered range instead of the whole table (300k rows: 90,532 → 34 reads). Offline whole-table rebuild — script `docs/plans/sql/FlipAuditLogClusteredKey.sql` (+ `.rollback.sql`), both centrals stopped: the partition-switch staging table now reads the live key order, but binaries older than the flip hard-code the old one. DEV/QA not applied (sizing + window in #111). - **Parquet cold archive (T2, ScadaBridge#42) — default OFF (`AuditLog:Archive:Enabled` + absolute `Directory`, shared by both centrals).** The purge actor writes each expired partition to `{Directory}/{YYYY}/auditlog-{YYYY-MM}-{stamp}Z.parquet` (10 stored columns + `ChainSeq`/`RowHash`, tick-exact `OccurredAtUtcTicks`) plus `.tombstones.parquet` and a `.manifest.json` (written last; chain head at archive time, SHA-256 per file), read under the chain lock SHARED; then the switch runs with a guard (under the EXCLUSIVE lock: partition row count + each month's `LastSeq` must still equal the archive's, else THROW 51002/51003 and nothing is purged). Fail closed: a partition that cannot be archived is not purged. `audit archive list|verify` = `ListAuditArchives`/`VerifyAuditArchive` (verify-chain's gate): file hashes, chain recomputed from the archive, chain end vs the DB purge anchor (`Matches`/`Superseded`/`NotPurged`/`Mismatch`), CLI exit 3. `audit export --format parquet` streams the 24 export columns (was 501). No migration; no re-import path (open decision), object storage not wired (mount it). -- **Hash-chain tamper evidence (T1, ScadaBridge#41) — default OFF (`AuditLog:HashChain:Enabled`).** One SHA-256 chain per UTC calendar month: `ChainSeq` + `RowHash = SHA-256(prev || canonical(row))` on `AuditLog` (migration `AddAuditLogHashChain`), linked by the central `AuditLogHashChainService` (both nodes, serialised by `sp_getapplock`) — never by the ingest path. `AuditLogChainHead` catches tail deletion and carries the partition-switch checkpoint anchor; the per-channel DELETE writes an `AuditLogChainTombstone` per chained row in the same transaction. `audit verify-chain --month` = management command `VerifyAuditChain` (OperationalAudit, unscoped callers only), first break with partition/ChainSeq/EventId, CLI exit 3. A chained row's `SourceNode` is frozen (backfill skips it). Unsigned; signing/external anchoring are open. +- **Hash-chain tamper evidence (T1, ScadaBridge#41) — default OFF (`AuditLog:HashChain:Enabled`).** One SHA-256 chain per UTC calendar month: `ChainSeq` + `RowHash = SHA-256(prev || canonical(row))` on `AuditLog` (migration `AddAuditLogHashChain`), linked by the central `AuditLogHashChainService` (both nodes, serialised by `sp_getapplock`) — never by the ingest path. `AuditLogChainHead` catches tail deletion and carries the partition-switch checkpoint anchor; the per-channel DELETE writes an `AuditLogChainTombstone` per chained row in the same transaction. `audit verify-chain --month` = management command `VerifyAuditChain` (OperationalAudit, unscoped callers only), first break with partition/ChainSeq/EventId, CLI exit 3. A chained row's `SourceNode` is frozen (backfill skips it). **Unsigned by owner decision (#107); external anchor = off-box head export** (`AuditLog:HashChain:HeadExport`, default off: both centrals append changed per-month heads — month, `LastSeq`/`LastHash`, purge anchor, row + tombstone counts — to append-only JSONL files `{Directory}/{yyyy}/audit-chain-heads-{date}-{node}.jsonl` or RFC 5424 syslog UDP/TCP); `audit verify-heads --path` = `VerifyAuditChainHeads` checks them against the DB (a re-hashed forgery passes verify-chain, fails verify-heads; exit 3). **Two-login hardening (#107 item 3):** `AuditLog:MaintenanceConnectionString` (`${secret:}`-capable, unset = runtime login) runs the chain job, verify/head export, partition switch-out + archive and per-channel purge on `AuditMaintenanceContext` (keyed `audit-maintenance` repository/archive); that login gets `scadabridge_audit_chainer` + `scadabridge_audit_purger`, the runtime login `scadabridge_audit_writer` (whose DENYs block chaining/purging) — pinned by `AuditMaintenanceLoginTests`. - Node-of-origin is captured alongside site-of-origin: `SourceNode` (`varchar(64)` NULL) on `AuditLog`, `Notifications`, and `SiteCalls` — `node-a`/`node-b` for site rows (qualified by `SourceSiteId`/`SourceSite`), `central-a`/`central-b` for central direct-write rows. Stamped at the writing node, carried verbatim through telemetry + reconciliation, and indexed via `IX_AuditLog_Node_Occurred (SourceNode, OccurredAtUtc)` on `AuditLog`. - Per-node stuck KPIs (M5.3 T6): Notification Outbox and Site Call Audit expose `PerNodeNotificationKpiRequest`/`PerNodeSiteCallKpiRequest` messages that group stuck/parked/delivered counts by `SourceNode`, surfacing per-node breakdowns on the Health dashboard. - `audit tree --execution-id ` CLI command (M5.3 T8) + `GET /api/audit/tree` endpoint — resolves any node to its chain root and renders the full execution tree; backed by `IAuditLogRepository.GetExecutionTreeAsync`. diff --git a/alog.md b/alog.md index 1e265223..f0d19ee1 100644 --- a/alog.md +++ b/alog.md @@ -501,7 +501,11 @@ See §8.2. Contract: unredacted secrets never persist. Safety net over-redacts o - Purges keep the chain verifiable: the partition switch records a checkpoint anchor on the month's chain head; the per-channel DELETE leaves a tombstone per chained row. - Verifiable via `scadabridge audit verify-chain --month YYYY-MM` (first break reported - with partition, `ChainSeq`, `EventId`). Unsigned; signing / external anchoring are open. + with partition, `ChainSeq`, `EventId`). Unsigned (owner decision, ScadaBridge#107): instead + the per-month heads are exported off-box (`AuditLog:HashChain:HeadExport`, append-only files + or syslog, default off) and `scadabridge audit verify-heads` checks them against the database. + The chain job and both purge paths can run on a separate maintenance login + (`AuditLog:MaintenanceConnectionString`). - Default **off** (`AuditLog:HashChain:Enabled`). Full design: Component-AuditLog.md, Security & Tamper-Evidence. @@ -598,6 +602,7 @@ New `scadabridge audit` command group: - `scadabridge audit query --site --since --kind [...]` — same filter set as the UI. - `scadabridge audit export --since --until --format csv|jsonl|parquet --output ` — bulk export, server-side streaming. - `scadabridge audit verify-chain --month ` — hash-chain verification (when §11.4 is enabled). +- `scadabridge audit verify-heads --path ` — off-box exported chain heads vs the database (ScadaBridge#107). Requires the same **OperationalAudit** / **AuditExport** permissions as the UI. diff --git a/docs/components/CLI.md b/docs/components/CLI.md index 40ebac58..18e2e870 100644 --- a/docs/components/CLI.md +++ b/docs/components/CLI.md @@ -23,7 +23,8 @@ Direct-to-node access (`http://localhost:9001` for `central-a`, `http://localhos Management commands use `POST /management`. Audit query, export, and tree use `GET /api/audit/query`, `/api/audit/export`, and `/api/audit/tree`; source-node backfill uses `POST /api/audit/backfill-source-node`. `audit verify-chain` sends the -`VerifyAuditChain` management command (ScadaBridge#41); `audit archive list|verify` send +`VerifyAuditChain` management command (ScadaBridge#41); `audit verify-heads` reads off-box +exported chain heads and sends `VerifyAuditChainHeads` (ScadaBridge#107); `audit archive list|verify` send `ListAuditArchives` / `VerifyAuditArchive` (ScadaBridge#42). `debug stream` uses a persistent SignalR connection. Audit export streams the response without buffering it all. @@ -121,7 +122,7 @@ scadabridge instance deploy --id 7 | `db-connection` | `list`, `get`, `create`, `update`, `delete` | `Designer` | | `api-method` | `list`, `get`, `create`, `update`, `delete`, `validate`, `recompile` | `Designer` | | `bundle` | `export`, `preview`, `import` | Designer export; Administrator preview/import | -| `audit` | `query`, `export`, `tree`, `backfill-source-node`, `verify-chain`, `archive list`, `archive verify` | Audit roles; Administrator backfill; verify-chain and archive need an unscoped audit role | +| `audit` | `query`, `export`, `tree`, `backfill-source-node`, `verify-chain`, `verify-heads`, `archive list`, `archive verify` | Audit roles; Administrator backfill; verify-chain, verify-heads and archive need an unscoped audit role | | `audit-config` | `query` (config-change audit trail; was `audit-log` pre-M8) | `Admin` | | `health` | `summary`, `pending-updates`, `site`, `event-log`, `parked-messages` | Authenticated reads with server-side scope checks | | `debug` | `snapshot`, `stream` | `Deployer` | diff --git a/docs/requirements/Component-AuditLog.md b/docs/requirements/Component-AuditLog.md index 87b55ece..cab7e42e 100644 --- a/docs/requirements/Component-AuditLog.md +++ b/docs/requirements/Component-AuditLog.md @@ -833,9 +833,9 @@ MS SQL for direct-write events). Unredacted secrets never persist. ## Security & Tamper-Evidence - **Append-only enforcement — what actually holds in the default deployment.** - ScadaBridge runs with **one** connection principal for both the writer and the - maintenance/purge paths (the runtime does not open a second, lower-privilege - connection for purge). So in the default deployment the append-only invariant is + By default ScadaBridge runs with **one** connection principal for both the writer and the + maintenance/purge paths. (A second, maintenance login is supported but opt-in — see + *Two-login hardening* below.) So in the default deployment the append-only invariant is enforced by two application-level controls, not by database permissions: 1. **CI grep guard** — `AuditLogAppendOnlyGuardTests` scans the ConfigurationDatabase source for any `UPDATE`/`DELETE` DML targeting `dbo.AuditLog` and fails the build @@ -846,20 +846,41 @@ MS SQL for direct-write events). Unredacted secrets never persist. other UPDATE/DELETE trips the guard. 2. **Code review** — the marker is deliberately specific so an unrelated mutation cannot inherit the exemption without a reviewer noticing. -- **Optional DBA hardening (two DB roles).** For a deployment that wants - database-level enforcement on top of the application controls, migrations provision - two roles: `scadabridge_audit_writer` (`INSERT` + `SELECT` only; `DENY UPDATE` + - `DENY DELETE` so a later `db_datawriter` membership cannot silently re-enable - mutation) and `scadabridge_audit_purger` (the maintenance principal). To use them, - a DBA provisions **two logins** — the runtime connection mapped to the writer role, - and a separate maintenance job/connection mapped to the purger role. The purger role - now carries the permissions its purge path genuinely needs: `SELECT` + `ALTER ON - SCHEMA::dbo` **plus** `CREATE TABLE` (the switch-out dance CREATEs a staging table) - and a scoped `DELETE ON dbo.AuditLog` (the per-channel retention override is a bounded - row DELETE) — granted by the `FixAuditPurgerRoleGrants` migration (arch-review 04, S3). - Without those two grants the purger role could not actually execute the switch-out or - the per-channel purge; the earlier claim that "row-level `DELETE` is not granted even - to purge" was stale once `PerChannelRetentionDays` shipped. +- **Two-login hardening (ScadaBridge#107 item 3) — opt-in.** Migrations provision three + roles: `scadabridge_audit_writer` (`INSERT` + `SELECT` on `AuditLog`; `DENY UPDATE` + + `DENY DELETE` so a later `db_datawriter` membership cannot silently re-enable mutation; + `DENY INSERT/UPDATE/DELETE` on `AuditLogChainHead` and `AuditLogChainTombstone`), + `scadabridge_audit_chainer` (`SELECT` on `AuditLog`, column-level `UPDATE (ChainSeq, RowHash)`, + `SELECT/INSERT/UPDATE` on the head, `SELECT` on tombstones) and `scadabridge_audit_purger` + (`SELECT` + `ALTER ON SCHEMA::dbo` + `CREATE TABLE` for the switch-out staging table, + `DELETE ON dbo.AuditLog` for the per-channel purge — `FixAuditPurgerRoleGrants`, arch-review 04 + S3 — and `SELECT/UPDATE` on the head, `SELECT/INSERT/DELETE` on tombstones). Because the writer + role's DENYs win over any grant, the chain job and the purges **cannot** run on a login that is + in the writer role. Set `AuditLog:MaintenanceConnectionString` (central nodes; `${secret:}` + references resolve, as for every pre-host key) and the hash-chain job, chain verification / + head export, the partition switch-out (with its Parquet archive) and the per-channel DELETE run + on their own connection with that string (`AuditMaintenanceContext`); unset, they run on + `ScadaBridge:Database:ConfigurationDb` exactly as before. A DBA provisions: + + | Login | Used by | Database roles | + |---|---|---| + | runtime (writer) — `ScadaBridge:Database:ConfigurationDb` | everything else, incl. audit ingest | `db_datareader`, `db_datawriter`, `scadabridge_audit_writer` (+ whatever the rest of the app needs) | + | maintenance — `AuditLog:MaintenanceConnectionString` | chain job, verify-chain / verify-heads, head export, partition switch-out + archive, per-channel purge | `scadabridge_audit_chainer`, `scadabridge_audit_purger` — and NOT `scadabridge_audit_writer` | + + ```sql + CREATE USER [svc_scadabridge_audit_maint] FOR LOGIN [svc_scadabridge_audit_maint]; + ALTER ROLE scadabridge_audit_chainer ADD MEMBER [svc_scadabridge_audit_maint]; + ALTER ROLE scadabridge_audit_purger ADD MEMBER [svc_scadabridge_audit_maint]; + ALTER ROLE scadabridge_audit_writer ADD MEMBER [svc_scadabridge_app]; -- the runtime login + ``` + + `AuditMaintenanceLoginTests` runs exactly this setup on SQL Server: the writer ingests and is + refused the chain UPDATE and the per-channel DELETE; the maintenance login chains, verifies, + reads heads, purges per channel, enumerates partitions, archives and switches out. Not moved + to the maintenance login: the monthly partition roll-forward (`ALTER PARTITION FUNCTION`, + needs `ALTER ANY DATASPACE` on the runtime login, or run it as a DBA job) and the one-time + `audit backfill-source-node` (needs `UPDATE` on `SourceNode` — a DBA-run step in a hardened + setup). Migrations themselves are applied by a DBA with the production SQL scripts. - **Authorization.** Reading the Audit Log requires the existing **Audit** role extended with a new **OperationalAudit** permission. Per-site row scoping reuses the existing site-permission model; bulk export requires an additional @@ -910,10 +931,29 @@ MS SQL for direct-write events). Unredacted secrets never persist. inside the hashed bytes), `MissingRow` (a deleted row with no tombstone), `DuplicateSequence`, `IncompleteLink`, or `HeadMismatch` (rows deleted from the end). CLI exit code 3 on `Broken`. - - **What it does NOT prove.** The chain is unsigned and anchored only inside the same - database: a principal that can rewrite rows AND recompute every later link AND the - head can produce a consistent forgery. Signing the head (key custody) and anchoring - it outside the database are open owner decisions (ScadaBridge#41). + - **External anchor: off-box head export (ScadaBridge#107 item 1) — default OFF.** Inside the + database alone, a principal that can rewrite rows AND recompute every later link AND the + head produces a chain `verify-chain` accepts. Owner decision: **no signing**; instead + `AuditLog:HashChain:HeadExport:Enabled` makes both central nodes, every `IntervalMinutes` + (60), read every month's head (`LastSeq`, `LastHash`, the purge anchor, live chained rows, + tombstones — one snapshot under the chain lock SHARED) and append the heads that changed + since that node's last export (all of them after a restart; `ExportUnchanged` for every + tick) to a sink the database login cannot write: `Sink: File` — append-only JSON lines in + `{File:Directory}/{yyyy}/audit-chain-heads-{yyyy-MM-dd}-{node}.jsonl` (local or UNC; opened + `FileMode.Append` only, one file per node per day; give the service account create + + append-data rights and nothing else), or `Sink: Syslog` — RFC 5424 (`1 … chainhead - + {json}`, facility 13 "log audit", severity Notice) over UDP (RFC 5426) or TCP (RFC 6587 + octet counting). A failed write is logged and retried with the same heads next tick. + `scadabridge audit verify-heads --path ` reads those records back (file-sink + lines or a syslog collector's log) and the `VerifyAuditChainHeads` command checks each + against the database: the link stored at the exported sequence (the head, a live row's + `RowHash`, a tombstone, or the purge anchor) must still carry the exported hash — + `Matches` / `Extended` / `Purged` (removed by a partition switch; verify the archive), or + the failures `Mismatch` (rewritten chain), `Regressed` (head rolled back), `MissingHead`, + `MissingLink` (chained row deleted outside the purge paths), `CountMismatch` (same head, + rows + tombstones changed). `AuditChainHeadStoreTests` forges a chain with the production + hasher: `verify-chain` reports it Intact, `verify-heads` reports Mismatch. Coverage is from + the first export on: a forgery of links never exported is not detectable this way. - **Roles (DBA hardening).** `AddAuditLogHashChain` adds `scadabridge_audit_chainer` (SELECT on `AuditLog`, column-level `UPDATE (ChainSeq, RowHash)`, read/write on the head), grants the purger the head/tombstone rights its purge paths need, and DENYs @@ -987,6 +1027,31 @@ the global window; `PerChannelRetentionDays` specifies per-channel windows that are strictly shorter — any channel whose override equals or exceeds the global value is silently ignored (the global partition switch-out already governs it). +The nested `AuditLog:HashChain` section (ScadaBridge#41) runs the chaining job; its +`HeadExport` sub-section (ScadaBridge#107) exports the chain heads off-box, and the root +`AuditLog:MaintenanceConnectionString` moves the maintenance paths to a second login (see +Security & Tamper-Evidence): + +```jsonc +"AuditLog": { + "MaintenanceConnectionString": "${secret:SB-AUDIT-MAINT-DB}", // optional; unset = runtime connection + "HashChain": { + "Enabled": false, // default off + "IntervalSeconds": 60, + "BatchSize": 1000, + "MaxBatchesPerTick": 100, + "HeadExport": { + "Enabled": false, // default off + "IntervalMinutes": 60, + "ExportUnchanged": false, // true = every head every tick + "Sink": "File", // File | Syslog + "File": { "Directory": "\\\\fileserver\\audit-heads" }, // absolute, local or UNC + "Syslog": { "Host": "syslog.example", "Port": 514, "Protocol": "Udp", "Facility": 13, "AppName": "scadabridge", "TimeoutSeconds": 10 } + } + } +} +``` + The nested `AuditLog:Archive` section (ScadaBridge#42) turns on the Parquet cold archive (see Retention & Purge): diff --git a/docs/requirements/Component-CLI.md b/docs/requirements/Component-CLI.md index 53a525c3..f91685fb 100644 --- a/docs/requirements/Component-CLI.md +++ b/docs/requirements/Component-CLI.md @@ -304,7 +304,7 @@ The `scadabridge audit` group targets the centralized Audit Log component (#23) exposes the UI-equivalent operational audit surface. Permissions follow the same read-vs-export split the Central UI uses (see Component-AuditLog.md, Security & Tamper-Evidence, and Security & Auth #10): `audit query`, `audit tree`, and -`audit verify-chain` and `audit archive list|verify` require the `OperationalAudit` +`audit verify-chain`, `audit verify-heads` and `audit archive list|verify` require the `OperationalAudit` permission; `audit export` additionally requires `AuditExport`; `audit backfill-source-node` requires the `Admin` role (maintenance path only). The server enforces permission checks and @@ -316,6 +316,7 @@ scadabridge audit export --since --until --format csv|jsonl|parquet --ou scadabridge audit tree --execution-id [--format table|json] scadabridge audit backfill-source-node --before [--sentinel ] [--batch ] scadabridge audit verify-chain --month +scadabridge audit verify-heads --path scadabridge audit archive list scadabridge audit archive verify --name ``` @@ -350,6 +351,18 @@ scadabridge audit archive verify --name `0` intact or not chained, `3` broken, `1` error, `2` authorization failure. Requires `OperationalAudit` and a caller not scoped to specific sites. See Component-AuditLog.md, Security & Tamper-Evidence. +- `audit verify-heads --path ` (ScadaBridge#107) — reads the AuditLog + chain heads exported off-box by `AuditLog:HashChain:HeadExport` (JSON lines from the file + sink, or any log a syslog collector wrote — each line's JSON object is taken from its first + `{` to its last `}`; a directory is read recursively for `*.jsonl`/`*.log`/`*.txt`), + collapses duplicates and sends them in the `VerifyAuditChainHeads` management command. The + server checks each exported head against the database chain — `Matches`, `Extended` (chain + grew; the link at the exported sequence still carries the exported hash), `Purged` (removed + by a partition purge; verify the archive), or the failures `Mismatch` (rewritten chain), + `Regressed`, `MissingHead`, `MissingLink`, `CountMismatch`. Exit `0` verified, `3` a + failure, `1` error (no records found, bad input, `CHAIN_BUSY`), `2` authorization failure. + Same gate as `verify-chain`. Run it together with `verify-chain`: a forger who rewrites rows + and re-hashes the chain inside SQL Server passes `verify-chain` but not `verify-heads`. - `audit archive list` / `audit archive verify --name ` (ScadaBridge#42) — send the `ListAuditArchives` / `VerifyAuditArchive` management commands for the AuditLog Parquet cold archives the retention purge writes (`AuditLog:Archive`, diff --git a/docs/requirements/Component-ManagementService.md b/docs/requirements/Component-ManagementService.md index 781f7ead..380f486e 100644 --- a/docs/requirements/Component-ManagementService.md +++ b/docs/requirements/Component-ManagementService.md @@ -200,6 +200,7 @@ The two-person authorization workflow for writes through the MxAccess Gateway. B - **QueryAuditLog**: Legacy configuration-change audit query (filtered by entity type, user, date range, etc.) routed through `/management`. Gated to the `Admin` role; superseded for the centralized Audit Log component (#23) by the dedicated `/api/audit/*` REST endpoints described below. - **VerifyAuditChain** (ScadaBridge#41): verifies one UTC calendar month (`Month`, `YYYY-MM`) of the central AuditLog hash chain and returns an `AuditChainVerificationResult` (status, rows verified, tombstones applied, purge anchor, head sequence, unchained rows, first break with partition / `ChainSeq` / `EventId` / kind). Gated on the OperationalAudit roles (`Administrator`, `Viewer`); a site-scoped non-Administrator caller is refused (`ManagementUnauthorized`) because a month's chain spans every site. Long-running Ask timeout. Errors: `INVALID_ARGUMENT` (bad month), `CHAIN_BUSY` (a chaining/purge pass held the chain lock past 60 s). Backs `scadabridge audit verify-chain`. +- **VerifyAuditChainHeads** (ScadaBridge#107): checks chain heads exported off-box (`Records`, at most 50,000 `AuditChainHeadRecord`s; duplicates collapsed) against the database chain and returns an `AuditChainHeadVerificationResult` (overall `Verified`, one `AuditChainHeadCheck` per distinct exported head: `Matches` / `Extended` / `Purged`, or the failures `Mismatch` / `Regressed` / `MissingHead` / `MissingLink` / `CountMismatch`). Same gate as VerifyAuditChain; long-running Ask timeout. Errors: `INVALID_ARGUMENT` (no records / over the cap), `CHAIN_BUSY`. Backs `scadabridge audit verify-heads`. - **ListAuditArchives** / **VerifyAuditArchive** (ScadaBridge#42): list the AuditLog Parquet cold archives under `AuditLog:Archive:Directory` (`AuditArchiveListResult`: enabled flag + every manifest), and verify one by `Name` (`AuditArchiveVerificationResult`: file integrity, per-month chain recomputed from the archived rows, anchor `Matches`/`Superseded`/`NotPurged`/`Mismatch`, overall `Verified`). Same gate as VerifyAuditChain (OperationalAudit roles, site-scoped non-Administrator refused). VerifyAuditArchive uses the long-running Ask timeout. Errors: `ARCHIVE_NOT_CONFIGURED`, `INVALID_ARGUMENT` (not an archive name), `NOT_FOUND`. Back `scadabridge audit archive list|verify`. ### Transport (Bundle Import / Export) diff --git a/docs/requirements/HighLevelReqs.md b/docs/requirements/HighLevelReqs.md index e09b05f5..e1cd1aab 100644 --- a/docs/requirements/HighLevelReqs.md +++ b/docs/requirements/HighLevelReqs.md @@ -480,7 +480,7 @@ Sections 10.1–10.4 cover **configuration-database audit** (config-mutating use - **AL-9**: The site SQLite Audit Log is purged only when `ForwardState ∈ {Forwarded, Reconciled}` — i.e., a row must be either confirmed-forwarded *or* confirmed-reconciled before it can be removed. A central outage therefore **cannot cause audit loss at sites**. - **AL-10**: The Central UI exposes an **Audit Log page** with a cross-channel filter (by site, target, status, time range, correlation ID), plus **drill-ins from existing operational pages** (Site Calls, Notification Outbox, Inbound API). - **AL-11**: Append-only semantics are **enforced via DB roles** (no UPDATE/DELETE granted on the `AuditLog` table to application accounts); a **tamper-evidence hash chain** (per-month SHA-256 chain, default off) shipped later by ScadaBridge#41. -- **AL-12**: The CLI provides a `scadabridge audit` command group for query, export (csv, jsonl, parquet), hash-chain verification (verify-chain reports the first break of a month's hash chain), and listing/verifying the Parquet cold archives (ScadaBridge#42) against the central Audit Log. +- **AL-12**: The CLI provides a `scadabridge audit` command group for query, export (csv, jsonl, parquet), hash-chain verification (verify-chain reports the first break of a month's hash chain; verify-heads checks off-box exported chain heads against the database, ScadaBridge#107), and listing/verifying the Parquet cold archives (ScadaBridge#42) against the central Audit Log. ## 11. Health Monitoring diff --git a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogHashChainOptions.cs b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogHashChainOptions.cs index db2ff85e..4deb3734 100644 --- a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogHashChainOptions.cs +++ b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogHashChainOptions.cs @@ -1,4 +1,5 @@ using ZB.MOM.WW.Configuration; +using ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport; namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central; @@ -33,6 +34,12 @@ public sealed class AuditLogHashChainOptions /// Upper bound on batches per tick so a backlog yields between ticks (default 100). public int MaxBatchesPerTick { get; set; } = 100; + + /// + /// Off-box export of the per-month chain heads (AuditLog:HashChain:HeadExport, + /// ScadaBridge#107 item 1) — the chain's external anchor. Default off. + /// + public AuditChainHeadExportOptions HeadExport { get; set; } = new(); } /// Validates at host startup. @@ -47,5 +54,33 @@ public sealed class AuditLogHashChainOptionsValidator : OptionsValidatorBase 0, $"AuditLog:HashChain:{nameof(AuditLogHashChainOptions.MaxBatchesPerTick)} ({options.MaxBatchesPerTick}) must be > 0."); + + var export = options.HeadExport ?? new AuditChainHeadExportOptions(); + if (!export.Enabled) + { + return; + } + + const string section = "AuditLog:HashChain:HeadExport"; + builder.RequireThat(export.IntervalMinutes > 0, + $"{section}:IntervalMinutes ({export.IntervalMinutes}) must be > 0."); + if (export.Sink == AuditChainHeadSinkKind.File) + { + var dir = export.File?.Directory; + builder.RequireThat(!string.IsNullOrWhiteSpace(dir) && Path.IsPathFullyQualified(dir), + $"{section}:File:Directory ('{dir}') must be an absolute path (local or UNC) when Sink=File."); + } + else + { + var syslog = export.Syslog ?? new AuditChainHeadSyslogSinkOptions(); + builder.RequireThat(!string.IsNullOrWhiteSpace(syslog.Host), + $"{section}:Syslog:Host is required when Sink=Syslog."); + builder.RequireThat(syslog.Port is > 0 and <= 65535, + $"{section}:Syslog:Port ({syslog.Port}) must be between 1 and 65535."); + builder.RequireThat(syslog.Facility is >= 0 and <= 23, + $"{section}:Syslog:Facility ({syslog.Facility}) must be between 0 and 23."); + builder.RequireThat(syslog.TimeoutSeconds > 0, + $"{section}:Syslog:TimeoutSeconds ({syslog.TimeoutSeconds}) must be > 0."); + } } } diff --git a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogPurgeActor.cs b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogPurgeActor.cs index ef1fc6f2..33240372 100644 --- a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogPurgeActor.cs +++ b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/AuditLogPurgeActor.cs @@ -48,7 +48,9 @@ namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central; /// /// /// DI scopes. is a scoped EF Core -/// service registered by AddConfigurationDatabase. The singleton +/// service registered by AddConfigurationDatabase — the +/// -keyed one, on the AuditLog maintenance +/// connection (ScadaBridge#107 item 3), when registered. The singleton /// opens one DI scope per tick and reuses the same repository across every /// boundary in that tick — mirrors the /// pattern. @@ -149,7 +151,11 @@ public class AuditLogPurgeActor : ReceiveActor IAuditLogRepository repository; try { - repository = scope.ServiceProvider.GetRequiredService(); + // The maintenance-connection repository when the host registered one + // (AddConfigurationDatabase, ScadaBridge#107 item 3 — on the separate maintenance + // login if AuditLog:MaintenanceConnectionString is set), else the plain one. + repository = scope.ServiceProvider.GetKeyedService(AuditMaintenanceContext.ServiceKey) + ?? scope.ServiceProvider.GetRequiredService(); } catch (Exception ex) { @@ -202,7 +208,8 @@ public class AuditLogPurgeActor : ReceiveActor { try { - archive = scope.ServiceProvider.GetRequiredService(); + archive = scope.ServiceProvider.GetKeyedService(AuditMaintenanceContext.ServiceKey) + ?? scope.ServiceProvider.GetRequiredService(); } catch (Exception ex) { diff --git a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/AuditChainHeadExportOptions.cs b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/AuditChainHeadExportOptions.cs new file mode 100644 index 00000000..cf00b8d5 --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/AuditChainHeadExportOptions.cs @@ -0,0 +1,87 @@ +namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport; + +/// Where exported chain heads go. +public enum AuditChainHeadSinkKind +{ + /// Append-only JSON-lines files in a directory (local or UNC). + File, + + /// RFC 5424 syslog over UDP or TCP. + Syslog, +} + +/// Syslog transport. +public enum AuditSyslogProtocol +{ + /// One datagram per record (RFC 5426). + Udp, + + /// Octet-counted frames on one TCP connection per export (RFC 6587). + Tcp, +} + +/// +/// Off-box export of the AuditLog hash-chain heads (section AuditLog:HashChain:HeadExport, +/// ScadaBridge#107 item 1). Default OFF. Every both central +/// nodes read each month's chain head (month, last sequence + hash, partition-purge anchor, +/// chained-row and tombstone counts) and append the heads that changed since their previous +/// export to a sink the database login cannot write. audit verify-heads later compares +/// those records with the database: a chain someone rewrote and re-hashed inside SQL Server no +/// longer carries the hash that was exported. +/// +public sealed class AuditChainHeadExportOptions +{ + /// Whether the export runs (default false). + public bool Enabled { get; set; } + + /// Minutes between exports (default 60). + public int IntervalMinutes { get; set; } = 60; + + /// + /// Re-export unchanged heads every tick too (default false: only heads whose sequence, + /// anchor or counts moved since this node's previous export, plus every head on the first + /// tick after start). + /// + public bool ExportUnchanged { get; set; } + + /// Which sink (default ). + public AuditChainHeadSinkKind Sink { get; set; } = AuditChainHeadSinkKind.File; + + /// File sink settings. + public AuditChainHeadFileSinkOptions File { get; set; } = new(); + + /// Syslog sink settings. + public AuditChainHeadSyslogSinkOptions Syslog { get; set; } = new(); +} + +/// File sink: {Directory}/{yyyy}/audit-chain-heads-{yyyy-MM-dd}-{node}.jsonl, opened for append only. +public sealed class AuditChainHeadFileSinkOptions +{ + /// + /// Absolute directory (local path or UNC share). Give the service account create + append + /// rights only; the SQL Server login has no access to it by construction. + /// + public string? Directory { get; set; } +} + +/// Syslog sink (RFC 5424). +public sealed class AuditChainHeadSyslogSinkOptions +{ + /// Collector host name or address. + public string? Host { get; set; } + + /// Collector port (default 514). + public int Port { get; set; } = 514; + + /// Transport (default UDP). + public AuditSyslogProtocol Protocol { get; set; } = AuditSyslogProtocol.Udp; + + /// Syslog facility 0–23 (default 13, "log audit"). + public int Facility { get; set; } = 13; + + /// APP-NAME field (default scadabridge). + public string AppName { get; set; } = "scadabridge"; + + /// Send/connect timeout in seconds (default 10). + public int TimeoutSeconds { get; set; } = 10; +} diff --git a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/AuditChainHeadExportService.cs b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/AuditChainHeadExportService.cs new file mode 100644 index 00000000..9a261f34 --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/AuditChainHeadExportService.cs @@ -0,0 +1,197 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; +using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance; + +namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport; + +/// +/// Central hosted service that exports the AuditLog hash-chain heads off-box +/// (AuditLog:HashChain:HeadExport, ScadaBridge#107 item 1). Default off. +/// +/// +/// +/// Runs on both central nodes (each record names its node). A tick reads every month's head in +/// one consistent snapshot (, chain lock +/// SHARED) and writes the heads that changed since this node's last successful export — all of +/// them on the first tick after start — to the configured . +/// A failed write is logged and retried on the next tick with the same heads (the in-memory +/// "last exported" state only advances on success). No signing: the protection is that the +/// sink sits outside the database login's reach. +/// +/// +public sealed class AuditChainHeadExportService : IHostedService, IDisposable +{ + private readonly IServiceScopeFactory _scopeFactory; + private readonly AuditChainHeadExportOptions _options; + private readonly IAuditChainHeadSink? _sink; + private readonly string? _node; + private readonly TimeProvider _time; + private readonly ILogger _logger; + private readonly Dictionary _lastExported = new(); + private CancellationTokenSource? _cts; + private Task? _loop; + + /// Initializes the service. + /// Scope factory for the scoped head store. + /// Hash-chain options (the HeadExport sub-section is used). + /// The sink (null when the export is disabled). + /// This node's name for the records. + /// Logger. + /// Clock. + public AuditChainHeadExportService( + IServiceScopeFactory scopeFactory, + IOptions options, + IAuditChainHeadSink? sink, + string? nodeName, + ILogger logger, + TimeProvider? time = null) + { + _scopeFactory = scopeFactory ?? throw new ArgumentNullException(nameof(scopeFactory)); + _options = (options ?? throw new ArgumentNullException(nameof(options))).Value.HeadExport; + _sink = sink; + _node = nodeName; + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _time = time ?? TimeProvider.System; + } + + /// + public Task StartAsync(CancellationToken cancellationToken) + { + if (!_options.Enabled || _sink is null) + { + _logger.LogInformation("AuditLog chain-head export disabled (AuditLog:HashChain:HeadExport:Enabled=false)."); + return Task.CompletedTask; + } + + _logger.LogInformation("AuditLog chain-head export every {Minutes} min to {Sink}.", _options.IntervalMinutes, _sink.Describe); + var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + _cts = cts; + var token = cts.Token; + _loop = Task.Run(() => RunLoopAsync(token), CancellationToken.None); + return Task.CompletedTask; + } + + private async Task RunLoopAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + await TickAsync(ct).ConfigureAwait(false); + try + { + await Task.Delay(TimeSpan.FromMinutes(_options.IntervalMinutes), ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + break; + } + } + } + + /// Runs one export tick. + /// Cancellation token. + /// The number of records written (0 on failure or nothing changed). + internal async Task TickAsync(CancellationToken ct) + { + if (_sink is null) + { + return 0; + } + + try + { + IReadOnlyList heads; + await using (var scope = _scopeFactory.CreateAsyncScope()) + { + heads = await scope.ServiceProvider.GetRequiredService() + .ReadHeadsAsync(ct).ConfigureAwait(false); + } + + var now = _time.GetUtcNow().UtcDateTime; + var changed = heads + .Where(h => _options.ExportUnchanged + || !_lastExported.TryGetValue(h.Month, out var last) + || last != (h.LastSeq, h.PurgedThroughSeq, h.ChainedRows + h.Tombstones)) + .ToList(); + if (changed.Count == 0) + { + return 0; + } + + var records = changed.Select(h => ToRecord(h, now, _node)).ToList(); + await _sink.WriteAsync(records, ct).ConfigureAwait(false); + foreach (var h in changed) + { + _lastExported[h.Month] = (h.LastSeq, h.PurgedThroughSeq, h.ChainedRows + h.Tombstones); + } + + _logger.LogDebug("Exported {Count} AuditLog chain head(s) to {Sink}.", records.Count, _sink.Describe); + return records.Count; + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + return 0; + } + catch (Exception ex) + { + // Contained: the next tick retries the same heads (nothing was marked exported). + _logger.LogError(ex, "AuditLog chain-head export to {Sink} failed; retrying next tick.", _sink.Describe); + return 0; + } + } + + /// Maps a database head to its exported record. + /// The head. + /// Export time. + /// Exporting node. + /// The record. + public static AuditChainHeadRecord ToRecord(AuditChainHeadSnapshot head, DateTime exportedAtUtc, string? node) + { + ArgumentNullException.ThrowIfNull(head); + return new AuditChainHeadRecord( + AuditChainHeadRecord.CurrentVersion, + AuditChainHasher.PartitionLabel(head.Month), + head.LastSeq, + AuditChainHeadComparer.ToHex(head.LastHash), + head.PurgedThroughSeq, + head.PurgedThroughHash is null ? null : AuditChainHeadComparer.ToHex(head.PurgedThroughHash), + head.ChainedRows, + head.Tombstones, + head.UpdatedAtUtc, + exportedAtUtc, + node); + } + + /// + public Task StopAsync(CancellationToken cancellationToken) + { + try + { + _cts?.Cancel(); + } + catch (ObjectDisposedException) + { + // Dispose already cancelled the loop. + } + + return _loop ?? Task.CompletedTask; + } + + /// + public void Dispose() + { + try + { + _cts?.Cancel(); + } + catch (ObjectDisposedException) + { + // Idempotent. + } + + _cts?.Dispose(); + } +} diff --git a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/AuditChainHeadJson.cs b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/AuditChainHeadJson.cs new file mode 100644 index 00000000..51bbc52e --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/AuditChainHeadJson.cs @@ -0,0 +1,21 @@ +using System.Text.Json; +using System.Text.Json.Serialization; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; + +namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport; + +/// The one JSON shape every sink writes (and audit verify-heads reads). +public static class AuditChainHeadJson +{ + /// camelCase, compact, nulls kept so every record has every field. + public static readonly JsonSerializerOptions Options = new(JsonSerializerDefaults.Web) + { + WriteIndented = false, + DefaultIgnoreCondition = JsonIgnoreCondition.Never, + }; + + /// Serialises one record to a single line. + /// The record. + /// Compact JSON. + public static string Serialize(AuditChainHeadRecord record) => JsonSerializer.Serialize(record, Options); +} diff --git a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/FileAuditChainHeadSink.cs b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/FileAuditChainHeadSink.cs new file mode 100644 index 00000000..9a6c4ba1 --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/FileAuditChainHeadSink.cs @@ -0,0 +1,79 @@ +using System.Text; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; + +namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport; + +/// +/// Appends exported heads as JSON lines to {Directory}/{yyyy}/audit-chain-heads-{yyyy-MM-dd}-{node}.jsonl. +/// +/// +/// The file is only ever opened with — never truncated, rewritten +/// or deleted by ScadaBridge — so the share can grant the service account create + append data +/// only. One file per node per UTC day keeps the two central nodes from interleaving writes in +/// one file. Each batch is flushed to disk before the call returns. +/// +public sealed class FileAuditChainHeadSink : IAuditChainHeadSink +{ + private readonly string _directory; + private readonly string _node; + private readonly TimeProvider _time; + + /// Creates the sink. + /// Absolute export directory. + /// This node's name (part of the file name). + /// Clock (file-per-day naming). + public FileAuditChainHeadSink(string directory, string node, TimeProvider? time = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(directory); + if (!Path.IsPathFullyQualified(directory)) + { + throw new ArgumentException($"Head-export directory '{directory}' must be absolute.", nameof(directory)); + } + + _directory = directory; + _node = SafeName(string.IsNullOrWhiteSpace(node) ? "node" : node); + _time = time ?? TimeProvider.System; + } + + /// + public string Describe => _directory; + + /// The file a write at lands in. + /// The write time. + /// The full path. + public string PathFor(DateTime nowUtc) => Path.Combine( + _directory, nowUtc.ToString("yyyy", System.Globalization.CultureInfo.InvariantCulture), + $"audit-chain-heads-{nowUtc:yyyy-MM-dd}-{_node}.jsonl"); + + /// + public async Task WriteAsync(IReadOnlyList records, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(records); + if (records.Count == 0) + { + return; + } + + var path = PathFor(_time.GetUtcNow().UtcDateTime); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + + var text = new StringBuilder(); + foreach (var record in records) + { + text.Append(AuditChainHeadJson.Serialize(record)).Append('\n'); + } + + var bytes = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false).GetBytes(text.ToString()); + await using var stream = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.Read, 4096, useAsync: true); + await stream.WriteAsync(bytes, ct).ConfigureAwait(false); + await stream.FlushAsync(ct).ConfigureAwait(false); + stream.Flush(flushToDisk: true); + } + + private static string SafeName(string value) + { + var invalid = Path.GetInvalidFileNameChars(); + var chars = value.Select(c => invalid.Contains(c) || c == ' ' ? '_' : c).ToArray(); + return new string(chars); + } +} diff --git a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/IAuditChainHeadSink.cs b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/IAuditChainHeadSink.cs new file mode 100644 index 00000000..70935578 --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/IAuditChainHeadSink.cs @@ -0,0 +1,16 @@ +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; + +namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport; + +/// An off-box destination for exported chain heads (ScadaBridge#107 item 1). +public interface IAuditChainHeadSink +{ + /// Short description for logs (e.g. the directory or host:port/udp). + string Describe { get; } + + /// Writes the records; throws when they could not all be delivered. + /// The records, oldest month first. + /// Cancellation token. + /// A task that completes when the records are durably handed off. + Task WriteAsync(IReadOnlyList records, CancellationToken ct); +} diff --git a/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/SyslogAuditChainHeadSink.cs b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/SyslogAuditChainHeadSink.cs new file mode 100644 index 00000000..7bb95354 --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.AuditLog/Central/HeadExport/SyslogAuditChainHeadSink.cs @@ -0,0 +1,108 @@ +using System.Globalization; +using System.Net.Sockets; +using System.Text; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; + +namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport; + +/// +/// Sends exported heads as RFC 5424 syslog messages — one message per head, the JSON record as +/// MSG — over UDP (RFC 5426, one datagram each) or TCP (RFC 6587 octet-counting framing, one +/// connection per export). +/// +/// +/// Message: <PRI>1 TIMESTAMP HOSTNAME APP-NAME PROCID chainhead - {json}, severity +/// Notice (5), PRI = facility × 8 + 5. No structured data (a private enterprise number would be +/// needed); the collector keeps the JSON body, which audit verify-heads reads back from +/// any line that contains it. UDP is fire-and-forget — use TCP where a lost datagram matters. +/// +public sealed class SyslogAuditChainHeadSink : IAuditChainHeadSink +{ + /// MSGID of every head message. + public const string MessageId = "chainhead"; + + private const int SeverityNotice = 5; + + private readonly AuditChainHeadSyslogSinkOptions _options; + private readonly string _hostName; + private readonly TimeProvider _time; + + /// Creates the sink. + /// Syslog settings. + /// HOSTNAME field (defaults to the machine name). + /// Clock for TIMESTAMP. + public SyslogAuditChainHeadSink(AuditChainHeadSyslogSinkOptions options, string? hostName = null, TimeProvider? time = null) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + ArgumentException.ThrowIfNullOrWhiteSpace(options.Host); + _hostName = Token(hostName ?? Environment.MachineName, 255); + _time = time ?? TimeProvider.System; + } + + /// + public string Describe => $"{_options.Host}:{_options.Port}/{_options.Protocol.ToString().ToLowerInvariant()}"; + + /// Formats one RFC 5424 message (no framing). + /// The record. + /// The message text. + public string Format(AuditChainHeadRecord record) + { + var pri = (_options.Facility * 8) + SeverityNotice; + var timestamp = _time.GetUtcNow().UtcDateTime.ToString("yyyy-MM-dd'T'HH:mm:ss.fff'Z'", CultureInfo.InvariantCulture); + return string.Create(CultureInfo.InvariantCulture, + $"<{pri}>1 {timestamp} {_hostName} {Token(_options.AppName, 48)} {Environment.ProcessId} {MessageId} - {AuditChainHeadJson.Serialize(record)}"); + } + + /// + public async Task WriteAsync(IReadOnlyList records, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(records); + if (records.Count == 0) + { + return; + } + + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct); + timeout.CancelAfter(TimeSpan.FromSeconds(Math.Max(1, _options.TimeoutSeconds))); + var token = timeout.Token; + + if (_options.Protocol == AuditSyslogProtocol.Udp) + { + using var udp = new UdpClient(); + udp.Connect(_options.Host!, _options.Port); + foreach (var record in records) + { + var bytes = Encoding.UTF8.GetBytes(Format(record)); + await udp.SendAsync(bytes, token).ConfigureAwait(false); + } + + return; + } + + using var tcp = new TcpClient(); + await tcp.ConnectAsync(_options.Host!, _options.Port, token).ConfigureAwait(false); + var stream = tcp.GetStream(); + foreach (var record in records) + { + var message = Encoding.UTF8.GetBytes(Format(record)); + var frame = Encoding.ASCII.GetBytes(message.Length.ToString(CultureInfo.InvariantCulture) + " "); + await stream.WriteAsync(frame, token).ConfigureAwait(false); + await stream.WriteAsync(message, token).ConfigureAwait(false); + } + + await stream.FlushAsync(token).ConfigureAwait(false); + tcp.Client.Shutdown(SocketShutdown.Send); + } + + // RFC 5424 header fields are PRINTUSASCII (33-126), no spaces; "-" when empty. + private static string Token(string? value, int max) + { + if (string.IsNullOrEmpty(value)) + { + return "-"; + } + + var chars = value.Where(c => c is >= '!' and <= '~').Take(max).ToArray(); + return chars.Length == 0 ? "-" : new string(chars); + } +} diff --git a/src/ZB.MOM.WW.ScadaBridge.AuditLog/ServiceCollectionExtensions.cs b/src/ZB.MOM.WW.ScadaBridge.AuditLog/ServiceCollectionExtensions.cs index 4b5db4d9..0bc2c98b 100644 --- a/src/ZB.MOM.WW.ScadaBridge.AuditLog/ServiceCollectionExtensions.cs +++ b/src/ZB.MOM.WW.ScadaBridge.AuditLog/ServiceCollectionExtensions.cs @@ -52,6 +52,12 @@ public static class ServiceCollectionExtensions /// Configuration section bound to . public const string PartitionMaintenanceSectionName = "AuditLog:PartitionMaintenance"; + /// + /// Optional connection string for the AuditLog maintenance paths — the hash-chain job and both + /// purge paths (ScadaBridge#107 item 3). Unset: they run on ScadaBridge:Database:ConfigurationDb. + /// + public const string MaintenanceConnectionStringKey = "AuditLog:MaintenanceConnectionString"; + /// Configuration section bound to (ScadaBridge#41). public const string HashChainSectionName = "AuditLog:HashChain"; @@ -411,6 +417,26 @@ public static class ServiceCollectionExtensions config, HashChainSectionName); services.AddHostedService(); + // Off-box chain-head export (ScadaBridge#107 item 1, AuditLog:HashChain:HeadExport, + // default off). The sink is built from the bound options; null when disabled. + services.AddHostedService(sp => + { + var options = sp.GetRequiredService>(); + var node = sp.GetService()?.NodeName; + var export = options.Value.HeadExport; + Central.HeadExport.IAuditChainHeadSink? sink = !export.Enabled ? null : export.Sink switch + { + Central.HeadExport.AuditChainHeadSinkKind.Syslog => new Central.HeadExport.SyslogAuditChainHeadSink(export.Syslog), + _ => new Central.HeadExport.FileAuditChainHeadSink(export.File.Directory!, node ?? Environment.MachineName), + }; + return new Central.HeadExport.AuditChainHeadExportService( + sp.GetRequiredService(), + options, + sink, + node, + sp.GetRequiredService>()); + }); + // Parquet cold archive (ScadaBridge#42). Default OFF; when enabled the purge actor // archives each expired partition before switching it out. IAuditLogArchive is // registered by AddConfigurationDatabase (central only) and reads these options. diff --git a/src/ZB.MOM.WW.ScadaBridge.CLI/Commands/AuditCommands.cs b/src/ZB.MOM.WW.ScadaBridge.CLI/Commands/AuditCommands.cs index 91976085..43528803 100644 --- a/src/ZB.MOM.WW.ScadaBridge.CLI/Commands/AuditCommands.cs +++ b/src/ZB.MOM.WW.ScadaBridge.CLI/Commands/AuditCommands.cs @@ -10,8 +10,10 @@ namespace ZB.MOM.WW.ScadaBridge.CLI.Commands; /// (GET /api/audit/query, GET /api/audit/export, /// GET /api/audit/tree), plus verify-chain, which sends the /// VerifyAuditChain management command (hash-chain tamper evidence, ScadaBridge#41), -/// and archive list / archive verify, which send ListAuditArchives / -/// VerifyAuditArchive (Parquet cold archive, ScadaBridge#42). +/// verify-heads, which checks off-box exported chain heads with +/// VerifyAuditChainHeads (ScadaBridge#107), and archive list / archive verify, +/// which send ListAuditArchives / VerifyAuditArchive (Parquet cold archive, +/// ScadaBridge#42). /// public static class AuditCommands { @@ -32,6 +34,7 @@ public static class AuditCommands command.Add(BuildExport(urlOption, formatOption, usernameOption, passwordOption)); command.Add(BuildTree(urlOption, formatOption, usernameOption, passwordOption)); command.Add(BuildVerifyChain(urlOption, formatOption, usernameOption, passwordOption)); + command.Add(BuildVerifyHeads(urlOption, formatOption, usernameOption, passwordOption)); command.Add(BuildBackfillSourceNode(urlOption, formatOption, usernameOption, passwordOption)); command.Add(BuildArchive(urlOption, formatOption, usernameOption, passwordOption)); @@ -323,6 +326,54 @@ public static class AuditCommands /// Builds the audit archive group (ScadaBridge#42): list and verify /// over the AuditLog Parquet cold archives on the central archive directory. /// + private static Command BuildVerifyHeads(Option urlOption, Option formatOption, Option usernameOption, Option passwordOption) + { + var pathOption = new Option("--path") + { + Description = "Export file, or directory of *.jsonl/*.log/*.txt export files (file sink or syslog collector output)", + Required = true, + }; + var cmd = new Command("verify-heads") + { + Description = "Check AuditLog chain heads exported off-box against the database chain; exit 3 if any no longer matches", + }; + cmd.Add(pathOption); + cmd.SetAction(async (ParseResult result) => + { + AuditVerifyHeadsHelpers.ReadResult read; + try + { + read = AuditVerifyHeadsHelpers.ReadRecords(result.GetValue(pathOption)!); + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException) + { + OutputFormatter.WriteError(ex.Message, "INVALID_ARGUMENT"); + return 1; + } + + if (read.Records.Count == 0) + { + OutputFormatter.WriteError( + $"No chain-head records found in {read.Files} file(s) ({read.LinesSkipped} unparsable line(s)).", + "INVALID_ARGUMENT"); + return 1; + } + + if (read.LinesSkipped > 0) + { + Console.Error.WriteLine($"warning: skipped {read.LinesSkipped} line(s) without a chain-head record."); + } + + var format = AuditCommandHelpers.ResolveFormat(result, formatOption); + return await CommandHelpers.ExecuteCommandAsync( + result, urlOption, formatOption, usernameOption, passwordOption, + new VerifyAuditChainHeadsCommand(read.Records), + timeout: AuditVerifyHeadsHelpers.RequestTimeout, + onSuccess: json => AuditVerifyHeadsHelpers.Render(json, format, Console.Out)); + }); + return cmd; + } + private static Command BuildArchive(Option urlOption, Option formatOption, Option usernameOption, Option passwordOption) { var group = new Command("archive") { Description = "List and verify the AuditLog Parquet cold archives" }; diff --git a/src/ZB.MOM.WW.ScadaBridge.CLI/Commands/AuditVerifyHeadsHelpers.cs b/src/ZB.MOM.WW.ScadaBridge.CLI/Commands/AuditVerifyHeadsHelpers.cs new file mode 100644 index 00000000..c4cd73f4 --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.CLI/Commands/AuditVerifyHeadsHelpers.cs @@ -0,0 +1,160 @@ +using System.Globalization; +using System.Text.Json; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; + +namespace ZB.MOM.WW.ScadaBridge.CLI.Commands; + +/// +/// Helpers for audit verify-heads (ScadaBridge#107 item 1): read AuditLog chain heads +/// exported off-box (the AuditLog:HashChain:HeadExport JSON lines — straight from the +/// file sink, or from a syslog collector's log), send them in a +/// VerifyAuditChainHeads management command, and render the verdicts. +/// +/// +/// Exit codes: 0 every exported head is consistent with the database, +/// at least one is not (tamper evidence), 1 on a request +/// or input error, 2 on an authorization failure. +/// +public static class AuditVerifyHeadsHelpers +{ + /// Process exit code when a head check failed. + public const int FailedExitCode = 3; + + /// Client deadline (the server bounds the call with the long-running Ask). + public static readonly TimeSpan RequestTimeout = TimeSpan.FromMinutes(6); + + private static readonly JsonSerializerOptions ReadOptions = new(JsonSerializerDefaults.Web); + + /// The result of reading export files. + /// Distinct records (by month, sequence, hash), latest export kept. + /// Lines that held a head record. + /// Non-empty lines without a parsable record. + /// Files read. + public sealed record ReadResult(IReadOnlyList Records, int LinesRead, int LinesSkipped, int Files); + + /// + /// Reads every record from — one file, or every *.jsonl / + /// *.log / *.txt file below a directory. Each line's JSON object is taken from + /// its first { to its last }, so syslog-prefixed lines parse too. + /// + /// File or directory. + /// The records read. + /// The path does not exist. + public static ReadResult ReadRecords(string path) + { + string[] files; + if (File.Exists(path)) + { + files = [path]; + } + else if (Directory.Exists(path)) + { + files = Directory.EnumerateFiles(path, "*", SearchOption.AllDirectories) + .Where(f => f.EndsWith(".jsonl", StringComparison.OrdinalIgnoreCase) + || f.EndsWith(".log", StringComparison.OrdinalIgnoreCase) + || f.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)) + .Order(StringComparer.Ordinal) + .ToArray(); + } + else + { + throw new FileNotFoundException($"'{path}' is neither a file nor a directory.", path); + } + + var records = new List(); + int read = 0, skipped = 0; + foreach (var file in files) + { + foreach (var line in File.ReadLines(file)) + { + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + var record = TryParseLine(line); + if (record is null) + { + skipped++; + continue; + } + + read++; + records.Add(record); + } + } + + var distinct = records + .GroupBy(r => (r.Month, r.LastSeq, Hash: r.LastHash.ToLowerInvariant())) + .Select(g => g.OrderByDescending(r => r.ExportedAtUtc).First()) + .OrderBy(r => r.Month, StringComparer.Ordinal) + .ThenBy(r => r.LastSeq) + .ToList(); + return new ReadResult(distinct, read, skipped, files.Length); + } + + /// Parses one export line; null when it holds no valid record. + /// The line. + /// The record, or null. + public static AuditChainHeadRecord? TryParseLine(string line) + { + var start = line.IndexOf('{'); + var end = line.LastIndexOf('}'); + if (start < 0 || end <= start) + { + return null; + } + + try + { + var record = JsonSerializer.Deserialize(line.AsSpan(start, end - start + 1), ReadOptions); + return record is { V: >= 1, Month.Length: 7, LastHash.Length: > 0 } && record.LastSeq > 0 ? record : null; + } + catch (JsonException) + { + return null; + } + } + + /// Renders a VerifyAuditChainHeads result and maps it to the exit code. + /// Success body (an AuditChainHeadVerificationResult, camelCase). + /// Resolved output format. + /// Writer for the report. + /// 0 when verified, otherwise. + public static int Render(string json, string format, TextWriter output) + { + using var doc = JsonDocument.Parse(json); + var root = doc.RootElement; + var verified = root.TryGetProperty("verified", out var v) && v.ValueKind == JsonValueKind.True; + + if (!string.Equals(format, "table", StringComparison.OrdinalIgnoreCase)) + { + output.WriteLine(json); + output.Flush(); + return verified ? 0 : FailedExitCode; + } + + output.WriteLine($"Verified : {(verified ? "yes" : "NO")}"); + output.WriteLine($"Heads checked : {Num(root, "recordsChecked")}"); + if (root.TryGetProperty("checks", out var checks) && checks.ValueKind == JsonValueKind.Array) + { + output.WriteLine($"{"MONTH",-8} {"EXPORTED SEQ",12} {"DB SEQ",8} {"STATUS",-13} DETAIL"); + foreach (var c in checks.EnumerateArray()) + { + output.WriteLine(string.Create(CultureInfo.InvariantCulture, + $"{Str(c, "month"),-8} {Num(c, "exportedSeq"),12} {Num(c, "databaseSeq", "-"),8} {Str(c, "status"),-13} {Str(c, "detail")}")); + } + } + + output.Flush(); + return verified ? 0 : FailedExitCode; + } + + private static string Str(JsonElement e, string name) + => e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.String ? v.GetString() ?? "" : ""; + + private static string Num(JsonElement e, string name, string missing = "0") + => e.TryGetProperty(name, out var v) && v.ValueKind == JsonValueKind.Number + ? v.GetInt64().ToString(CultureInfo.InvariantCulture) + : missing; +} diff --git a/src/ZB.MOM.WW.ScadaBridge.CLI/README.md b/src/ZB.MOM.WW.ScadaBridge.CLI/README.md index f6fa627b..e499bd83 100644 --- a/src/ZB.MOM.WW.ScadaBridge.CLI/README.md +++ b/src/ZB.MOM.WW.ScadaBridge.CLI/README.md @@ -2071,6 +2071,41 @@ end, `DuplicateSequence`, `IncompleteLink`). `--format table` prints a summary; The chain only exists once `AuditLog:HashChain:Enabled` is on for the central nodes (default off); see `Component-AuditLog.md`, Security & Tamper-Evidence. +#### `audit verify-heads` + +Check AuditLog chain heads that were exported off-box (ScadaBridge#107) against the +database chain. + +```sh +scadabridge --url audit verify-heads --path +``` + +| Option | Required | Default | Description | +|--------|----------|---------|-------------| +| `--path` | yes | — | An export file, or a directory read recursively for `*.jsonl` / `*.log` / `*.txt` | + +The central nodes write one JSON record per month head (`month`, `lastSeq`, `lastHash`, +purge anchor, chained-row and tombstone counts, `exportedAtUtc`, `node`) when +`AuditLog:HashChain:HeadExport:Enabled` is on — to append-only files +(`{Directory}/{yyyy}/audit-chain-heads-{yyyy-MM-dd}-{node}.jsonl`) or to syslog (RFC 5424, the +JSON record as the message). The CLI reads both forms (the JSON object is taken from each +line's first `{` to its last `}`, so a syslog collector's prefixes do not matter), keeps one +record per distinct (month, sequence, hash), and sends `VerifyAuditChainHeads`. Per record the +server reports `Matches`, `Extended`, `Purged` (the link was removed by a partition purge — +check the month's archive), or a failure: `Mismatch` (the chain was rewritten and re-hashed), +`Regressed` (head rolled back), `MissingHead`, `MissingLink` (a chained row deleted outside the +purge paths), `CountMismatch`. + +| Exit code | Meaning | +|-----------|---------| +| `0` | Every exported head is consistent with the database | +| `3` | At least one failure — tamper evidence | +| `1` | No records found, bad input, `CHAIN_BUSY`, server error | +| `2` | Authorization failure (needs `Administrator` or `Viewer`, not site-scoped) | + +Run it with `audit verify-chain`: `verify-chain` recomputes each month's links; +`verify-heads` proves the database still holds the heads that left the box. + #### `audit archive list` / `audit archive verify` List and verify the AuditLog Parquet cold archives (ScadaBridge#42) that the retention diff --git a/src/ZB.MOM.WW.ScadaBridge.Commons/Interfaces/Repositories/IAuditChainHeadStore.cs b/src/ZB.MOM.WW.ScadaBridge.Commons/Interfaces/Repositories/IAuditChainHeadStore.cs new file mode 100644 index 00000000..f6aa122a --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.Commons/Interfaces/Repositories/IAuditChainHeadStore.cs @@ -0,0 +1,28 @@ +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; + +namespace ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; + +/// +/// Reads the AuditLog hash-chain heads for off-box export and checks exported heads against +/// the database chain (ScadaBridge#107 item 1 — the external anchor of the #41 chain). +/// +public interface IAuditChainHeadStore +{ + /// + /// Reads every month's chain head with its live chained-row and tombstone counts, in one + /// consistent snapshot (under the chain lock SHARED, so no chaining pass or purge is half-way). + /// + /// Cancellation token. + /// One snapshot per month that has a head, oldest first. + Task> ReadHeadsAsync(CancellationToken ct = default); + + /// + /// Checks exported head records against the database: for each distinct exported head, does + /// the database still carry the exported hash at the exported sequence? + /// + /// Exported records (duplicates are collapsed). + /// Cancellation token. + /// The per-record verdicts. + Task VerifyExportedHeadsAsync( + IReadOnlyList records, CancellationToken ct = default); +} diff --git a/src/ZB.MOM.WW.ScadaBridge.Commons/Messages/Management/AuditLogCommands.cs b/src/ZB.MOM.WW.ScadaBridge.Commons/Messages/Management/AuditLogCommands.cs index 5fd94e17..4f0be926 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Commons/Messages/Management/AuditLogCommands.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Commons/Messages/Management/AuditLogCommands.cs @@ -1,4 +1,5 @@ using ZB.MOM.WW.ScadaBridge.Commons.Entities.Audit; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; namespace ZB.MOM.WW.ScadaBridge.Commons.Messages.Management; @@ -32,6 +33,15 @@ public record AuditLogQueryResult(IReadOnlyList Entries, int Tota /// The month to verify, YYYY-MM. public record VerifyAuditChainCommand(string Month); +/// +/// Checks AuditLog chain heads exported off-box (AuditLog:HashChain:HeadExport, +/// ScadaBridge#107 item 1) against the database chain: does the database still carry each +/// exported hash at its exported sequence? Read-only; same gate as +/// . The CLI reads the records from the export files. +/// +/// The exported head records (at most 50,000; duplicates are collapsed). +public record VerifyAuditChainHeadsCommand(IReadOnlyList Records); + /// /// Lists the AuditLog Parquet cold archives under the configured archive directory (audit /// roadmap T2, ScadaBridge#42). Read-only; OperationalAudit roles, refused for a site-scoped diff --git a/src/ZB.MOM.WW.ScadaBridge.Commons/Types/Audit/AuditChainHeads.cs b/src/ZB.MOM.WW.ScadaBridge.Commons/Types/Audit/AuditChainHeads.cs new file mode 100644 index 00000000..05edff96 --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.Commons/Types/Audit/AuditChainHeads.cs @@ -0,0 +1,126 @@ +using System.Text.Json.Serialization; + +namespace ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; + +/// +/// One month's AuditLog hash-chain head as exported off-box (ScadaBridge#107 item 1). The +/// export is the external anchor the in-database chain lacks: a principal able to rewrite rows, +/// recompute every later link and update AuditLogChainHead produces a chain that +/// verify-chain accepts, but cannot change a head hash that already left the box. +/// +/// +/// Serialised as one JSON object per line (camelCase) by both export sinks; the +/// audit verify-heads CLI reads those lines back and the server compares them with the +/// database. Hashes are lowercase hex. The format is versioned by . +/// +/// Record format version (1). +/// Chain partition, the UTC calendar month YYYY-MM. +/// The head's last sequence number — links written for the month so far. +/// The head's last link hash (hex). +/// The partition-purge checkpoint anchor's sequence, or null. +/// The anchor's hash (hex), or null. +/// Chained rows of the month still in dbo.AuditLog at export time. +/// Per-channel purge tombstones of the month at export time. +/// When the head row was last advanced. +/// When this record was exported. +/// The central node that exported it. +public sealed record AuditChainHeadRecord( + int V, + string Month, + long LastSeq, + string LastHash, + long? PurgedThroughSeq, + string? PurgedThroughHash, + long ChainedRows, + long Tombstones, + DateTime HeadUpdatedAtUtc, + DateTime ExportedAtUtc, + string? Node) +{ + /// The current record format version. + public const int CurrentVersion = 1; +} + +/// Outcome of comparing one exported head with the database chain. +[JsonConverter(typeof(JsonStringEnumConverter))] +public enum AuditChainHeadStatus +{ + /// The database head is exactly the exported head (same sequence, same hash, same row + tombstone count). + Matches, + + /// The chain has grown since the export and the link at the exported sequence still carries the exported hash. + Extended, + + /// + /// The link at the exported sequence was removed by a monthly partition purge (it is at or + /// before the checkpoint anchor and not the anchor itself), so the database cannot confirm + /// it; verify the month's Parquet archive instead (audit archive verify). + /// + Purged, + + /// The link at the exported sequence carries a different hash: the chain was rewritten. + Mismatch, + + /// The database head's sequence is BELOW the exported one: links were removed and the head rolled back. + Regressed, + + /// The month's head row is gone although links had been exported for it. + MissingHead, + + /// No row, tombstone or anchor carries the exported sequence any more: a chained row was deleted outside the purge paths. + MissingLink, + + /// + /// Same head as exported, no partition purge since, but chained rows + tombstones differ + /// from the exported counts: a chained row was removed or its link cleared. + /// + CountMismatch, +} + +/// One exported head checked against the database. +/// Chain partition (YYYY-MM). +/// The exported head's sequence. +/// When it was exported. +/// Which node exported it. +/// The verdict. +/// The database head's current sequence, or null when it has none. +/// Human-readable explanation. +public sealed record AuditChainHeadCheck( + string Month, + long ExportedSeq, + DateTime ExportedAtUtc, + string? Node, + AuditChainHeadStatus Status, + long? DatabaseSeq, + string Detail) +{ + /// True for the verdicts that are tamper evidence. + [JsonIgnore] + public bool IsFailure => Status is AuditChainHeadStatus.Mismatch or AuditChainHeadStatus.Regressed + or AuditChainHeadStatus.MissingHead or AuditChainHeadStatus.MissingLink or AuditChainHeadStatus.CountMismatch; +} + +/// Result of audit verify-heads. +/// True when no check is a failure. +/// Distinct exported heads checked. +/// Every check, failures first, then by month and sequence. +public sealed record AuditChainHeadVerificationResult(bool Verified, int RecordsChecked, IReadOnlyList Checks); + +/// The database's current chain head for a month, as read for export. +/// First instant of the month (UTC). +/// Head sequence. +/// Head hash. +/// Checkpoint anchor sequence, or null. +/// Checkpoint anchor hash, or null. +/// Chained rows of the month still present. +/// Tombstones of the month. +/// When the head row was last advanced. +public sealed record AuditChainHeadSnapshot( + DateTime Month, + long LastSeq, + byte[] LastHash, + long? PurgedThroughSeq, + byte[]? PurgedThroughHash, + long ChainedRows, + long Tombstones, + DateTime UpdatedAtUtc); diff --git a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditChainHeadComparer.cs b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditChainHeadComparer.cs new file mode 100644 index 00000000..d4c20006 --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditChainHeadComparer.cs @@ -0,0 +1,139 @@ +using System.Globalization; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; + +namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance; + +/// +/// Pure comparison of one exported chain head (ScadaBridge#107 item 1) with the database's +/// current state for that month. Kept free of I/O so every verdict is unit-testable; the +/// database side supplies the head snapshot and, when needed, the link stored at the +/// exported sequence number. +/// +public static class AuditChainHeadComparer +{ + /// What the database stores at one chain sequence number. + /// The RowHash of the live row carrying the sequence, or null. + /// The RowHash of a per-channel purge tombstone carrying it, or null. + public readonly record struct StoredLink(byte[]? RowHash, byte[]? TombstoneHash); + + /// Compares one exported head with the database. + /// The exported head. + /// The database head for the record's month, or null when there is none. + /// Reads the stored link at a sequence number (only called when the chain has grown). + /// The verdict. + public static async Task CompareAsync( + AuditChainHeadRecord record, AuditChainHeadSnapshot? head, Func> linkAt) + { + ArgumentNullException.ThrowIfNull(record); + ArgumentNullException.ThrowIfNull(linkAt); + + AuditChainHeadCheck Result(AuditChainHeadStatus status, string detail) => + new(record.Month, record.LastSeq, record.ExportedAtUtc, record.Node, status, head?.LastSeq, detail); + + if (!TryParseHex(record.LastHash, out var exported)) + { + return Result(AuditChainHeadStatus.Mismatch, $"Exported hash '{record.LastHash}' is not 32 bytes of hex."); + } + + if (head is null) + { + return Result(AuditChainHeadStatus.MissingHead, + $"{record.Month} has no chain head in the database, but {record.LastSeq} links were exported for it."); + } + + if (head.LastSeq < record.LastSeq) + { + return Result(AuditChainHeadStatus.Regressed, + $"Database head is at sequence {head.LastSeq}, below the exported {record.LastSeq}: links were removed and the head rolled back."); + } + + if (head.LastSeq == record.LastSeq) + { + if (!exported.AsSpan().SequenceEqual(head.LastHash)) + { + return Result(AuditChainHeadStatus.Mismatch, + $"Database head hash at sequence {head.LastSeq} is {ToHex(head.LastHash)}, exported {record.LastHash}: the chain was rewritten."); + } + + // No link was added since the export. Unless a partition purge set (or moved) the + // anchor since, rows + tombstones are invariant: the per-channel purge turns a row + // into a tombstone, nothing else may remove a chained row. + if (head.PurgedThroughSeq == record.PurgedThroughSeq + && head.ChainedRows + head.Tombstones != record.ChainedRows + record.Tombstones) + { + return Result(AuditChainHeadStatus.CountMismatch, + $"Chained rows + tombstones are {head.ChainedRows} + {head.Tombstones} in the database, {record.ChainedRows} + {record.Tombstones} exported."); + } + + return Result(AuditChainHeadStatus.Matches, "Database head equals the exported head."); + } + + // The chain has grown: the link at the exported sequence must still carry the exported hash. + if (head.PurgedThroughSeq is { } anchor && record.LastSeq <= anchor) + { + if (record.LastSeq < anchor) + { + return Result(AuditChainHeadStatus.Purged, + $"Sequence {record.LastSeq} was removed by a partition purge (anchor at {anchor}); verify the month's archive."); + } + + return head.PurgedThroughHash is { } anchorHash && exported.AsSpan().SequenceEqual(anchorHash) + ? Result(AuditChainHeadStatus.Extended, $"Exported head is the partition-purge anchor at {anchor}.") + : Result(AuditChainHeadStatus.Mismatch, $"Partition-purge anchor at {anchor} does not carry the exported hash."); + } + + var link = await linkAt(record.LastSeq).ConfigureAwait(false); + var stored = link.RowHash ?? link.TombstoneHash; + if (stored is null) + { + return Result(AuditChainHeadStatus.MissingLink, + $"No row, tombstone or anchor carries sequence {record.LastSeq}: a chained row was deleted outside the purge paths."); + } + + return exported.AsSpan().SequenceEqual(stored) + ? Result(AuditChainHeadStatus.Extended, + $"Chain grew to {head.LastSeq}; the {(link.RowHash is null ? "tombstone" : "row")} at {record.LastSeq} still carries the exported hash.") + : Result(AuditChainHeadStatus.Mismatch, + $"The {(link.RowHash is null ? "tombstone" : "row")} at sequence {record.LastSeq} carries {ToHex(stored)}, exported {record.LastHash}: the chain was rewritten."); + } + + /// Lowercase hex of a hash. + /// The bytes. + /// Hex string. + public static string ToHex(byte[] bytes) => Convert.ToHexString(bytes).ToLowerInvariant(); + + /// Parses a YYYY-MM month into its first instant (UTC). + /// The month text. + /// The parsed month. + /// True when well-formed. + public static bool TryParseMonth(string? month, out DateTime monthStart) + { + if (DateTime.TryParseExact(month, "yyyy-MM", CultureInfo.InvariantCulture, DateTimeStyles.None, out var parsed)) + { + monthStart = new DateTime(parsed.Year, parsed.Month, 1, 0, 0, 0, DateTimeKind.Utc); + return true; + } + + monthStart = default; + return false; + } + + private static bool TryParseHex(string? hex, out byte[] bytes) + { + bytes = []; + if (hex is null || hex.Length != AuditChainHasher.HashLength * 2) + { + return false; + } + + try + { + bytes = Convert.FromHexString(hex); + return true; + } + catch (FormatException) + { + return false; + } + } +} diff --git a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditLogHashChain.cs b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditLogHashChain.cs index 51c882ed..1bd3fe5a 100644 --- a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditLogHashChain.cs +++ b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditLogHashChain.cs @@ -45,7 +45,7 @@ public static class AuditChainLock /// those two columns; the ingest writer role keeps its table-wide DENY UPDATE. /// /// -public sealed class AuditLogHashChain : IAuditLogHashChain +public sealed class AuditLogHashChain : IAuditLogHashChain, IAuditChainHeadStore { // 4 parameters per row; 200 rows = 800 parameters, far under SQL Server's 2,100 cap. private const int UpdateChunkRows = 200; @@ -273,6 +273,151 @@ public sealed class AuditLogHashChain : IAuditLogHashChain } } + /// + public async Task> ReadHeadsAsync(CancellationToken ct = default) + { + var conn = _context.Database.GetDbConnection(); + var openedHere = await OpenAsync(conn, ct).ConfigureAwait(false); + try + { + await using var tx = await conn.BeginTransactionAsync(IsolationLevel.ReadCommitted, ct).ConfigureAwait(false); + await AcquireSharedOrThrowAsync(conn, tx, ct).ConfigureAwait(false); + var heads = await ReadHeadsCoreAsync(conn, tx, ct).ConfigureAwait(false); + await tx.CommitAsync(ct).ConfigureAwait(false); + return heads; + } + finally + { + if (openedHere) + { + await conn.CloseAsync().ConfigureAwait(false); + } + } + } + + /// + public async Task VerifyExportedHeadsAsync( + IReadOnlyList records, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(records); + + // Collapse duplicates (both centrals export, and every unchanged re-export repeats a + // record): one check per distinct (month, sequence, hash) claim. + var distinct = records + .GroupBy(r => (r.Month, r.LastSeq, Hash: r.LastHash?.ToLowerInvariant())) + .Select(g => g.OrderByDescending(r => r.ExportedAtUtc).First()) + .ToList(); + + var conn = _context.Database.GetDbConnection(); + var openedHere = await OpenAsync(conn, ct).ConfigureAwait(false); + try + { + await using var tx = await conn.BeginTransactionAsync(IsolationLevel.ReadCommitted, ct).ConfigureAwait(false); + await AcquireSharedOrThrowAsync(conn, tx, ct).ConfigureAwait(false); + var heads = (await ReadHeadsCoreAsync(conn, tx, ct).ConfigureAwait(false)) + .ToDictionary(h => AuditChainHasher.PartitionLabel(h.Month), StringComparer.Ordinal); + + var checks = new List(distinct.Count); + foreach (var record in distinct) + { + if (!AuditChainHeadComparer.TryParseMonth(record.Month, out var month)) + { + checks.Add(new AuditChainHeadCheck(record.Month ?? "", record.LastSeq, record.ExportedAtUtc, record.Node, + AuditChainHeadStatus.Mismatch, null, $"Exported month '{record.Month}' is not YYYY-MM.")); + continue; + } + + heads.TryGetValue(AuditChainHasher.PartitionLabel(month), out var head); + checks.Add(await AuditChainHeadComparer.CompareAsync( + record, head, seq => ReadStoredLinkAsync(conn, tx, month, seq, ct)).ConfigureAwait(false)); + } + + await tx.CommitAsync(ct).ConfigureAwait(false); + var ordered = checks + .OrderByDescending(c => c.IsFailure) + .ThenBy(c => c.Month, StringComparer.Ordinal) + .ThenBy(c => c.ExportedSeq) + .ToList(); + return new AuditChainHeadVerificationResult(ordered.All(c => !c.IsFailure), ordered.Count, ordered); + } + finally + { + if (openedHere) + { + await conn.CloseAsync().ConfigureAwait(false); + } + } + } + + private async Task AcquireSharedOrThrowAsync(DbConnection conn, DbTransaction tx, CancellationToken ct) + { + if (!await TryAcquireLockAsync(conn, tx, "Shared", VerifyLockTimeout, ct).ConfigureAwait(false)) + { + await tx.RollbackAsync(ct).ConfigureAwait(false); + throw new TimeoutException( + $"The AuditLog hash chain was busy (chaining or purge in progress) for {VerifyLockTimeout.TotalSeconds:0}s; retry."); + } + } + + private async Task> ReadHeadsCoreAsync(DbConnection conn, DbTransaction tx, CancellationToken ct) + { + // Per month: the head row, the chained rows still present (the filtered ChainSeq index, + // partition-eliminated by the month range) and the tombstones. The head table holds one + // row per month ever chained, so this is a few dozen rows. + var heads = new List(); + await using var cmd = Command(conn, tx, @" +SELECT h.ChainMonth, h.LastSeq, h.LastHash, h.PurgedThroughSeq, h.PurgedThroughHash, h.UpdatedAtUtc, + (SELECT COUNT_BIG(*) FROM dbo.AuditLog a + WHERE a.ChainSeq IS NOT NULL + AND a.OccurredAtUtc >= CAST(h.ChainMonth AS datetime2(7)) + AND a.OccurredAtUtc < DATEADD(MONTH, 1, CAST(h.ChainMonth AS datetime2(7)))) AS ChainedRows, + (SELECT COUNT_BIG(*) FROM dbo.AuditLogChainTombstone t WHERE t.ChainMonth = h.ChainMonth) AS Tombstones +FROM dbo.AuditLogChainHead h +ORDER BY h.ChainMonth;"); + cmd.CommandTimeout = VerifyCommandTimeoutSeconds; + await using var reader = await cmd.ExecuteReaderAsync(ct).ConfigureAwait(false); + while (await reader.ReadAsync(ct).ConfigureAwait(false)) + { + heads.Add(new AuditChainHeadSnapshot( + DateTime.SpecifyKind(reader.GetDateTime(0), DateTimeKind.Utc), + reader.GetInt64(1), + (byte[])reader.GetValue(2), + reader.IsDBNull(3) ? null : reader.GetInt64(3), + reader.IsDBNull(4) ? null : (byte[])reader.GetValue(4), + reader.GetInt64(6), + reader.GetInt64(7), + DateTime.SpecifyKind(reader.GetDateTime(5), DateTimeKind.Utc))); + } + + return heads; + } + + private async Task ReadStoredLinkAsync( + DbConnection conn, DbTransaction tx, DateTime month, long seq, CancellationToken ct) + { + byte[]? rowHash; + await using (var cmd = Command(conn, tx, + "SELECT TOP (1) RowHash FROM dbo.AuditLog WHERE ChainSeq = @seq AND OccurredAtUtc >= @monthStart AND OccurredAtUtc < @monthEnd;")) + { + cmd.CommandTimeout = VerifyCommandTimeoutSeconds; + cmd.Parameters.Add(new SqlParameter("@seq", SqlDbType.BigInt) { Value = seq }); + AddMonthRange(cmd, month); + rowHash = await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false) as byte[]; + } + + byte[]? tombstoneHash = null; + if (rowHash is null) + { + await using var cmd = Command(conn, tx, + "SELECT RowHash FROM dbo.AuditLogChainTombstone WHERE ChainMonth = @month AND ChainSeq = @seq;"); + AddDate(cmd, "@month", month); + cmd.Parameters.Add(new SqlParameter("@seq", SqlDbType.BigInt) { Value = seq }); + tombstoneHash = await cmd.ExecuteScalarAsync(ct).ConfigureAwait(false) as byte[]; + } + + return new AuditChainHeadComparer.StoredLink(rowHash, tombstoneHash); + } + /// /// Takes the chain application lock inside (released at commit / /// rollback). Returns false when the lock could not be granted within . diff --git a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditMaintenanceContext.cs b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditMaintenanceContext.cs new file mode 100644 index 00000000..6ca5a587 --- /dev/null +++ b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/Maintenance/AuditMaintenanceContext.cs @@ -0,0 +1,82 @@ +using Microsoft.AspNetCore.DataProtection; +using Microsoft.EntityFrameworkCore; + +namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance; + +/// +/// The connection the AuditLog maintenance paths run on — the hash-chain job and both purge +/// paths (partition switch-out with its archive, per-channel DELETE) — ScadaBridge#107 item 3. +/// +/// +/// +/// With AuditLog:MaintenanceConnectionString unset (the default), this is the scope's +/// ordinary : one login for everything, exactly as before. +/// With it set, the maintenance paths get their OWN context on that connection string, so a +/// DBA can give the runtime (writer) login only scadabridge_audit_writer — whose DENYs +/// stop it writing chain state or deleting audit rows — and a separate maintenance login the +/// scadabridge_audit_chainer + scadabridge_audit_purger roles. The string is read +/// after the pre-host ${secret:} expansion, so it may (and in production should) be a +/// secret reference. +/// +/// Scoped: an owned context is disposed with the scope; a borrowed one is left to DI. +/// +public sealed class AuditMaintenanceContext : IAsyncDisposable, IDisposable +{ + /// DI key of the maintenance-login IAuditLogRepository and IAuditLogArchive. + public const string ServiceKey = "audit-maintenance"; + + private readonly bool _owned; + + private AuditMaintenanceContext(ScadaBridgeDbContext context, bool owned) + { + Context = context; + _owned = owned; + } + + /// The context to run maintenance SQL on. + public ScadaBridgeDbContext Context { get; } + + /// True when is on the separate maintenance connection string. + public bool UsesSeparateLogin => _owned; + + /// Wraps the scope's ordinary context (no separate maintenance login configured). + /// The scope's context. + /// The wrapper. + public static AuditMaintenanceContext Shared(ScadaBridgeDbContext context) + => new(context ?? throw new ArgumentNullException(nameof(context)), owned: false); + + /// Opens an owned context on the maintenance connection string. + /// The maintenance connection string. + /// Data protection for secret-bearing columns (same as the runtime context). + /// The wrapper; disposing it disposes the context. + public static AuditMaintenanceContext Separate(string connectionString, IDataProtectionProvider? protectionProvider) + { + ArgumentException.ThrowIfNullOrWhiteSpace(connectionString); + var options = new DbContextOptionsBuilder() + .UseSqlServer(connectionString, sql => sql.EnableRetryOnFailure( + maxRetryCount: 5, maxRetryDelay: TimeSpan.FromSeconds(30), errorNumbersToAdd: null)) + .ConfigureWarnings(w => w.Ignore( + Microsoft.EntityFrameworkCore.Diagnostics.RelationalEventId.PendingModelChangesWarning)) + .Options; + var context = protectionProvider is null + ? new ScadaBridgeDbContext(options) + : new ScadaBridgeDbContext(options, protectionProvider); + return new AuditMaintenanceContext(context, owned: true); + } + + /// + public ValueTask DisposeAsync() => _owned ? Context.DisposeAsync() : ValueTask.CompletedTask; + + /// + public void Dispose() + { + if (_owned) + { + Context.Dispose(); + } + } +} + +/// The configured maintenance connection string (null = use the runtime connection). +/// The connection string, or null. +public sealed record AuditMaintenanceConnection(string? ConnectionString); diff --git a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/ServiceCollectionExtensions.cs b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/ServiceCollectionExtensions.cs index 967914af..a8b5593a 100644 --- a/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/ServiceCollectionExtensions.cs +++ b/src/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase/ServiceCollectionExtensions.cs @@ -20,8 +20,14 @@ public static class ServiceCollectionExtensions /// /// The service collection to register into. /// SQL Server connection string for the central configuration database. + /// + /// Optional separate connection string for the AuditLog maintenance paths — hash-chain job + /// and both purge paths (AuditLog:MaintenanceConnectionString, ScadaBridge#107 item 3). + /// Null or blank runs them on as before. + /// /// The same collection, for chaining. - public static IServiceCollection AddConfigurationDatabase(this IServiceCollection services, string connectionString) + public static IServiceCollection AddConfigurationDatabase( + this IServiceCollection services, string connectionString, string? auditMaintenanceConnectionString = null) { // The DbContext is constructed via the (options, IDataProtectionProvider) overload so // secret-bearing configuration columns are encrypted at rest. AddDataProtection below @@ -111,9 +117,31 @@ public static class ServiceCollectionExtensions // fresh scope on each tick (mirrors AuditLogPurgeActor / AuditLogIngestActor). services.AddScoped(); + // AuditLog maintenance connection (ScadaBridge#107 item 3): the chain job and both purge + // paths run on AuditMaintenanceContext — the scope's own context unless a separate + // maintenance login is configured, in which case an owned context on that string. + services.AddSingleton(new AuditMaintenanceConnection( + string.IsNullOrWhiteSpace(auditMaintenanceConnectionString) ? null : auditMaintenanceConnectionString)); + services.AddScoped(sp => + { + var maintenance = sp.GetRequiredService().ConnectionString; + return maintenance is null + ? AuditMaintenanceContext.Shared(sp.GetRequiredService()) + : AuditMaintenanceContext.Separate(maintenance, sp.GetService()); + }); + services.AddKeyedScoped(AuditMaintenanceContext.ServiceKey, (sp, _) => new AuditLogRepository( + sp.GetRequiredService().Context, + sp.GetService>())); + // AuditLog hash-chain tamper evidence (ScadaBridge#41): chained by the central - // AuditLogHashChainService, verified by the VerifyAuditChain management command. - services.AddScoped(); + // AuditLogHashChainService, verified by the VerifyAuditChain management command; the + // same instance reads/verifies the off-box head export (#107 item 1). Both on the + // maintenance connection. + services.AddScoped(sp => new AuditLogHashChain( + sp.GetRequiredService().Context, + sp.GetService>())); + services.AddScoped(sp => sp.GetRequiredService()); + services.AddScoped(sp => sp.GetRequiredService()); // AuditLog Parquet cold archive (ScadaBridge#42): written by the purge actor before a // partition switch, listed/verified by the ListAuditArchives / VerifyAuditArchive @@ -123,6 +151,12 @@ public static class ServiceCollectionExtensions sp.GetRequiredService(), sp.GetService>()?.Value ?? new AuditLogArchiveOptions(), sp.GetService>())); + // The purge actor's archive-then-purge runs on the maintenance connection (#107 item 3); + // list/verify from the management surface stay on the runtime connection (reads only). + services.AddKeyedScoped(AuditMaintenanceContext.ServiceKey, (sp, _) => new AuditLogPartitionArchive( + sp.GetRequiredService().Context, + sp.GetService>()?.Value ?? new AuditLogArchiveOptions(), + sp.GetService>())); // Opt-in config-audit retention (ScadaBridge:ConfigAudit:RetentionDays, default 0 = off). // Options binding to configuration is done by the host; without it the defaults apply diff --git a/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs b/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs index 17b767a1..ecdfc7dc 100644 --- a/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs +++ b/src/ZB.MOM.WW.ScadaBridge.Host/Program.cs @@ -531,7 +531,12 @@ try var configDbConnectionString = configuration["ScadaBridge:Database:ConfigurationDb"] ?? throw new InvalidOperationException("ScadaBridge:Database:ConfigurationDb connection string is required for Central role."); - builder.Services.AddConfigurationDatabase(configDbConnectionString); + // Optional second login for the AuditLog maintenance paths (chain job + both purge + // paths, ScadaBridge#107 item 3). Read from the pre-host-expanded configuration, so it + // may be a ${secret:} reference; unset = everything on the runtime connection, as before. + builder.Services.AddConfigurationDatabase( + configDbConnectionString, + configuration[ZB.MOM.WW.ScadaBridge.AuditLog.ServiceCollectionExtensions.MaintenanceConnectionStringKey]); builder.Services.AddOptions() .Bind(configuration.GetSection(ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance.ConfigAuditOptions.SectionName)) .ValidateOnStart(); diff --git a/src/ZB.MOM.WW.ScadaBridge.ManagementService/ManagementActor.cs b/src/ZB.MOM.WW.ScadaBridge.ManagementService/ManagementActor.cs index 7a96ed5e..6f18874a 100644 --- a/src/ZB.MOM.WW.ScadaBridge.ManagementService/ManagementActor.cs +++ b/src/ZB.MOM.WW.ScadaBridge.ManagementService/ManagementActor.cs @@ -392,6 +392,8 @@ public class ManagementActor : ReceiveActor // permission). A site-scoped caller is refused in the handler: a month's chain // spans every site. VerifyAuditChainCommand => OperationalAuditReaders, + // Off-box chain-head check (ScadaBridge#107 item 1): the same audit read. + VerifyAuditChainHeadsCommand => OperationalAuditReaders, // AuditLog Parquet archives (ScadaBridge#42): audit reads with verify-chain's gate — // OperationalAudit roles, site-scoped callers refused in the handler. @@ -569,6 +571,7 @@ public class ManagementActor : ReceiveActor // Audit Log QueryAuditLogCommand cmd => await HandleQueryAuditLog(sp, cmd), VerifyAuditChainCommand cmd => await HandleVerifyAuditChain(sp, cmd, user), + VerifyAuditChainHeadsCommand cmd => await HandleVerifyAuditChainHeads(sp, cmd, user), ListAuditArchivesCommand => await HandleListAuditArchives(sp, user), VerifyAuditArchiveCommand cmd => await HandleVerifyAuditArchive(sp, cmd, user), @@ -3273,6 +3276,48 @@ public class ManagementActor : ReceiveActor return new AuditArchiveListResult(archive.IsEnabled, await archive.ListAsync()); } + /// Upper bound on records per . + internal const int MaxChainHeadRecords = 50_000; + + /// + /// Checks exported AuditLog chain heads against the database (ScadaBridge#107 item 1). Same + /// gate as verify-chain: a site-scoped, non-Administrator caller is refused because the + /// chain spans every site. + /// + private static async Task HandleVerifyAuditChainHeads( + IServiceProvider sp, VerifyAuditChainHeadsCommand cmd, AuthenticatedUser user) + { + if (cmd.Records is null || cmd.Records.Count == 0) + { + throw new ManagementCommandException("No exported chain-head records were supplied.", "INVALID_ARGUMENT"); + } + + if (cmd.Records.Count > MaxChainHeadRecords) + { + throw new ManagementCommandException( + $"{cmd.Records.Count} records exceed the {MaxChainHeadRecords} limit; verify fewer export files at a time.", + "INVALID_ARGUMENT"); + } + + if (user.PermittedSiteIds.Length > 0 + && !user.Roles.Contains(Roles.Administrator, StringComparer.OrdinalIgnoreCase)) + { + throw new SiteScopeViolationException( + "Access denied: the audit hash chain spans every site; verify-heads needs an " + + "OperationalAudit role that is not scoped to specific sites."); + } + + var store = sp.GetRequiredService(); + try + { + return await store.VerifyExportedHeadsAsync(cmd.Records); + } + catch (TimeoutException ex) + { + throw new ManagementCommandException(ex.Message, "CHAIN_BUSY"); + } + } + /// /// Verifies one AuditLog Parquet archive (ScadaBridge#42): files against the manifest, /// the chain recomputed from the archived rows, the chain end against the purge anchor. diff --git a/src/ZB.MOM.WW.ScadaBridge.ManagementService/ManagementEndpoints.cs b/src/ZB.MOM.WW.ScadaBridge.ManagementService/ManagementEndpoints.cs index 5e178854..cb3dff59 100644 --- a/src/ZB.MOM.WW.ScadaBridge.ManagementService/ManagementEndpoints.cs +++ b/src/ZB.MOM.WW.ScadaBridge.ManagementService/ManagementEndpoints.cs @@ -63,6 +63,9 @@ public static class ManagementEndpoints // AuditLog hash-chain verification (ScadaBridge#41) recomputes one hash per row of // a whole month — millions of rows on a busy central. typeof(VerifyAuditChainCommand), + // Off-box chain-head check (ScadaBridge#107): one link lookup per exported head, under + // the chain lock SHARED (waits behind a chaining pass). + typeof(VerifyAuditChainHeadsCommand), // AuditLog archive verification (ScadaBridge#42) hashes and re-chains a whole month's // Parquet archive. typeof(VerifyAuditArchiveCommand), diff --git a/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/AuditLogPurgeActorTests.cs b/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/AuditLogPurgeActorTests.cs index 8f9e58bb..23b9cc5e 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/AuditLogPurgeActorTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/AuditLogPurgeActorTests.cs @@ -792,4 +792,34 @@ public class AuditLogPurgeActorTests : TestKit, IClassFixture + /// ScadaBridge#107 item 3: when the host registered the maintenance-connection repository + /// (keyed audit-maintenance), the purge runs on it, never on the runtime one. + /// + [Fact] + public void Tick_UsesTheMaintenanceConnectionRepository_WhenRegistered() + { + var runtime = new RecordingRepo { Boundaries = [new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc)] }; + var maintenance = new RecordingRepo { Boundaries = [new DateTime(2025, 1, 1, 0, 0, 0, DateTimeKind.Utc)] }; + var services = new ServiceCollection(); + services.AddScoped(_ => runtime); + services.AddKeyedScoped( + ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance.AuditMaintenanceContext.ServiceKey, (_, _) => maintenance); + var sp = services.BuildServiceProvider(); + var actor = Sys.ActorOf(Props.Create(() => new AuditLogPurgeActor( + sp, + Options.Create(FastTickOptions()), + Options.Create(new AuditLogOptions()), + NullLogger.Instance))); + + AwaitAssert( + () => Assert.NotEmpty(maintenance.SwitchedBoundaries), + duration: TimeSpan.FromSeconds(3), + interval: TimeSpan.FromMilliseconds(50)); + Assert.Empty(runtime.ThresholdQueries); + Assert.Empty(runtime.SwitchedBoundaries); + + StopPurgeActor(actor); + } } diff --git a/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/HeadExport/AuditChainHeadExportServiceTests.cs b/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/HeadExport/AuditChainHeadExportServiceTests.cs new file mode 100644 index 00000000..863d4b98 --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/HeadExport/AuditChainHeadExportServiceTests.cs @@ -0,0 +1,164 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.ScadaBridge.AuditLog.Central; +using ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport; +using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; + +namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Central.HeadExport; + +/// +/// (ScadaBridge#107 item 1): first tick exports every +/// head, later ticks only the changed ones, a failed write is retried with the same heads, and +/// the options validator guards each sink's settings. +/// +public sealed class AuditChainHeadExportServiceTests +{ + private sealed class FakeStore : IAuditChainHeadStore + { + public List Heads { get; } = []; + + public Task> ReadHeadsAsync(CancellationToken ct = default) + => Task.FromResult>(Heads.ToList()); + + public Task VerifyExportedHeadsAsync( + IReadOnlyList records, CancellationToken ct = default) => throw new NotSupportedException(); + } + + private sealed class RecordingSink : IAuditChainHeadSink + { + public List> Writes { get; } = []; + + public bool Fail { get; set; } + + public string Describe => "recording"; + + public Task WriteAsync(IReadOnlyList records, CancellationToken ct) + { + if (Fail) + { + throw new IOException("share offline"); + } + + Writes.Add(records); + return Task.CompletedTask; + } + } + + private static AuditChainHeadSnapshot Head(int month, long seq, long rows) => new( + new DateTime(2026, month, 1, 0, 0, 0, DateTimeKind.Utc), seq, Enumerable.Repeat((byte)seq, 32).ToArray(), + null, null, rows, 0, new DateTime(2026, month, 2, 0, 0, 0, DateTimeKind.Utc)); + + private static (AuditChainHeadExportService Service, FakeStore Store, RecordingSink Sink) Create(bool exportUnchanged = false) + { + var store = new FakeStore(); + var sink = new RecordingSink(); + var services = new ServiceCollection(); + services.AddScoped(_ => store); + var options = Options.Create(new AuditLogHashChainOptions + { + HeadExport = new AuditChainHeadExportOptions { Enabled = true, ExportUnchanged = exportUnchanged }, + }); + var service = new AuditChainHeadExportService( + services.BuildServiceProvider().GetRequiredService(), + options, sink, "central-a", NullLogger.Instance); + return (service, store, sink); + } + + /// First tick: every head; next tick: only the head that moved; unchanged: nothing. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Tick_ExportsAllHeadsFirst_ThenOnlyChangedOnes() + { + var (service, store, sink) = Create(); + store.Heads.AddRange([Head(4, 10, 10), Head(5, 20, 20)]); + + Assert.Equal(2, await service.TickAsync(default)); + Assert.Equal(0, await service.TickAsync(default)); + + store.Heads[1] = Head(5, 25, 25); + Assert.Equal(1, await service.TickAsync(default)); + + // A per-channel purge (row -> tombstone) keeps rows + tombstones, so it is not a change; + // a row disappearing without a tombstone is. + store.Heads[0] = Head(4, 10, 9); + Assert.Equal(1, await service.TickAsync(default)); + + Assert.Equal(3, sink.Writes.Count); + Assert.Equal(["2026-04", "2026-05"], sink.Writes[0].Select(r => r.Month)); + var moved = Assert.Single(sink.Writes[1]); + Assert.Equal(("2026-05", 25L, string.Concat(Enumerable.Repeat("19", 32))), (moved.Month, moved.LastSeq, moved.LastHash)); + Assert.Equal("central-a", moved.Node); + Assert.Equal(AuditChainHeadRecord.CurrentVersion, moved.V); + } + + /// ExportUnchanged re-exports every head every tick. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Tick_WithExportUnchanged_ExportsEveryHeadEveryTick() + { + var (service, store, _) = Create(exportUnchanged: true); + store.Heads.AddRange([Head(4, 10, 10), Head(5, 20, 20)]); + + Assert.Equal(2, await service.TickAsync(default)); + Assert.Equal(2, await service.TickAsync(default)); + } + + /// A failed write is contained and the same heads go out on the next tick. + /// A task that represents the asynchronous operation. + [Fact] + public async Task Tick_AfterAFailedWrite_RetriesTheSameHeads() + { + var (service, store, sink) = Create(); + store.Heads.Add(Head(4, 10, 10)); + sink.Fail = true; + + Assert.Equal(0, await service.TickAsync(default)); + sink.Fail = false; + Assert.Equal(1, await service.TickAsync(default)); + Assert.Equal(10, Assert.Single(Assert.Single(sink.Writes)).LastSeq); + } + + /// The validator ignores a disabled export and checks each sink's settings when enabled. + /// Which settings to apply. + /// A fragment of the expected failure, or null when valid. + [Theory] + [InlineData("disabled-bad", null)] + [InlineData("file-ok", null)] + [InlineData("file-relative", "File:Directory")] + [InlineData("file-missing", "File:Directory")] + [InlineData("syslog-ok", null)] + [InlineData("syslog-no-host", "Syslog:Host")] + [InlineData("syslog-bad-port", "Syslog:Port")] + [InlineData("syslog-bad-facility", "Syslog:Facility")] + [InlineData("interval-zero", "IntervalMinutes")] + public void Validator_ChecksTheSelectedSink(string configure, string? expectedError) + { + var root = OperatingSystem.IsWindows() ? @"C:\heads" : "/var/heads"; + var export = configure switch + { + "disabled-bad" => new AuditChainHeadExportOptions { Enabled = false, IntervalMinutes = 0 }, + "file-ok" => new AuditChainHeadExportOptions { Enabled = true, File = { Directory = root } }, + "file-relative" => new AuditChainHeadExportOptions { Enabled = true, File = { Directory = "heads" } }, + "file-missing" => new AuditChainHeadExportOptions { Enabled = true }, + "syslog-ok" => new AuditChainHeadExportOptions { Enabled = true, Sink = AuditChainHeadSinkKind.Syslog, Syslog = { Host = "collector" } }, + "syslog-no-host" => new AuditChainHeadExportOptions { Enabled = true, Sink = AuditChainHeadSinkKind.Syslog }, + "syslog-bad-port" => new AuditChainHeadExportOptions { Enabled = true, Sink = AuditChainHeadSinkKind.Syslog, Syslog = { Host = "c", Port = 0 } }, + "syslog-bad-facility" => new AuditChainHeadExportOptions { Enabled = true, Sink = AuditChainHeadSinkKind.Syslog, Syslog = { Host = "c", Facility = 24 } }, + _ => new AuditChainHeadExportOptions { Enabled = true, IntervalMinutes = 0, File = { Directory = root } }, + }; + + var result = new AuditLogHashChainOptionsValidator().Validate(null, new AuditLogHashChainOptions { HeadExport = export }); + + if (expectedError is null) + { + Assert.True(result.Succeeded, result.FailureMessage); + } + else + { + Assert.True(result.Failed); + Assert.Contains(expectedError, result.FailureMessage); + } + } +} diff --git a/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/HeadExport/AuditChainHeadSinkTests.cs b/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/HeadExport/AuditChainHeadSinkTests.cs new file mode 100644 index 00000000..a882b8f0 --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.AuditLog.Tests/Central/HeadExport/AuditChainHeadSinkTests.cs @@ -0,0 +1,135 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Text.Json; +using ZB.MOM.WW.ScadaBridge.AuditLog.Central.HeadExport; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; + +namespace ZB.MOM.WW.ScadaBridge.AuditLog.Tests.Central.HeadExport; + +/// +/// The two off-box chain-head sinks (ScadaBridge#107 item 1): the append-only JSON-lines file +/// sink, and RFC 5424 syslog over UDP and TCP (octet-counted framing). +/// +public sealed class AuditChainHeadSinkTests : IDisposable +{ + private readonly DirectoryInfo _dir = Directory.CreateTempSubdirectory("head-sink-"); + + /// + public void Dispose() => _dir.Delete(recursive: true); + + private static AuditChainHeadRecord Record(string month, long seq) => new( + 1, month, seq, new string('a', 64), null, null, seq, 0, + new DateTime(2026, 5, 3, 0, 0, 0, DateTimeKind.Utc), new DateTime(2026, 5, 3, 1, 0, 0, DateTimeKind.Utc), "central-a"); + + private sealed class FixedClock(DateTimeOffset now) : TimeProvider + { + public override DateTimeOffset GetUtcNow() => now; + } + + /// + /// Writes append to the node's file for the day — never truncate it — one JSON record per + /// line, and a pre-existing line survives. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task FileSink_AppendsJsonLines_ToThePerNodePerDayFile_WithoutTruncating() + { + var clock = new FixedClock(new DateTimeOffset(2026, 5, 3, 1, 0, 0, TimeSpan.Zero)); + var sink = new FileAuditChainHeadSink(_dir.FullName, "central-a", clock); + var path = sink.PathFor(clock.GetUtcNow().UtcDateTime); + Assert.Equal(Path.Combine(_dir.FullName, "2026", "audit-chain-heads-2026-05-03-central-a.jsonl"), path); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + await File.WriteAllTextAsync(path, "earlier line\n"); + + await sink.WriteAsync([Record("2026-04", 3), Record("2026-05", 7)], default); + await sink.WriteAsync([Record("2026-05", 9)], default); + + var lines = await File.ReadAllLinesAsync(path); + Assert.Equal(4, lines.Length); + Assert.Equal("earlier line", lines[0]); + var parsed = lines.Skip(1).Select(l => JsonSerializer.Deserialize(l, new JsonSerializerOptions(JsonSerializerDefaults.Web))!).ToList(); + Assert.Equal([3L, 7L, 9L], parsed.Select(r => r.LastSeq)); + Assert.Equal(Record("2026-05", 9), parsed[2]); + } + + /// A relative export directory is refused. + [Fact] + public void FileSink_RejectsARelativeDirectory() + { + Assert.Throws(() => new FileAuditChainHeadSink("relative/dir", "central-a")); + } + + /// UDP: one RFC 5424 datagram per record — PRI, version, MSGID, nil SD, JSON body. + /// A task that represents the asynchronous operation. + [Fact] + public async Task SyslogSink_Udp_SendsOneRfc5424DatagramPerRecord() + { + using var server = new UdpClient(new IPEndPoint(IPAddress.Loopback, 0)); + var port = ((IPEndPoint)server.Client.LocalEndPoint!).Port; + var sink = new SyslogAuditChainHeadSink( + new AuditChainHeadSyslogSinkOptions { Host = "127.0.0.1", Port = port, Protocol = AuditSyslogProtocol.Udp, Facility = 13 }, + hostName: "central a", time: new FixedClock(new DateTimeOffset(2026, 5, 3, 1, 2, 3, 456, TimeSpan.Zero))); + + await sink.WriteAsync([Record("2026-04", 3), Record("2026-05", 7)], default); + + var first = Encoding.UTF8.GetString((await server.ReceiveAsync().WaitAsync(TimeSpan.FromSeconds(5))).Buffer); + var second = Encoding.UTF8.GetString((await server.ReceiveAsync().WaitAsync(TimeSpan.FromSeconds(5))).Buffer); + // facility 13 (log audit) * 8 + severity 5 (notice) = 109; the space in the host name is dropped. + Assert.StartsWith($"<109>1 2026-05-03T01:02:03.456Z centrala scadabridge {Environment.ProcessId} chainhead - {{", first); + AssertRecordBody(first, 3); + AssertRecordBody(second, 7); + } + + /// TCP: octet-counted frames (RFC 6587), one per record, on one connection. + /// A task that represents the asynchronous operation. + [Fact] + public async Task SyslogSink_Tcp_SendsOctetCountedFrames() + { + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + try + { + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + var received = Task.Run(async () => + { + using var client = await listener.AcceptTcpClientAsync(); + using var ms = new MemoryStream(); + await client.GetStream().CopyToAsync(ms); + return ms.ToArray(); + }); + var sink = new SyslogAuditChainHeadSink( + new AuditChainHeadSyslogSinkOptions { Host = "127.0.0.1", Port = port, Protocol = AuditSyslogProtocol.Tcp }); + + await sink.WriteAsync([Record("2026-04", 3), Record("2026-05", 7)], default); + + var bytes = await received.WaitAsync(TimeSpan.FromSeconds(5)); + var frames = new List(); + var i = 0; + while (i < bytes.Length) + { + var space = Array.IndexOf(bytes, (byte)' ', i); + var length = int.Parse(Encoding.ASCII.GetString(bytes, i, space - i)); + frames.Add(Encoding.UTF8.GetString(bytes, space + 1, length)); + i = space + 1 + length; + } + + Assert.Equal(2, frames.Count); + Assert.All(frames, f => Assert.StartsWith("<109>1 ", f)); + AssertRecordBody(frames[0], 3); + AssertRecordBody(frames[1], 7); + } + finally + { + listener.Stop(); + } + } + + private static void AssertRecordBody(string message, long seq) + { + var json = message[message.IndexOf('{')..]; + var record = JsonSerializer.Deserialize(json, new JsonSerializerOptions(JsonSerializerDefaults.Web))!; + Assert.Equal(seq, record.LastSeq); + Assert.Equal(new string('a', 64), record.LastHash); + } +} diff --git a/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/Commands/AuditCommandsScaffoldTests.cs b/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/Commands/AuditCommandsScaffoldTests.cs index a2540984..c237c5ba 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/Commands/AuditCommandsScaffoldTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/Commands/AuditCommandsScaffoldTests.cs @@ -27,14 +27,14 @@ public class AuditCommandsScaffoldTests Assert.False(string.IsNullOrWhiteSpace(audit.Description)); } - /// Verifies that the audit group exposes the query, export, tree, verify-chain, backfill-source-node and archive subcommands. + /// Verifies that the audit group exposes the query, export, tree, verify-chain, verify-heads, backfill-source-node and archive subcommands. [Fact] - public void Audit_HasSixSubcommands_QueryExportTreeVerifyChainBackfillSourceNodeArchive() + public void Audit_HasSevenSubcommands_QueryExportTreeVerifyChainVerifyHeadsBackfillSourceNodeArchive() { var audit = BuildAudit(); var names = audit.Subcommands.Select(c => c.Name).OrderBy(n => n).ToArray(); Assert.Equal( - new[] { "archive", "backfill-source-node", "export", "query", "tree", "verify-chain" }, + new[] { "archive", "backfill-source-node", "export", "query", "tree", "verify-chain", "verify-heads" }, names); } @@ -65,7 +65,7 @@ public class AuditCommandsScaffoldTests // `archive` is a group (list / verify, ScadaBridge#42): check its leaves, not it. var audit = BuildAudit(); var leaves = audit.Subcommands.SelectMany(c => c.Subcommands.Count == 0 ? [c] : c.Subcommands).ToList(); - Assert.Equal(7, leaves.Count); + Assert.Equal(8, leaves.Count); // + verify-heads (ScadaBridge#107) Assert.All(leaves, sub => Assert.True(sub.Action != null, $"Leaf command '{sub.Name}' has no action.")); } diff --git a/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/Commands/AuditVerifyHeadsCommandTests.cs b/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/Commands/AuditVerifyHeadsCommandTests.cs new file mode 100644 index 00000000..b9dbcff2 --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/Commands/AuditVerifyHeadsCommandTests.cs @@ -0,0 +1,158 @@ +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Text.Json; +using ZB.MOM.WW.ScadaBridge.CLI.Commands; + +namespace ZB.MOM.WW.ScadaBridge.CLI.Tests.Commands; + +/// +/// Tests for scadabridge audit verify-heads (ScadaBridge#107 item 1): reading off-box +/// exported chain heads (file-sink lines and syslog-collector lines), collapsing duplicates, +/// sending VerifyAuditChainHeads, and mapping the verdict to the exit code. +/// +[Collection("Console")] +public sealed class AuditVerifyHeadsCommandTests : IDisposable +{ + private readonly DirectoryInfo _dir = Directory.CreateTempSubdirectory("verify-heads-"); + + /// + public void Dispose() => _dir.Delete(recursive: true); + + private static string Line(string month, long seq, char hash, string exportedAt = "2026-05-02T00:00:00Z", string node = "central-a") => + "{\"v\":1,\"month\":\"" + month + "\",\"lastSeq\":" + seq + ",\"lastHash\":\"" + new string(hash, 64) + + "\",\"purgedThroughSeq\":null,\"purgedThroughHash\":null,\"chainedRows\":" + seq + ",\"tombstones\":0," + + "\"headUpdatedAtUtc\":\"2026-05-02T00:00:00Z\",\"exportedAtUtc\":\"" + exportedAt + "\",\"node\":\"" + node + "\"}"; + + /// + /// File-sink lines and RFC 5424 syslog lines both parse; junk is skipped; the same head + /// exported by both nodes (and re-exported) collapses to one record. + /// + [Fact] + public void ReadRecords_ParsesFileAndSyslogLines_SkipsJunk_AndCollapsesDuplicates() + { + Directory.CreateDirectory(Path.Combine(_dir.FullName, "2026")); + File.WriteAllLines(Path.Combine(_dir.FullName, "2026", "audit-chain-heads-2026-05-02-central-a.jsonl"), + [ + Line("2026-05", 10, 'a'), + Line("2026-05", 12, 'b'), + "not json at all", + "", + ]); + File.WriteAllLines(Path.Combine(_dir.FullName, "syslog.log"), + [ + "Nov 10 03:00:00 collector <109>1 2026-05-02T00:00:00.000Z central-b scadabridge 42 chainhead - " + + Line("2026-05", 12, 'b', "2026-05-02T01:00:00Z", "central-b"), + "<109>1 2026-05-02T00:00:00.000Z central-b scadabridge 42 chainhead - " + Line("2026-04", 7, 'c'), + ]); + File.WriteAllText(Path.Combine(_dir.FullName, "ignored.csv"), Line("2026-03", 1, 'd')); + + var read = AuditVerifyHeadsHelpers.ReadRecords(_dir.FullName); + + Assert.Equal(2, read.Files); + Assert.Equal(4, read.LinesRead); + Assert.Equal(1, read.LinesSkipped); + Assert.Equal(["2026-04/7", "2026-05/10", "2026-05/12"], read.Records.Select(r => $"{r.Month}/{r.LastSeq}")); + // The duplicate kept is the latest export. + Assert.Equal("central-b", read.Records.Single(r => r.LastSeq == 12).Node); + } + + /// A record missing its hash or sequence is not a record. + /// The line. + [Theory] + [InlineData("{\"v\":1,\"month\":\"2026-05\",\"lastSeq\":0,\"lastHash\":\"aa\"}")] + [InlineData("{\"v\":1,\"month\":\"2026-05\",\"lastSeq\":3}")] + [InlineData("{\"month\":\"2026-05\",\"lastSeq\":3,\"lastHash\":\"aa\"}")] + [InlineData("{ broken")] + public void TryParseLine_RejectsIncompleteRecords(string line) + { + Assert.Null(AuditVerifyHeadsHelpers.TryParseLine(line)); + } + + /// + /// Round trip: the request carries VerifyAuditChainHeads with the distinct records, + /// and the verdict drives the exit code (0 verified, 3 not). + /// + /// Whether the fake server reports verified. + /// Output format. + /// Expected exit code. + /// Text the report must contain. + /// A task that represents the asynchronous operation. + [Theory] + [InlineData(true, "table", 0, "Verified : yes")] + [InlineData(false, "table", 3, "Mismatch")] + [InlineData(false, "json", 3, "\"verified\":false")] + public async Task VerifyHeads_SendsTheRecords_AndMapsVerdictToExitCode(bool verified, string format, int expectedExit, string expectedText) + { + var file = Path.Combine(_dir.FullName, "heads.jsonl"); + File.WriteAllLines(file, [Line("2026-05", 12, 'b'), Line("2026-05", 12, 'b', "2026-05-02T02:00:00Z")]); + var body = "{\"verified\":" + (verified ? "true" : "false") + ",\"recordsChecked\":1,\"checks\":[{\"month\":\"2026-05\"," + + "\"exportedSeq\":12,\"exportedAtUtc\":\"2026-05-02T02:00:00Z\",\"node\":\"central-a\",\"status\":\"" + + (verified ? "Matches" : "Mismatch") + "\",\"databaseSeq\":12,\"detail\":\"d\"}]}"; + + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + try + { + var port = ((IPEndPoint)listener.LocalEndpoint).Port; + string? requestBody = null; + var server = Task.Run(async () => + { + using var client = await listener.AcceptTcpClientAsync(); + await using var stream = client.GetStream(); + using var reader = new StreamReader(stream, Encoding.UTF8, leaveOpen: true); + var length = 0; + while (await reader.ReadLineAsync() is { Length: > 0 } line) + if (line.StartsWith("Content-Length:", StringComparison.OrdinalIgnoreCase)) + length = int.Parse(line[15..].Trim()); + var buffer = new char[length]; + var read = 0; + while (read < length) + { + var count = await reader.ReadAsync(buffer.AsMemory(read)); + if (count == 0) throw new EndOfStreamException(); + read += count; + } + requestBody = new string(buffer); + var bytes = Encoding.UTF8.GetBytes(body); + var headers = Encoding.ASCII.GetBytes( + $"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {bytes.Length}\r\nConnection: close\r\n\r\n"); + await stream.WriteAsync(headers); + await stream.WriteAsync(bytes); + }); + + var root = AuditCommandTestHarness.BuildRoot(); + var (exit, output, _) = AuditCommandTestHarness.Invoke( + root, "audit", "verify-heads", "--path", file, + "--url", $"http://127.0.0.1:{port}", "--username", "u", "--password", "p", "--format", format); + await server.WaitAsync(TimeSpan.FromSeconds(5)); + + Assert.Equal(expectedExit, exit); + Assert.Contains(expectedText, output); + using var payload = JsonDocument.Parse(requestBody!); + Assert.Equal("VerifyAuditChainHeads", payload.RootElement.GetProperty("command").GetString()); + var records = payload.RootElement.GetProperty("payload").EnumerateObject() + .First(p => string.Equals(p.Name, "records", StringComparison.OrdinalIgnoreCase)).Value; + Assert.Equal(1, records.GetArrayLength()); + Assert.Equal(new string('b', 64), records[0].EnumerateObject() + .First(p => string.Equals(p.Name, "lastHash", StringComparison.OrdinalIgnoreCase)).Value.GetString()); + } + finally + { + listener.Stop(); + } + } + + /// No records found is an input error, and nothing is sent. + [Fact] + public void VerifyHeads_NoRecords_ExitsOneWithoutContactingTheServer() + { + var file = Path.Combine(_dir.FullName, "empty.jsonl"); + File.WriteAllText(file, "nothing here\n"); + var root = AuditCommandTestHarness.BuildRoot(); + var (exit, _, _) = AuditCommandTestHarness.Invoke( + root, "audit", "verify-heads", "--path", file, "--url", "http://127.0.0.1:1", "--username", "u", "--password", "p"); + + Assert.Equal(1, exit); + } +} diff --git a/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/ManagementSurfaceParityTests.cs b/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/ManagementSurfaceParityTests.cs index 0e3d17b4..53ff80be 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/ManagementSurfaceParityTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.CLI.Tests/ManagementSurfaceParityTests.cs @@ -45,7 +45,7 @@ public class ManagementSurfaceParityTests } private static readonly HashSet AuditManagementLeaves = - ["audit verify-chain", "audit archive list", "audit archive verify"]; + ["audit verify-chain", "audit verify-heads", "audit archive list", "audit archive verify"]; private static async Task> ProbeProductionTree(string? omitGroup = null) { @@ -56,6 +56,12 @@ public class ManagementSurfaceParityTests await File.WriteAllTextAsync(file, "{}"); var csv = Path.Combine(directory.FullName, "native.csv"); await File.WriteAllTextAsync(csv, "SourceName,Connection,SourceReference,Filter\nSource,Connection,Path,\n"); + // `audit verify-heads` (ScadaBridge#107) reads exported chain heads before it sends anything. + HeadsFile = Path.Combine(directory.FullName, "heads.jsonl"); + await File.WriteAllTextAsync(HeadsFile, + "{\"v\":1,\"month\":\"2026-05\",\"lastSeq\":3,\"lastHash\":\"" + new string('a', 64) + + "\",\"purgedThroughSeq\":null,\"purgedThroughHash\":null,\"chainedRows\":3,\"tombstones\":0," + + "\"headUpdatedAtUtc\":\"2026-05-02T00:00:00Z\",\"exportedAtUtc\":\"2026-05-02T00:00:00Z\",\"node\":\"central-a\"}\n"); using var stop = new CancellationTokenSource(TimeSpan.FromSeconds(30)); var listener = new TcpListener(IPAddress.Loopback, 0); listener.Start(); @@ -96,7 +102,8 @@ public class ManagementSurfaceParityTests foreach (var (leaf, path) in Leaves(root, [])) { // Separate REST and persistent SignalR surfaces, not management contracts. - // `audit verify-chain` (VerifyAuditChain, ScadaBridge#41) and `audit archive + // `audit verify-chain` (VerifyAuditChain, ScadaBridge#41), `audit verify-heads` + // (VerifyAuditChainHeads, ScadaBridge#107) and `audit archive // list|verify` (ListAuditArchives / VerifyAuditArchive, ScadaBridge#42) are the // audit leaves that ARE management contracts, so they stay in the probe. if (path.Length == 0 || (path[0] == "audit" && !AuditManagementLeaves.Contains(string.Join(' ', path))) @@ -135,8 +142,11 @@ public class ManagementSurfaceParityTests foreach (var entry in Leaves(child, [.. path, child.Name])) yield return entry; } + private static string HeadsFile = ""; + private static string ValueFor(Option option, string path, string file, string csv) { + if (path == "audit verify-heads" && option.Name == "--path") return HeadsFile; var type = option.GetType().GetGenericArguments().FirstOrDefault(); type = Nullable.GetUnderlyingType(type ?? typeof(string)) ?? type; if (type == typeof(int) || type == typeof(long) || type == typeof(double)) return "1"; diff --git a/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditChainHeadComparerTests.cs b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditChainHeadComparerTests.cs new file mode 100644 index 00000000..78f7a7ad --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditChainHeadComparerTests.cs @@ -0,0 +1,81 @@ +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; +using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance; + +namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Maintenance; + +/// +/// Pure verdict table of (ScadaBridge#107 item 1) for the +/// branches the SQL Server tests do not reach: a link carried by a tombstone, the partition +/// anchor itself, and malformed input. +/// +public sealed class AuditChainHeadComparerTests +{ + private static readonly byte[] H1 = Enumerable.Repeat((byte)1, 32).ToArray(); + private static readonly byte[] H2 = Enumerable.Repeat((byte)2, 32).ToArray(); + + private static AuditChainHeadRecord Exported(long seq, byte[] hash) => new( + 1, "2026-05", seq, AuditChainHeadComparer.ToHex(hash), null, null, seq, 0, DateTime.UtcNow, DateTime.UtcNow, "central-a"); + + private static AuditChainHeadSnapshot Head(long seq, byte[] hash, long? anchor = null, byte[]? anchorHash = null) => + new(new DateTime(2026, 5, 1, 0, 0, 0, DateTimeKind.Utc), seq, hash, anchor, anchorHash, seq, 0, DateTime.UtcNow); + + private static Func> Link(byte[]? row, byte[]? tombstone) + => _ => Task.FromResult(new AuditChainHeadComparer.StoredLink(row, tombstone)); + + private static Func> NoLookup + => _ => throw new InvalidOperationException("no link lookup expected"); + + /// The exported link was later purged per channel: the tombstone carries the hash. + /// A task that represents the asynchronous operation. + [Fact] + public async Task GrownChain_LinkOnATombstone_IsExtended_OrMismatch() + { + Assert.Equal(AuditChainHeadStatus.Extended, + (await AuditChainHeadComparer.CompareAsync(Exported(3, H1), Head(9, H2), Link(null, H1))).Status); + Assert.Equal(AuditChainHeadStatus.Mismatch, + (await AuditChainHeadComparer.CompareAsync(Exported(3, H1), Head(9, H2), Link(null, H2))).Status); + } + + /// The exported head is the partition-purge anchor: compared with the anchor's hash, no lookup. + /// A task that represents the asynchronous operation. + [Fact] + public async Task GrownChain_ExportedHeadIsTheAnchor_ComparesTheAnchorHash() + { + Assert.Equal(AuditChainHeadStatus.Extended, + (await AuditChainHeadComparer.CompareAsync(Exported(4, H1), Head(9, H2, 4, H1), NoLookup)).Status); + Assert.Equal(AuditChainHeadStatus.Mismatch, + (await AuditChainHeadComparer.CompareAsync(Exported(4, H1), Head(9, H2, 4, H2), NoLookup)).Status); + Assert.Equal(AuditChainHeadStatus.Purged, + (await AuditChainHeadComparer.CompareAsync(Exported(2, H1), Head(9, H2, 4, H2), NoLookup)).Status); + } + + /// A record whose hash is not 32 bytes of hex can never match. + /// The exported hash text. + /// A task that represents the asynchronous operation. + [Theory] + [InlineData("abc")] + [InlineData("zz0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f")] + public async Task MalformedExportedHash_IsMismatch(string hash) + { + var record = Exported(3, H1) with { LastHash = hash }; + Assert.Equal(AuditChainHeadStatus.Mismatch, + (await AuditChainHeadComparer.CompareAsync(record, Head(3, H1), NoLookup)).Status); + } + + /// Only the tamper verdicts count as failures. + /// The verdict. + /// Whether it is a failure. + [Theory] + [InlineData(AuditChainHeadStatus.Matches, false)] + [InlineData(AuditChainHeadStatus.Extended, false)] + [InlineData(AuditChainHeadStatus.Purged, false)] + [InlineData(AuditChainHeadStatus.Mismatch, true)] + [InlineData(AuditChainHeadStatus.Regressed, true)] + [InlineData(AuditChainHeadStatus.MissingHead, true)] + [InlineData(AuditChainHeadStatus.MissingLink, true)] + [InlineData(AuditChainHeadStatus.CountMismatch, true)] + public void IsFailure_IsTrueOnlyForTamperVerdicts(AuditChainHeadStatus status, bool failure) + { + Assert.Equal(failure, new AuditChainHeadCheck("2026-05", 1, DateTime.UtcNow, null, status, 1, "").IsFailure); + } +} diff --git a/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditChainHeadStoreTests.cs b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditChainHeadStoreTests.cs new file mode 100644 index 00000000..af79f3de --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditChainHeadStoreTests.cs @@ -0,0 +1,312 @@ +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; +using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance; +using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories; +using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations; + +namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Maintenance; + +/// +/// SQL Server tests for the chain-head export store (ScadaBridge#107 item 1): heads read with +/// their counts, and exported heads checked against the database. The key case is the forgery +/// the export exists for: rows rewritten AND every later link and the head recomputed — which +/// verify-chain accepts — is caught because the exported hash no longer matches. +/// +/// Each test owns one month (partition) of the shared fixture database. +public sealed class AuditChainHeadStoreTests : IClassFixture +{ + private readonly MsSqlMigrationFixture _fixture; + + /// Creates the test class with the shared migrated SQL Server fixture. + /// The shared migration fixture. + public AuditChainHeadStoreTests(MsSqlMigrationFixture fixture) => _fixture = fixture; + + /// The head snapshot carries the chain end and the live row + tombstone counts. + /// A task that represents the asynchronous operation. + [SkippableFact] + public async Task ReadHeads_ReportsTheChainEnd_AndRowAndTombstoneCounts() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + var month = Month(2026, 3); + await InsertAsync(Enumerable.Range(1, 5).Select(i => NewEvent(month.AddDays(i)))); + await InsertAsync([NewEvent(month.AddHours(1), AuditChannel.DbOutbound)]); + await DrainAsync(); + await using (var ctx = CreateContext()) + { + Assert.Equal(1, await new AuditLogRepository(ctx).PurgeChannelOlderThanAsync("DbOutbound", month.AddDays(1), 100)); + } + + var head = await HeadAsync(month); + + Assert.Equal(6, head.LastSeq); + Assert.Equal(5, head.ChainedRows); + Assert.Equal(1, head.Tombstones); + Assert.Null(head.PurgedThroughSeq); + } + + /// + /// Unchanged chain: Matches. The chain grew: Extended. Both centrals' duplicate exports + /// collapse to one check. + /// + /// A task that represents the asynchronous operation. + [SkippableFact] + public async Task ExportedHeads_MatchTheUnchangedChain_AndAreExtendedByLaterLinks() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + var month = Month(2026, 4); + await InsertAsync(Enumerable.Range(1, 4).Select(i => NewEvent(month.AddDays(i)))); + await DrainAsync(); + var exported = Export(await HeadAsync(month)); + + var same = await VerifyAsync(exported, exported with { Node = "central-b" }); + Assert.True(same.Verified); + Assert.Equal(AuditChainHeadStatus.Matches, Assert.Single(same.Checks).Status); + + await InsertAsync([NewEvent(month.AddDays(10))]); + await DrainAsync(); + var grown = await VerifyAsync(exported); + Assert.True(grown.Verified); + var check = Assert.Single(grown.Checks); + Assert.Equal(AuditChainHeadStatus.Extended, check.Status); + Assert.Equal(5, check.DatabaseSeq); + } + + /// + /// The forgery: a chained row's content is rewritten and every link from it plus the head + /// are recomputed with the real hasher. VerifyMonthAsync reports the forged chain + /// Intact (positive control: the in-database chain alone cannot see it); the exported head + /// does not match it. + /// + /// A task that represents the asynchronous operation. + [SkippableFact] + public async Task ARewrittenAndRehashedChain_PassesVerifyChain_ButFailsTheExportedHead() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + var month = Month(2026, 5); + await InsertAsync(Enumerable.Range(1, 6).Select(i => NewEvent(month.AddDays(i)))); + await DrainAsync(); + var exported = Export(await HeadAsync(month)); + + await ForgeChainAsync(month, fromSeq: 3); + + await using (var ctx = CreateContext()) + { + var chain = await new AuditLogHashChain(ctx).VerifyMonthAsync(month); + Assert.Equal(AuditChainStatus.Intact, chain.Status); + } + + var result = await VerifyAsync(exported); + Assert.False(result.Verified); + Assert.Equal(AuditChainHeadStatus.Mismatch, Assert.Single(result.Checks).Status); + } + + /// Each structural tampering maps to its own verdict. + /// A task that represents the asynchronous operation. + [SkippableFact] + public async Task TamperedHeadsAndRows_AreReported_WithTheirOwnVerdicts() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + + // Head rolled back below the export (tail deletion + head rewrite). + var regressed = Month(2026, 6); + await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(regressed.AddDays(i)))); + await DrainAsync(); + var regressedExport = Export(await HeadAsync(regressed)); + await ExecAsync("UPDATE dbo.AuditLogChainHead SET LastSeq = LastSeq - 1 WHERE ChainMonth = @m;", regressed); + + // Head row deleted. + var missingHead = Month(2026, 7); + await InsertAsync([NewEvent(missingHead.AddDays(1))]); + await DrainAsync(); + var missingHeadExport = Export(await HeadAsync(missingHead)); + await ExecAsync("DELETE FROM dbo.AuditLogChainHead WHERE ChainMonth = @m;", missingHead); + + // A chained row deleted with no tombstone, head unchanged: counts no longer add up. + var counted = Month(2026, 8); + await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(counted.AddDays(i)))); + await DrainAsync(); + var countedExport = Export(await HeadAsync(counted)); + await ExecAsync("DELETE FROM dbo.AuditLog WHERE ChainSeq = 2 AND OccurredAtUtc >= @m AND OccurredAtUtc < DATEADD(MONTH, 1, @m);", counted); + + // The exported link itself deleted after the chain grew past it. + var missingLink = Month(2026, 9); + await InsertAsync(Enumerable.Range(1, 2).Select(i => NewEvent(missingLink.AddDays(i)))); + await DrainAsync(); + var missingLinkExport = Export(await HeadAsync(missingLink)); + await InsertAsync([NewEvent(missingLink.AddDays(5))]); + await DrainAsync(); + await ExecAsync("DELETE FROM dbo.AuditLog WHERE ChainSeq = 2 AND OccurredAtUtc >= @m AND OccurredAtUtc < DATEADD(MONTH, 1, @m);", missingLink); + + var result = await VerifyAsync(regressedExport, missingHeadExport, countedExport, missingLinkExport); + + Assert.False(result.Verified); + var byMonth = result.Checks.ToDictionary(c => c.Month, c => c.Status); + Assert.Equal(AuditChainHeadStatus.Regressed, byMonth["2026-06"]); + Assert.Equal(AuditChainHeadStatus.MissingHead, byMonth["2026-07"]); + Assert.Equal(AuditChainHeadStatus.CountMismatch, byMonth["2026-08"]); + Assert.Equal(AuditChainHeadStatus.MissingLink, byMonth["2026-09"]); + } + + /// + /// Sanctioned purges are not tamper evidence: a per-channel purge (row to tombstone) keeps + /// Matches; after a partition switch the exported head is the anchor, so the check still + /// passes, and an older exported sequence the switch removed is reported Purged. + /// + /// A task that represents the asynchronous operation. + [SkippableFact] + public async Task SanctionedPurges_AreNotFailures() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + var month = Month(2026, 10); + await InsertAsync(Enumerable.Range(1, 3).Select(i => NewEvent(month.AddDays(i)))); + await DrainAsync(); + var early = Export(await HeadAsync(month)); + await InsertAsync([NewEvent(month.AddDays(5), AuditChannel.Notification)]); + await DrainAsync(); + var late = Export(await HeadAsync(month)); + + await using (var ctx = CreateContext()) + { + Assert.Equal(1, await new AuditLogRepository(ctx).PurgeChannelOlderThanAsync("Notification", month.AddDays(6), 100)); + } + + Assert.Equal(AuditChainHeadStatus.Matches, Assert.Single((await VerifyAsync(late)).Checks).Status); + + await using (var ctx = CreateContext()) + { + await new AuditLogRepository(ctx).SwitchOutPartitionAsync(month); + } + + var afterSwitch = await VerifyAsync(late, early); + Assert.True(afterSwitch.Verified); + var statuses = afterSwitch.Checks.ToDictionary(c => c.ExportedSeq, c => c.Status); + Assert.Equal(AuditChainHeadStatus.Matches, statuses[4]); + Assert.Equal(AuditChainHeadStatus.Purged, statuses[3]); + } + + private static AuditChainHeadRecord Export(AuditChainHeadSnapshot head) => new( + AuditChainHeadRecord.CurrentVersion, + AuditChainHasher.PartitionLabel(head.Month), + head.LastSeq, + AuditChainHeadComparer.ToHex(head.LastHash), + head.PurgedThroughSeq, + head.PurgedThroughHash is null ? null : AuditChainHeadComparer.ToHex(head.PurgedThroughHash), + head.ChainedRows, + head.Tombstones, + head.UpdatedAtUtc, + DateTime.UtcNow, + "central-a"); + + private async Task HeadAsync(DateTime month) + { + await using var ctx = CreateContext(); + var heads = await new AuditLogHashChain(ctx).ReadHeadsAsync(); + return Assert.Single(heads, h => h.Month == month); + } + + private async Task VerifyAsync(params AuditChainHeadRecord[] records) + { + await using var ctx = CreateContext(); + return await new AuditLogHashChain(ctx).VerifyExportedHeadsAsync(records); + } + + /// + /// Rewrites the row at and re-links it and every later row plus the + /// head with the production hasher — what a principal with full write access could do. + /// + private async Task ForgeChainAsync(DateTime month, long fromSeq) + { + await using var conn = _fixture.OpenConnection(); + await using (var edit = new SqlCommand( + "UPDATE dbo.AuditLog SET Target = N'forged' WHERE ChainSeq = @s AND OccurredAtUtc >= @m AND OccurredAtUtc < DATEADD(MONTH, 1, @m);", conn)) + { + edit.Parameters.AddWithValue("@s", fromSeq); + edit.Parameters.Add(new SqlParameter("@m", System.Data.SqlDbType.DateTime2) { Scale = 7, Value = month }); + Assert.Equal(1, await edit.ExecuteNonQueryAsync()); + } + + var rows = new List(); + await using (var read = new SqlCommand( + $"SELECT {AuditLogHashChain.RowColumns}, ChainSeq, RowHash FROM dbo.AuditLog " + + "WHERE ChainSeq IS NOT NULL AND OccurredAtUtc >= @m AND OccurredAtUtc < DATEADD(MONTH, 1, @m) ORDER BY ChainSeq;", conn)) + { + read.Parameters.Add(new SqlParameter("@m", System.Data.SqlDbType.DateTime2) { Scale = 7, Value = month }); + await using var reader = await read.ExecuteReaderAsync(); + while (await reader.ReadAsync()) + { + rows.Add(AuditLogHashChain.ReadRow(reader, withLink: true)); + } + } + + var previous = rows.Single(r => r.ChainSeq == fromSeq - 1).RowHash!; + foreach (var row in rows.Where(r => r.ChainSeq >= fromSeq)) + { + var hash = AuditChainHasher.ComputeRowHash(previous, month, row.ChainSeq!.Value, row); + await using var update = new SqlCommand("UPDATE dbo.AuditLog SET RowHash = @h WHERE EventId = @e AND OccurredAtUtc = @o;", conn); + update.Parameters.AddWithValue("@h", hash); + update.Parameters.AddWithValue("@e", row.EventId); + update.Parameters.Add(new SqlParameter("@o", System.Data.SqlDbType.DateTime2) { Scale = 7, Value = row.OccurredAtUtc }); + Assert.Equal(1, await update.ExecuteNonQueryAsync()); + previous = hash; + } + + await using var head = new SqlCommand("UPDATE dbo.AuditLogChainHead SET LastHash = @h WHERE ChainMonth = @m;", conn); + head.Parameters.AddWithValue("@h", previous); + head.Parameters.Add(new SqlParameter("@m", System.Data.SqlDbType.Date) { Value = month }); + Assert.Equal(1, await head.ExecuteNonQueryAsync()); + } + + private async Task ExecAsync(string sql, DateTime month) + { + await using var conn = _fixture.OpenConnection(); + await using var cmd = new SqlCommand(sql, conn); + cmd.Parameters.Add(new SqlParameter("@m", System.Data.SqlDbType.DateTime2) { Scale = 7, Value = month }); + Assert.True(await cmd.ExecuteNonQueryAsync() >= 1); + } + + private async Task InsertAsync(IEnumerable events) + { + await using var ctx = CreateContext(); + var repo = new AuditLogRepository(ctx); + foreach (var e in events) + { + await repo.InsertIfNotExistsAsync(e); + } + } + + private async Task DrainAsync() + { + await using var ctx = CreateContext(); + var chain = new AuditLogHashChain(ctx); + for (var i = 0; i < 1000; i++) + { + var r = await chain.ChainPendingAsync(500); + if (r.RowsChained == 0 && !r.LockBusy) + { + return; + } + } + + throw new InvalidOperationException("Chain did not drain."); + } + + private static DateTime Month(int year, int month) => new(year, month, 1, 0, 0, 0, DateTimeKind.Utc); + + private ScadaBridgeDbContext CreateContext() + => new(new DbContextOptionsBuilder().UseSqlServer(_fixture.ConnectionString).Options); + + private static AuditEvent NewEvent(DateTime occurredAtUtc, AuditChannel channel = AuditChannel.ApiOutbound) => + ScadaBridgeAuditEventFactory.Create( + channel: channel, + kind: channel == AuditChannel.DbOutbound ? AuditKind.DbWrite : AuditKind.ApiCall, + status: AuditStatus.Delivered, + occurredAtUtc: occurredAtUtc, + actor: "head-test", + target: "erp", + sourceNode: "central-a", + sourceSiteId: "head-test", + correlationId: Guid.NewGuid()); +} diff --git a/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditMaintenanceLoginTests.cs b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditMaintenanceLoginTests.cs new file mode 100644 index 00000000..29a9afe0 --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests/Maintenance/AuditMaintenanceLoginTests.cs @@ -0,0 +1,200 @@ +using Microsoft.Data.SqlClient; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Enums; +using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Maintenance; +using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories; +using ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Migrations; + +namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Tests.Maintenance; + +/// +/// The hardened two-login setup (ScadaBridge#107 item 3), end to end on SQL Server: a writer +/// login (db_datareader + db_datawriter + scadabridge_audit_writer) and a +/// maintenance login (scadabridge_audit_chainer + scadabridge_audit_purger only), +/// wired through AddConfigurationDatabase(writer, maintenance). The writer ingests but +/// cannot chain, update or delete; the maintenance services run on the maintenance login and +/// can chain, verify, export heads, per-channel purge, enumerate partitions, archive and switch. +/// +public sealed class AuditMaintenanceLoginTests : IClassFixture, IAsyncLifetime +{ + private const string Password = "Sb_W22_maint#2026"; + + private readonly MsSqlMigrationFixture _fixture; + private readonly string _suffix = Guid.NewGuid().ToString("N")[..10]; + private readonly DirectoryInfo _archiveDir = Directory.CreateTempSubdirectory("maint-login-archive-"); + + /// Creates the test class with its own migrated SQL Server database. + /// The per-class migration fixture. + public AuditMaintenanceLoginTests(MsSqlMigrationFixture fixture) => _fixture = fixture; + + private string WriterLogin => "sbw_" + _suffix; + + private string MaintenanceLogin => "sbm_" + _suffix; + + /// + public async Task InitializeAsync() + { + if (!_fixture.Available) + { + return; + } + + await using var conn = _fixture.OpenConnection(); + await using var cmd = new SqlCommand($@" +CREATE LOGIN [{WriterLogin}] WITH PASSWORD = N'{Password}', CHECK_POLICY = OFF; +CREATE LOGIN [{MaintenanceLogin}] WITH PASSWORD = N'{Password}', CHECK_POLICY = OFF; +CREATE USER [{WriterLogin}] FOR LOGIN [{WriterLogin}]; +CREATE USER [{MaintenanceLogin}] FOR LOGIN [{MaintenanceLogin}]; +ALTER ROLE db_datareader ADD MEMBER [{WriterLogin}]; +ALTER ROLE db_datawriter ADD MEMBER [{WriterLogin}]; +ALTER ROLE scadabridge_audit_writer ADD MEMBER [{WriterLogin}]; +ALTER ROLE scadabridge_audit_chainer ADD MEMBER [{MaintenanceLogin}]; +ALTER ROLE scadabridge_audit_purger ADD MEMBER [{MaintenanceLogin}];", conn); + await cmd.ExecuteNonQueryAsync(); + } + + /// + public async Task DisposeAsync() + { + _archiveDir.Delete(recursive: true); + if (!_fixture.Available) + { + return; + } + + SqlConnection.ClearAllPools(); + await using var conn = _fixture.OpenConnection(); + await using var cmd = new SqlCommand($@" +DECLARE @kill nvarchar(max) = N''; +SELECT @kill = @kill + N'KILL ' + CAST(session_id AS nvarchar(10)) + N';' FROM sys.dm_exec_sessions + WHERE login_name IN (N'{WriterLogin}', N'{MaintenanceLogin}'); +EXEC sp_executesql @kill; +IF USER_ID(N'{WriterLogin}') IS NOT NULL DROP USER [{WriterLogin}]; +IF USER_ID(N'{MaintenanceLogin}') IS NOT NULL DROP USER [{MaintenanceLogin}]; +IF SUSER_ID(N'{WriterLogin}') IS NOT NULL DROP LOGIN [{WriterLogin}]; +IF SUSER_ID(N'{MaintenanceLogin}') IS NOT NULL DROP LOGIN [{MaintenanceLogin}];", conn); + await cmd.ExecuteNonQueryAsync(); + } + + /// + /// Writer ingests and is refused every maintenance write; the maintenance services, resolved + /// from DI on the separate login, do the whole chain + purge + archive cycle. + /// + /// A task that represents the asynchronous operation. + [SkippableFact] + public async Task HardenedTwoLoginSetup_WriterIngestsOnly_MaintenanceLoginRunsChainAndPurges() + { + Skip.IfNot(_fixture.Available, _fixture.SkipReason); + var writerCs = As(WriterLogin); + var maintenanceCs = As(MaintenanceLogin); + var month = new DateTime(2027, 2, 1, 0, 0, 0, DateTimeKind.Utc); + + // The writer login ingests. + await using (var writerCtx = Context(writerCs)) + { + var repo = new AuditLogRepository(writerCtx); + foreach (var i in Enumerable.Range(1, 4)) + { + await repo.InsertIfNotExistsAsync(NewEvent(month.AddDays(i), AuditChannel.ApiOutbound)); + } + + await repo.InsertIfNotExistsAsync(NewEvent(month.AddHours(2), AuditChannel.DbOutbound)); + } + + // Negative controls: the writer login can neither chain nor delete. + await using (var writerCtx = Context(writerCs)) + { + await Assert.ThrowsAsync(() => new AuditLogHashChain(writerCtx).ChainPendingAsync(100)); + await Assert.ThrowsAsync(() => + new AuditLogRepository(writerCtx).PurgeChannelOlderThanAsync("DbOutbound", month.AddDays(1), 100)); + } + + var services = new ServiceCollection(); + services.AddLogging(); + services.AddConfigurationDatabase(writerCs, maintenanceCs); + services.AddSingleton(Options.Create(new AuditLogArchiveOptions + { + Enabled = true, + Directory = _archiveDir.FullName, + RowGroupSize = 1_000, + })); + await using var provider = services.BuildServiceProvider(); + + await using (var scope = provider.CreateAsyncScope()) + { + var maintenance = scope.ServiceProvider.GetRequiredService(); + Assert.True(maintenance.UsesSeparateLogin); + Assert.Equal(MaintenanceLogin, await LoginNameAsync(maintenance.Context)); + Assert.Equal(WriterLogin, await LoginNameAsync(scope.ServiceProvider.GetRequiredService())); + + var chain = scope.ServiceProvider.GetRequiredService(); + Assert.Equal(5, (await chain.ChainPendingAsync(100)).RowsChained); + Assert.Equal(AuditChainStatus.Intact, (await chain.VerifyMonthAsync(month)).Status); + var head = Assert.Single(await scope.ServiceProvider.GetRequiredService().ReadHeadsAsync()); + Assert.Equal(5, head.LastSeq); + + var repo = scope.ServiceProvider.GetRequiredKeyedService(AuditMaintenanceContext.ServiceKey); + Assert.Equal(1, await repo.PurgeChannelOlderThanAsync("DbOutbound", month.AddDays(1), 100)); + Assert.Contains(month, await repo.GetPartitionBoundariesOlderThanAsync(month.AddYears(5))); + + var archive = scope.ServiceProvider.GetRequiredKeyedService(AuditMaintenanceContext.ServiceKey); + var manifest = await archive.ArchivePartitionAsync(month); + Assert.Equal(4, manifest.RowCount); + Assert.Equal(4, await archive.PurgeArchivedPartitionAsync(manifest)); + Assert.Equal(AuditChainStatus.Intact, (await chain.VerifyMonthAsync(month)).Status); + } + + // Unset maintenance string: the maintenance context is the runtime context, as before. + var plain = new ServiceCollection(); + plain.AddLogging(); + plain.AddConfigurationDatabase(writerCs); + await using var plainProvider = plain.BuildServiceProvider(); + await using var plainScope = plainProvider.CreateAsyncScope(); + var shared = plainScope.ServiceProvider.GetRequiredService(); + Assert.False(shared.UsesSeparateLogin); + Assert.Same(plainScope.ServiceProvider.GetRequiredService(), shared.Context); + } + + private string As(string login) => new SqlConnectionStringBuilder(_fixture.ConnectionString) + { + UserID = login, + Password = Password, + IntegratedSecurity = false, + }.ConnectionString; + + private static ScadaBridgeDbContext Context(string cs) + => new(new DbContextOptionsBuilder().UseSqlServer(cs).Options); + + private static async Task LoginNameAsync(ScadaBridgeDbContext ctx) + { + var conn = ctx.Database.GetDbConnection(); + await conn.OpenAsync(); + try + { + await using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT SUSER_SNAME();"; + return (string)(await cmd.ExecuteScalarAsync())!; + } + finally + { + await conn.CloseAsync(); + } + } + + private static AuditEvent NewEvent(DateTime occurredAtUtc, AuditChannel channel) => + ScadaBridgeAuditEventFactory.Create( + channel: channel, + kind: channel == AuditChannel.DbOutbound ? AuditKind.DbWrite : AuditKind.ApiCall, + status: AuditStatus.Delivered, + occurredAtUtc: occurredAtUtc, + actor: "maint-login-test", + target: "erp", + sourceNode: "central-a", + sourceSiteId: "maint-login-test", + correlationId: Guid.NewGuid()); +} diff --git a/tests/ZB.MOM.WW.ScadaBridge.ManagementService.Tests/RequiredRoleMatrixTests.cs b/tests/ZB.MOM.WW.ScadaBridge.ManagementService.Tests/RequiredRoleMatrixTests.cs index afd85b6a..2999eec3 100644 --- a/tests/ZB.MOM.WW.ScadaBridge.ManagementService.Tests/RequiredRoleMatrixTests.cs +++ b/tests/ZB.MOM.WW.ScadaBridge.ManagementService.Tests/RequiredRoleMatrixTests.cs @@ -171,6 +171,8 @@ public class RequiredRoleMatrixTests // AuditLog hash-chain verification (ScadaBridge#41) carries audit query's // OperationalAudit permission (alog.md §15.1); written out by hand to keep the freeze. ["VerifyAuditChain"] = [Roles.Administrator, Roles.Viewer], + // Off-box exported chain heads (ScadaBridge#107): same audit-read gate. + ["VerifyAuditChainHeads"] = [Roles.Administrator, Roles.Viewer], // AuditLog Parquet archives (ScadaBridge#42): same audit-read gate as verify-chain. ["ListAuditArchives"] = [Roles.Administrator, Roles.Viewer], ["VerifyAuditArchive"] = [Roles.Administrator, Roles.Viewer], diff --git a/tests/ZB.MOM.WW.ScadaBridge.ManagementService.Tests/VerifyAuditChainHeadsCommandTests.cs b/tests/ZB.MOM.WW.ScadaBridge.ManagementService.Tests/VerifyAuditChainHeadsCommandTests.cs new file mode 100644 index 00000000..b6d0247e --- /dev/null +++ b/tests/ZB.MOM.WW.ScadaBridge.ManagementService.Tests/VerifyAuditChainHeadsCommandTests.cs @@ -0,0 +1,110 @@ +using System.Text.Json; +using Akka.Actor; +using Akka.TestKit.Xunit2; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using NSubstitute; +using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories; +using ZB.MOM.WW.ScadaBridge.Commons.Messages.Management; +using ZB.MOM.WW.ScadaBridge.Commons.Types.Audit; + +namespace ZB.MOM.WW.ScadaBridge.ManagementService.Tests; + +/// +/// The VerifyAuditChainHeads management command (ScadaBridge#107 item 1): role gate, +/// site-scope refusal, input bounds, and the verdicts reaching the wire with enums as names. +/// +public class VerifyAuditChainHeadsCommandTests : TestKit +{ + private readonly IAuditChainHeadStore _store = Substitute.For(); + + private static readonly AuditChainHeadRecord Record = new( + 1, "2026-05", 12, new string('a', 64), null, null, 12, 0, + new DateTime(2026, 5, 3, 0, 0, 0, DateTimeKind.Utc), new DateTime(2026, 5, 3, 1, 0, 0, DateTimeKind.Utc), "central-a"); + + private IActorRef CreateActor() + { + var services = new ServiceCollection().AddTestConfigurationTransaction(); + services.AddScoped(_ => _store); + return Sys.ActorOf(Props.Create(() => new ManagementActor( + services.BuildServiceProvider(), NullLogger.Instance))); + } + + private static ManagementEnvelope Envelope(object command, string[] sites, params string[] roles) => + new(new AuthenticatedUser("u", "U", roles, sites), command, Guid.NewGuid().ToString("N")); + + /// A Viewer gets the store's verdicts; the records reach the store unchanged. + [Fact] + public void Viewer_GetsTheVerdicts_ForTheSuppliedRecords() + { + var check = new AuditChainHeadCheck("2026-05", 12, Record.ExportedAtUtc, "central-a", + AuditChainHeadStatus.Mismatch, 12, "rewritten"); + _store.VerifyExportedHeadsAsync(Arg.Any>(), Arg.Any()) + .Returns(new AuditChainHeadVerificationResult(false, 1, [check])); + + CreateActor().Tell(Envelope(new VerifyAuditChainHeadsCommand([Record]), [], "Viewer")); + + var response = ExpectMsg(TimeSpan.FromSeconds(5)); + using var json = JsonDocument.Parse(response.JsonData); + Assert.False(json.RootElement.GetProperty("verified").GetBoolean()); + Assert.Equal("Mismatch", json.RootElement.GetProperty("checks")[0].GetProperty("status").GetString()); + _store.Received(1).VerifyExportedHeadsAsync( + Arg.Is>(r => r.Count == 1 && r[0] == Record), Arg.Any()); + } + + /// A Designer/Deployer without an OperationalAudit role is refused. + [Fact] + public void NonAuditRole_IsUnauthorized() + { + CreateActor().Tell(Envelope(new VerifyAuditChainHeadsCommand([Record]), [], "Designer", "Deployer")); + + ExpectMsg(TimeSpan.FromSeconds(5)); + _store.DidNotReceiveWithAnyArgs().VerifyExportedHeadsAsync(default!, default); + } + + /// A site-scoped Viewer is refused: the chain spans every site. + [Fact] + public void SiteScopedViewer_IsRefused() + { + CreateActor().Tell(Envelope(new VerifyAuditChainHeadsCommand([Record]), ["3"], "Viewer")); + + var response = ExpectMsg(TimeSpan.FromSeconds(5)); + Assert.Contains("spans every site", response.Message); + _store.DidNotReceiveWithAnyArgs().VerifyExportedHeadsAsync(default!, default); + } + + /// No records, or more than the cap, is a caller error. + /// Records supplied. + [Theory] + [InlineData(0)] + [InlineData(ManagementActor.MaxChainHeadRecords + 1)] + public void EmptyOrOversizedInput_IsInvalidArgument(int count) + { + CreateActor().Tell(Envelope( + new VerifyAuditChainHeadsCommand(Enumerable.Repeat(Record, count).ToList()), [], "Administrator")); + + var response = ExpectMsg(TimeSpan.FromSeconds(5)); + Assert.Equal("INVALID_ARGUMENT", response.ErrorCode); + _store.DidNotReceiveWithAnyArgs().VerifyExportedHeadsAsync(default!, default); + } + + /// A chain lock that cannot be had surfaces as CHAIN_BUSY. + [Fact] + public void BusyChain_IsChainBusy() + { + _store.VerifyExportedHeadsAsync(Arg.Any>(), Arg.Any()) + .Returns>(_ => throw new TimeoutException("busy")); + + CreateActor().Tell(Envelope(new VerifyAuditChainHeadsCommand([Record]), [], "Administrator")); + + Assert.Equal("CHAIN_BUSY", ExpectMsg(TimeSpan.FromSeconds(5)).ErrorCode); + } + + /// Runs under the long-running Ask timeout. + [Fact] + public void VerifyAuditChainHeads_UsesLongRunningAskTimeout() + { + Assert.Equal(TimeSpan.FromMinutes(5), + ManagementEndpoints.ResolveAskTimeout(new ManagementServiceOptions(), typeof(VerifyAuditChainHeadsCommand))); + } +}