Compare commits
89 Commits
1429ddaa0e
...
1930f19b1a
| Author | SHA1 | Date | |
|---|---|---|---|
| 1930f19b1a | |||
| 53889aec9f | |||
| ccee558ca7 | |||
| ca4b0dd849 | |||
| 699af8fc16 | |||
| 84fdea60cf | |||
| 3ad5ad0d7e | |||
| 774e3918db | |||
| b7f5470666 | |||
| 9b8ce0f51e | |||
| 2751a6dba9 | |||
| 1b88ca1296 | |||
| 5b16429635 | |||
| 56d39b5fce | |||
| 4f39cfd6c6 | |||
| 80b967681e | |||
| 76ef97f729 | |||
| 7943d73e71 | |||
| 925c869826 | |||
| d33337a834 | |||
| 973e59de84 | |||
| 1fcfa4fb16 | |||
| 80e5b36852 | |||
| 4a0462e4d0 | |||
| 92936747b7 | |||
| c2707fdcf8 | |||
| 9b9222d223 | |||
| 11efe4be05 | |||
| e4765b2eed | |||
| 24c2097ec3 | |||
| f85f036b09 | |||
| b532a1e30e | |||
| 65e0f770d1 | |||
| e0dddac0be | |||
| e1a692016e | |||
| 30356803e7 | |||
| d536450611 | |||
| 8ddcbbd6f4 | |||
| badc97afe8 | |||
| c573d8cac6 | |||
| c83b51fe9c | |||
| 8f46b6cec2 | |||
| 41ffe42de3 | |||
| 3d6973cc89 | |||
| 4e228110ec | |||
| 439dee52d3 | |||
| b7f8632db8 | |||
| c29e588905 | |||
| cd93ad3976 | |||
| bb8be55382 | |||
| 1c8b2bc745 | |||
| cc4ff7029d | |||
| 9a1b160e79 | |||
| 61442b5f81 | |||
| 3ee98c8af6 | |||
| 54417e1ae4 | |||
| 6e1ab62d27 | |||
| 9b4890326e | |||
| db623c1435 | |||
| 7b95ef3752 | |||
| 91bde0fae6 | |||
| 84bc2380f3 | |||
| b02c15bbca | |||
| 350d530135 | |||
| fb9dc7ad14 | |||
| 3b9f6b9572 | |||
| 3f937b399b | |||
| 128d2bab73 | |||
| b41b3b1b31 | |||
| 1cc05f132f | |||
| f3bec6922a | |||
| cc94b3a68c | |||
| 26dce8b69f | |||
| 15018e4158 | |||
| ab77094a26 | |||
| 9668600c38 | |||
| db0285e2de | |||
| f149626a76 | |||
| 31fa5eafa4 | |||
| 3a5b885a44 | |||
| d233ecbe8f | |||
| 6e0fa21307 | |||
| 06d79f6516 | |||
| f25812fc7c | |||
| 5573a9eeb9 | |||
| a1c7c4ef26 | |||
| 91209070d0 | |||
| 90cdb6e51a | |||
| 29c1286943 |
@@ -44,5 +44,14 @@ docker-env2/*/data/
|
||||
# Local credentials / login captures — never commit
|
||||
*login*.txt
|
||||
|
||||
# Ad-hoc scratch/credential captures at the repo root — never commit
|
||||
# (arch-review 08 round 2 NF1: a live API key sat in an untracked test.txt)
|
||||
/test.txt
|
||||
/*credentials*
|
||||
/*secret*
|
||||
|
||||
# Claude tool worktrees (NF7)
|
||||
.claire/
|
||||
|
||||
# Sister-project deployment artifacts (not part of this solution)
|
||||
/deploy/
|
||||
|
||||
+34
-2
@@ -4,13 +4,45 @@ All notable changes to ScadaBridge are documented in this file.
|
||||
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
|
||||
|
||||
Detailed per-change history lives in `git log`,
|
||||
`archreview/plans/00-MASTER-TRACKER.md`, and `docs/plans/` — this file records
|
||||
operator-facing and breaking changes only.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Changed — Architecture-review hardening initiative (PLAN-01…PLAN-08, 2026-07-08 → 2026-07-10)
|
||||
|
||||
Eight architecture reviews produced eight fix plans; 191 of 192 tasks landed. The
|
||||
initiative added eager startup options-validation across 12+ components, ClusterClient
|
||||
message-contract lock tests, secured-write pending-TTL, a login throttle, transport
|
||||
import single-flight, store-and-forward sweep bounding, and a deploy-readiness
|
||||
handshake; it excised the vestigial site-notification delivery surface and established
|
||||
a canonical deferred-work register. Authoritative record:
|
||||
`archreview/plans/00-MASTER-TRACKER.md` plus the per-plan docs.
|
||||
|
||||
### Added — M8 Transport
|
||||
|
||||
Encrypted bundle export/import extended from central-only config to site/instance-scoped
|
||||
config (`Site`s, site `DataConnection`s, `Instance`s + `Area` by name), reconciled across
|
||||
environments by the `BundleNameMap` name-mapping subsystem, with a per-line Myers diff and
|
||||
the script trust gate enforced at import review. Post-initiative additions (both 2026-07-10):
|
||||
an aggregated live alarm stream feeding the Alarm Summary page, and hourly KPI rollups
|
||||
unblocking 30d/90d trend windows.
|
||||
|
||||
### Added — M7 Native alarms + operational UX
|
||||
|
||||
Read-only native-alarm mirroring (OPC UA Alarms & Conditions + MxAccess Gateway) onto a
|
||||
unified condition model; an operator Alarm Summary page; OPC UA node browse/search/type-info
|
||||
and site-local certificate trust; **two-person secured writes, introducing the `Operator`
|
||||
and `Verifier` roles**; and an SMS (Twilio) notification delivery adapter.
|
||||
|
||||
### Changed — BREAKING: canonical role names + audit separation-of-duties collapse (Task 1.7)
|
||||
|
||||
Role string VALUES are standardized onto the canonical vocabulary
|
||||
(`Administrator`/`Designer`/`Deployer`/`Viewer`; `Operator`/`Engineer` are unused
|
||||
by ScadaBridge). The legacy ScadaBridge role names were renamed and two were
|
||||
(`Administrator`/`Designer`/`Deployer`/`Viewer`; `Engineer` remains unused by
|
||||
ScadaBridge; `Operator` has since been adopted by the M7 two-person secured-write
|
||||
flow, together with the new `Verifier` role — see the M7 entry above). The legacy
|
||||
ScadaBridge role names were renamed and two were
|
||||
**collapsed**:
|
||||
|
||||
| Legacy role | Canonical role | Notes |
|
||||
|
||||
@@ -95,7 +95,7 @@ Other peers in the `scadaproj` family (see `scadaproj/CLAUDE.md` for details): `
|
||||
21. Notification Outbox — Central component ingesting store-and-forwarded notifications, `Notifications` audit table, dispatcher loop, retry/parking, delivery KPIs.
|
||||
22. Site Call Audit — Central component auditing site cached calls (`CachedCall`/`CachedWrite`); `SiteCalls` audit table, telemetry ingest, reconciliation, KPIs, central→site Retry/Discard relay; sites remain the source of truth.
|
||||
23. Audit Log — Central append-only AuditLog table spanning every script-trust-boundary action (outbound API sync+cached, outbound DB sync+cached, notifications, inbound API). Site SQLite hot-path + gRPC telemetry + reconciliation; combined telemetry with Site Call Audit; central direct-write for Notification Outbox dispatch + Inbound API; monthly partitioning, 365-day retention.
|
||||
24. Transport — File-based, encrypted bundle export/import via Central UI. Templates (with **all** child collections — attributes/alarms/scripts/compositions/**native-alarm-sources**, field-complete incl. `LockedInDerived` + script cadence/timeout), system artifacts, central-only configuration, plus site/instance-scoped config (`Site`s, site `DataConnection`s, `Instance`s + real `AreaName` by name) reconciled across environments by a `BundleNameMap` name-mapping subsystem. Per-conflict resolution with a per-line Myers diff. Runs the **script trust gate** at import review (5th `ScriptTrustValidator` call site — forbidden-API scripts rejected pre-runtime; template-script name-resolution findings are advisory warnings, ApiMethod findings hard-block). Correlated audit via `BundleImportId`. Never touches site runtime nodes (imported instances land `NotDeployed`).
|
||||
24. Transport — File-based, encrypted bundle export/import via Central UI. Templates (with **all** child collections — attributes/alarms/scripts/compositions/**native-alarm-sources**, field-complete incl. `LockedInDerived` + script cadence/timeout), system artifacts, central-only configuration, plus site/instance-scoped config (`Site`s, site `DataConnection`s, `Instance`s + real `AreaName` by name) reconciled across environments by a `BundleNameMap` name-mapping subsystem. Per-conflict resolution with a per-line Myers diff. Runs the **script trust gate** at import review (5th `ScriptTrustValidator` call site — forbidden-API scripts rejected pre-runtime; covers template/shared/ApiMethod bodies, template script + alarm Expression-trigger bodies, **and instance alarm-override trigger expressions** — all hard-block; template-script name-resolution findings are advisory warnings, ApiMethod findings hard-block). Correlated audit via `BundleImportId`. Never touches site runtime nodes (imported instances land `NotDeployed`).
|
||||
25. Script Analysis — Shared authoritative script-trust analyzer: unified forbidden-API deny-list (`ScriptTrustPolicy`), fused semantic+syntactic validator (`ScriptTrustValidator`), Roslyn compile wrapper (`RoslynScriptCompiler`), and compile-only globals stubs (`ScriptCompileSurface`/`TriggerCompileSurface`); consumed by Template Engine, Site Runtime, Inbound API, and Central UI.
|
||||
26. KPI History — Reusable central KPI-history backbone: tall/EAV `KpiSample` store in central MS SQL, `KpiHistoryRecorderActor` cluster singleton (`kpi-history-recorder`, not readiness-gated) sampling DI-registered `IKpiSampleSource`s every minute, bucketed query (`GetRawSeriesAsync` + `KpiSeriesBucketer`) + scoped `KpiHistoryQueryService`, and a reusable custom-SVG `KpiTrendChart`; ships trends for Notification Outbox, Site Call Audit, Audit Log, and Site Health.
|
||||
27. DelmiaNotifier — Standalone external client tool (NOT a cluster component, NOT in the Host): a compact Native-AOT (`win-x64`) console app (`WWNotifier.exe`) that DELMIA Apriso shells out to per recipe download. POSTs to the Inbound API `DelmiaRecipeDownload` method (`X-API-Key`, key from `SCADABRIDGE_API_KEY`), with connect-failure-only failover across a comma-list of base URLs, and reports the legacy `YES`/`NO` + exit-code stdout contract — a drop-in replacement for the legacy `WWNotifier` (see `docs/former-api-specs/dnc/`). Zero-dependency BCL-only, `System.Text.Json` source-gen. Project README: `src/ZB.MOM.WW.ScadaBridge.DelmiaNotifier/README.md`; design: `docs/plans/2026-06-26-delmia-recipe-notifier-design.md`.
|
||||
@@ -150,7 +150,7 @@ Other peers in the `scadaproj` family (see `scadaproj/CLAUDE.md` for details): `
|
||||
- Last-write-wins for concurrent template editing (no optimistic concurrency on templates).
|
||||
- Optimistic concurrency on deployment status records.
|
||||
- Naming collisions in composed feature modules are design-time errors.
|
||||
- Transport (#24, M8): bundles now move site/instance config (`Site`s, site-scoped `DataConnection`s, `Instance`s + override children + `Area` by name), not just central-only config — but still never touch site runtime nodes (imported instances land `NotDeployed`). A `BundleNameMap` name-mapping subsystem (Commons: `SiteMapping`/`ConnectionMapping`/`MappingAction`) reconciles environment-specific identifiers (sites by `SiteIdentifier`, connections by site-qualified `{SiteIdentifier}/{Name}`, instances by `UniqueName`): auto-match → operator override via the import-wizard Map step / CLI `--map-site`/`--map-connection`/`--create-missing-*` → blocker rows for the unresolved. D3 "carry full config (encrypted secrets)": Site addresses travel; `DataConnection` config rides the encrypted `SecretsBlock` (presence-only in diffs). Per-line Myers diff for code fields via the pure `LineDiffer` (`ArtifactDiff` embeds a size-capped structured `lineDiff`; input capped at `MaxInputLines`=4000 → summary-only, so a crafted bundle can't OOM the active node); `ImportResult.StaleInstanceIds` is real via the `IStaleInstanceProbe` seam (Commons; implemented in DeploymentManager). `schemaVersion` 1.0→1.1 (additive); `bundleFormatVersion` stays 1; no new EF tables/columns. **Arch-review 05 (plan 05):** template child fidelity completed (round-trip-guard-verified: `LockedInDerived`, native-alarm-sources, script cadence/timeout, real `AreaName`, ES retry config + methods-on-Add, notification recipients, folder parent edges); import runs the **script trust gate** (template/shared/ApiMethod bodies **and** Expression-trigger bodies) as a hard pre-runtime gate + a name-resolution severity split (template=advisory `ConflictKind.Warning`+`ImportResult.Warnings`, ApiMethod=hard error); `ArtifactDiff` resolves folder/base/composition names (unchanged structured templates classify Identical again); `MaxConcurrentImportSessions`=8 session cap. Rename does NOT rewrite call sites (deploy gate on the target is the real check). Plan-06 handoff: `ScriptArtifactsChanged`/`IScriptArtifactChangeBus` seam + Host in-process bus + Transport post-commit publisher shipped (consumers = plan 06).
|
||||
- Transport (#24, M8): bundles now move site/instance config (`Site`s, site-scoped `DataConnection`s, `Instance`s + override children + `Area` by name), not just central-only config — but still never touch site runtime nodes (imported instances land `NotDeployed`). A `BundleNameMap` name-mapping subsystem (Commons: `SiteMapping`/`ConnectionMapping`/`MappingAction`) reconciles environment-specific identifiers (sites by `SiteIdentifier`, connections by site-qualified `{SiteIdentifier}/{Name}`, instances by `UniqueName`): auto-match → operator override via the import-wizard Map step / CLI `--map-site`/`--map-connection`/`--create-missing-*` → blocker rows for the unresolved. D3 "carry full config (encrypted secrets)": Site addresses travel; `DataConnection` config rides the encrypted `SecretsBlock` (presence-only in diffs). Per-line Myers diff for code fields via the pure `LineDiffer` (`ArtifactDiff` embeds a size-capped structured `lineDiff`; input capped at `MaxInputLines`=4000 → summary-only, so a crafted bundle can't OOM the active node); `ImportResult.StaleInstanceIds` is real via the `IStaleInstanceProbe` seam (Commons; implemented in DeploymentManager). `schemaVersion` 1.0→1.1 (additive); `bundleFormatVersion` stays 1; no new EF tables/columns. **Arch-review 05 (plan 05):** template child fidelity completed (round-trip-guard-verified: `LockedInDerived`, native-alarm-sources, script cadence/timeout, real `AreaName`, ES retry config + methods-on-Add, notification recipients, folder parent edges); import runs the **script trust gate** (template/shared/ApiMethod bodies, template script + alarm Expression-trigger bodies, **and instance alarm-override trigger expressions** — R2-05) as a hard pre-runtime gate + a name-resolution severity split (template=advisory `ConflictKind.Warning`+`ImportResult.Warnings`, ApiMethod=hard error); `ArtifactDiff` resolves folder/base/composition names (unchanged structured templates classify Identical again); `MaxConcurrentImportSessions`=8 session cap (startup-validated `>0`, R2-05). **R2-05 also:** the post-commit `ScriptArtifactsChanged` publisher now covers `Add` resolutions (not just Overwrite/Rename), import warns on instance overrides targeting locked template members (advisory, rows still written), and the Expression-trigger syntax check is surface-keyed-cached + skipped on read-only staleness/comparison paths (`validateScriptCompilation:false`). Rename does NOT rewrite call sites (deploy gate on the target is the real check). Plan-06 handoff: `ScriptArtifactsChanged`/`IScriptArtifactChangeBus` seam + Host in-process bus + Transport post-commit publisher shipped (ApiMethod consumer = InboundAPI `ScriptArtifactChangeSubscriber`, wired in plan R2-06; SharedScript/Template publish with no consumer — nothing at central caches those kinds).
|
||||
|
||||
### Store-and-Forward
|
||||
- Fixed retry interval, no max buffer size. Only transient failures buffered.
|
||||
|
||||
@@ -76,7 +76,7 @@ Status values: ⬜ Not started · 🟨 In progress · ✅ Complete · ⏸ Blocke
|
||||
|
||||
**Wave 1 — PLAN-02 COMPLETE (2026-07-08):** all 24 tasks delivered and **merged to `main`** (`ce239f6f`). Store-and-Forward: active-node delivery gate (`Func<bool>` seam), bounded/short-circuiting retry sweep, upsert-based standby applies + anti-entropy peer resync, WAL+busy_timeout, epoch-ms due-check, ordered single-reader audit-observer channel; `Notify.Send` enqueue-only (`deferToSweep`, `maxRetries:0`, corrupt-payload parks not discards); gRPC channel keying + DropOldest debug-stream eviction counting + 5s application heartbeat; oldest-parked-age health field. Docker cluster rebuilt on merged image (Healthy, 5 singletons, corrected active-node semantics) and 3 sites seeded. Initiative total: **52 of 192** (sum of the Done column above).
|
||||
|
||||
**Wave 1 — PLAN-01 COMPLETE (2026-07-08):** all 23 tasks delivered on worktree branch `worktree-archreview-wave1-plan01` (**merged to `main` `ec47cb56` 2026-07-09; branch since removed**). Beyond the P0/early-Wave-1 spine (SBR downing + two-node rig T1/T3/T4; oldest-member active-node unification via `ClusterActivityEvaluator` T5–T8; `CentralSingletonRegistrar` drains T2/T9/T10; health-report acking T11–T12), Wave 1 also landed: metrics-staleness + status-transition state machine and its dashboard surface (T13/T14), deleted-site eviction from the aggregator (T18), seed-vs-metrics-port validator + `AllowSingleNodeCluster` + docker `stop_grace_period` (T15–T17), rate-limited dead-letter warnings (T19), the down-if-alone process-exit recovery loop (T20), the `failover-drill.sh` SIGKILL drill (T21), and the REQ-HOST-6 + Traefik/Host doc reconciliation with unused-Akka.Hosting-package removal (T22/T23). PLAN-01 now **23 / 23**. Initiative total: **30 of 192**. Each task TDD + pathspec-committed; full solution builds clean (0 warnings). Three deploy-artifact sub-edits (`deploy/wonder-app-vd03/` appsettings + install.ps1, Tasks 16/20/23) were deferred because that production artifact is not tracked in this repo — logged in the deferred register below.
|
||||
**Wave 1 — PLAN-01 COMPLETE (2026-07-08):** all 23 tasks delivered on worktree branch `worktree-archreview-wave1-plan01` (**merged to `main` `ec47cb56` 2026-07-09; branch since removed**). Beyond the P0/early-Wave-1 spine (SBR downing + two-node rig T1/T3/T4; oldest-member active-node unification via `ClusterActivityEvaluator` T5–T8; `CentralSingletonRegistrar` drains T2/T9/T10; health-report acking T11–T12), Wave 1 also landed: metrics-staleness + status-transition state machine and its dashboard surface (T13/T14), deleted-site eviction from the aggregator (T18), seed-vs-metrics-port validator + `AllowSingleNodeCluster` + docker `stop_grace_period` (T15–T17), rate-limited dead-letter warnings (T19), the down-if-alone process-exit recovery loop (T20), the `failover-drill.sh` SIGKILL drill (T21), and the REQ-HOST-6 + Traefik/Host doc reconciliation with unused-Akka.Hosting-package removal (T22/T23). PLAN-01 now **23 / 23**. Initiative total: **30 of 192**. Each task TDD + pathspec-committed; full solution builds clean (0 warnings). Three deploy-artifact sub-edits (`deploy/wonder-app-vd03/` appsettings + install.ps1, Tasks 16/20/23) were deferred on a mistaken not-in-repo rationale; corrected and applied on-disk by PLAN-R2-01 (2026-07-12) — see the Deferred-during-Wave-1 entry.
|
||||
|
||||
**Wave 1 — PLAN-03 COMPLETE (2026-07-09):** all 26 tasks delivered and **merged to `main`** (`c7a0c922`). Site Runtime & DCL hardening across nine TDD batches: reconnect/backoff leak fixes and the `DeployCompileValidator` compile gate, DCL adapter lifecycle + alarm-filter overwrite warning (S10) + observed adapter dispose on stop (S9), site-node cert reconcile (CertStoreActor trusted-store export → `DeploymentManagerActor` MemberUp-driven additive push), replicated config-fetch retry (`ConfigFetchRetryCount`), invariant-culture conditional-trigger fallback (C6), bounded/DropOldest event-channel comment fix (C2), and the SiteRuntime/DCL requirements-doc sync. Three low-severity items deferred with rationale (see PLAN-03 coverage table). Full CentralUI/SiteRuntime/DCL suites green. Initiative total: **78 of 192** (sum of the Done column above). Two out-of-band fixes also landed post-merge: PLAN-07's SecuredWrites `IBrowseService` bUnit-harness gap (resolved 2026-07-09, `9049690b`) — a pre-existing failure surfaced during PLAN-03's full-suite run, not a plan task.
|
||||
|
||||
@@ -84,7 +84,7 @@ Status values: ⬜ Not started · 🟨 In progress · ✅ Complete · ⏸ Blocke
|
||||
|
||||
**Wave 3 — PLAN-05 COMPLETE (2026-07-10):** all 25 tasks delivered on worktree branch `worktree-archreview-wave3-plan05` and **merged to `main` `ec23db35` via fast-forward** (main was still at PLAN-04's `dbd3ed2f`, zero divergence; worktree removed, branch deleted). Templates/Deployment/Transport hardening across seven report-and-wait waves: import correctness spine (BundleImporter consumes `BaseTemplateName`/composition/inheritance edges + import-time acyclicity, `MinTimeBetweenRuns`/`ExecutionTimeoutSeconds`/`LockedInDerived`/native-alarm-sources/real `AreaName` carried through — T1–T7), a reflection round-trip equivalence guard that surfaced and fixed 5 silent-data-loss holes (ES retry+methods-on-Add, notification recipients, folder parent edges — T8), flattener/collision-detector repeated-composition fixes (T9/T10), revision-hash + DiffService native-source coverage (T11/T12), native-source lock enforcement (T13), the `IScriptArtifactChangeBus` invalidation contract + Host in-process bus + Transport post-commit publisher (T14, plan-06 handoff), compile-verdict cache + read-only-path compile skip + post-success audit isolation + central NotDeployed-delete (T15–T18), and the closing security/quality chain: **import blocker severity split** (template-script findings advisory, ApiMethod hard — T19), **the script trust gate on bundle import** (5th `ScriptTrustValidator` call site, over script bodies *and* Expression-trigger bodies — T20, given a code-reviewer security pass that caught + fixed 2 Important gaps: expression-trigger coverage and per-origin local-declaration scoping), ScriptTrustPolicy deny-list hardening (T21), LineDiffer input cap (T22), ArtifactDiff placeholder-identity resolution (T23), the low-severity batch (session cap / full error lists / leaf-fallback warning — T24), and the design-doc + CLAUDE.md sweep (T25). **Terminal verification:** full slnx build 0 warn/0 err; the entire `dotnet test` sweep green (~5,900 unit + integration tests across every suite) plus a direct `CLI.Tests` run (341). Only residue: the same environmental 8 `CentralUI.PlaywrightTests` Deployment E2E failures (`SQLite Error 14` on `instance deploy` — needs a live docker cluster; PLAN-05 touched zero files in that project's deployment-DB path). Initiative total: **128 of 192** (sum of the Done column).
|
||||
|
||||
**Wave 4 — PLAN-06 COMPLETE (2026-07-10):** all 24 tasks delivered and **merged to `main` `3e964ac2` via fast-forward** (main was still at PLAN-05's `9b0cf44c`, zero divergence; worktree removed, branch deleted). Edge Integrations hardening across six report-and-wait waves of file-disjoint parallel subagents: the **InboundAPI** spine — revision-checked compiled-handler cache (heals failover / direct-SQL / known-bad staleness, DB row now authoritative — T1/T2), `InvalidateMethod` seam consuming PLAN-05's `IScriptArtifactChangeBus` contract (T3), deferred DI-scope disposal + orphan-execution telemetry (T4), machine-readable error `code`s incl. `SITE_UNREACHABLE` + 415-on-non-JSON (T5), `IOptionsMonitor` hot-reload (T6), parallel startup compile (T7); the **ESG** per-system `TimeoutSeconds` full vertical slice (entity→EF migration→artifact→site SQLite→client honor→mgmt→CLI→UI→doc — T8–T12) plus `{param}` path-template substitution (T13), bounded outbound response buffering (T14), structured JSON `AuthConfiguration` with legacy-colon fallback (T15), cancellation-aware `IsTransient` (T16); the **Notification** batch — SMS first-transient short-circuit (T17), wired the dead `SmsConfiguration.MaxRetries/RetryDelay` per-type retry policy (T18), configurable OAuth2 authority/scope (entity→EF migration→token service→mgmt/CLI/UI surfaces — T19/T20), deterministic lowest-Id SMTP selection + atomic OAuth2 token cache (T21), Twilio `AccountSid` format validation (T22); and the docs (DelmiaNotifier at-least-once semantics — T23; dual-key API-key rotation procedure — T24). Two EF migrations added (`AddExternalSystemTimeoutSeconds` T8, `AddSmtpOAuth2AuthorityScope` T19, serialized on the shared snapshot). **Terminal verification:** full slnx build 0 warn/0 err; every unit + integration suite green (InboundAPI 253, ESG 132, NotificationOutbox 132, NotificationService 57, ManagementService 245, CLI 346, CentralUI 893, SiteRuntime 494, Commons 597, ConfigDB 338, Host 260, + all others). Only residue: the same environmental 8 `CentralUI.PlaywrightTests` Deployment E2E failures (`SQLite Error 14` on `instance deploy` — needs a live docker cluster; PLAN-06 touched zero files in that project's deployment-DB path). Execution note: `isolation:"worktree"` subagents proved non-isolating here (shared index → commit races); the working model was pre-created dedicated `git worktree`s per task + cherry-pick onto an integration branch + per-wave combined build/impacted-suite barrier. Initiative total: **152 of 192** (sum of the Done column).
|
||||
**Wave 4 — PLAN-06 COMPLETE (2026-07-10):** all 24 tasks delivered and **merged to `main` `3e964ac2` via fast-forward** (main was still at PLAN-05's `9b0cf44c`, zero divergence; worktree removed, branch deleted). Edge Integrations hardening across six report-and-wait waves of file-disjoint parallel subagents: the **InboundAPI** spine — revision-checked compiled-handler cache (heals failover / direct-SQL / known-bad staleness, DB row now authoritative — T1/T2), `InvalidateMethod` seam (T3; the bus *subscriber* was NOT wired here — arch-review 06 round-2 N1 — and shipped later via PLAN-R2-06 T1), deferred DI-scope disposal + orphan-execution telemetry (T4), machine-readable error `code`s incl. `SITE_UNREACHABLE` + 415-on-non-JSON (T5), `IOptionsMonitor` hot-reload (T6), parallel startup compile (T7); the **ESG** per-system `TimeoutSeconds` full vertical slice (entity→EF migration→artifact→site SQLite→client honor→mgmt→CLI→UI→doc — T8–T12) plus `{param}` path-template substitution (T13), bounded outbound response buffering (T14), structured JSON `AuthConfiguration` with legacy-colon fallback (T15), cancellation-aware `IsTransient` (T16); the **Notification** batch — SMS first-transient short-circuit (T17), wired the dead `SmsConfiguration.MaxRetries/RetryDelay` per-type retry policy (T18), configurable OAuth2 authority/scope (entity→EF migration→token service→mgmt/CLI/UI surfaces — T19/T20), deterministic lowest-Id SMTP selection + atomic OAuth2 token cache (T21), Twilio `AccountSid` format validation (T22); and the docs (DelmiaNotifier at-least-once semantics — T23; dual-key API-key rotation procedure — T24). Two EF migrations added (`AddExternalSystemTimeoutSeconds` T8, `AddSmtpOAuth2AuthorityScope` T19, serialized on the shared snapshot). **Terminal verification:** full slnx build 0 warn/0 err; every unit + integration suite green (InboundAPI 253, ESG 132, NotificationOutbox 132, NotificationService 57, ManagementService 245, CLI 346, CentralUI 893, SiteRuntime 494, Commons 597, ConfigDB 338, Host 260, + all others). Only residue: the same environmental 8 `CentralUI.PlaywrightTests` Deployment E2E failures (`SQLite Error 14` on `instance deploy` — needs a live docker cluster; PLAN-06 touched zero files in that project's deployment-DB path). Execution note: `isolation:"worktree"` subagents proved non-isolating here (shared index → commit races); the working model was pre-created dedicated `git worktree`s per task + cherry-pick onto an integration branch + per-wave combined build/impacted-suite barrier. Initiative total: **152 of 192** (sum of the Done column).
|
||||
|
||||
**Wave 5 — PLAN-07 COMPLETE (2026-07-10):** all 32 active tasks (T4–T32; T1–T3 landed in P0; **T33 `skipped-duplicate`** — CLI.Tests already in the slnx via PLAN-08 T1) delivered on worktree branch `plan07` and **merged to `main` via fast-forward** (main was still at PLAN-06's `4fb754f9`, zero divergence; all worktrees removed, branch deleted). UI/Management/Security hardening across six report-and-wait waves of file-disjoint parallel subagents against the dominant **ManagementActor single-writer lane** (15 tasks serialized in one integration worktree; the file-disjoint remainder ran in dedicated per-task worktrees, cherry-picked in with a per-wave combined build + impacted-suite barrier): **C1** auth-disabled artifact killed + `DisableLoginGuard` fail-fast (T1–T3, P0); **C2** `DeployToSiteAsync` single-site deploy + fleet-wide site-scope guard (T4/T5); **C3** connection/external-system/DB-connection secret elision via `ConfigSecretScrubber` + sentinel-preserving update (T6–T9); **UA1/C6/C5** any-of role gates + `ListSecuredWrites` gating + area-gate reconciliation (Designer|Deployer) + role-casing canonicalization (T10–T12); **UA2** the frozen `GetRequiredRoles` authorization matrix over all **138** registered commands + dispatch-coverage guard (T13/T24); **S2/P3** real server-side secured-write TTL + expiry-at-approve/reject + opportunistic list sweep + list paging + UI history pager & submission-age dialog (T14–T18); **S1** 5-min transport/deploy Ask timeout + shipped config (T19/T20); **C4** additive `SiteIdentifier` on browse/verify/cert commands + actor dispatch (Designer browse/verify, Admin cert-trust) + dispatch-coverage + CLI parity (T21–T25); **P1/UA5** `LoginThrottle` per-username+IP LDAP-bind lockout wired into every bind surface (T26/T27); **P2** additive Skip/Take paging on template/instance lists (T28); **S3** `PollGate` reentrancy guard on Health/AlarmSummary timers (T29); **S4** transport single-flight + export base64-copy elimination (T30); **S5/C7/C8/P4/UA6** debug-stream fault logging + event-log-filter doc fix + AlarmSummary memoization/a11y + the stale CLAUDE.md `SourceNode` note corrected (C7/C8 both verified true — T31/T32). Two message-contract additions (`AuditKind.SecuredWriteExpire`, secured-write list paging params) landed additively; no EF migrations needed. **Terminal verification:** full slnx build 0 warn/0 err; 28 of 30 test assemblies green (ManagementService 448 incl. the frozen matrix + dispatch-coverage, CentralUI 898, CLI 359, Security 175, Commons 609, Communication 263, Transport 138, ConfigDB 341, SiteRuntime 494, Host 260, + all others), after fixing one stale enum-count lock-in test (`AuditEnumTests` 15→16 for T15's additive `SecuredWriteExpire` — the per-task filtered runs never hit that Commons guard, so the full sweep caught it, exactly as in PLAN-04). Only residue: the same environmental 8 `CentralUI.PlaywrightTests` Deployment E2E failures (`SQLite Error 14` on `instance deploy` — needs a live docker cluster; PLAN-07 touched zero files in that project's deployment-DB path). Initiative total: **181 of 192** (sum of the Done column; PLAN-07's max is 32/33 with T33 a documented skip).
|
||||
|
||||
@@ -146,7 +146,7 @@ Run up to **five plan-lane executor sessions concurrently** (one each for PLAN-0
|
||||
- **PLAN-04 Task 13 (additive `RequestedBy` message change) before any PLAN-07 sender refactors** touching the same messages.
|
||||
|
||||
**Soft seams (either order works, coordinate):**
|
||||
- **PLAN-05 Task 14 (`IScriptArtifactChangeBus`) ↔ PLAN-06 Task 3 (`InboundScriptExecutor.InvalidateMethod`).** PLAN-06 Task 1's per-request revision check is the designed correctness fallback, so there is no hard sequencing; when both land, wire the bus to the invalidate seam.
|
||||
- **PLAN-05 Task 14 (`IScriptArtifactChangeBus`) ↔ PLAN-06 Task 3 (`InboundScriptExecutor.InvalidateMethod`).** PLAN-06 Task 1's per-request revision check is the designed correctness fallback, so there is no hard sequencing; when both land, wire the bus to the invalidate seam. **Closed 2026-07-13 (PLAN-R2-06 T1):** `ScriptArtifactChangeSubscriber` (InboundAPI hosted service) subscribes and invalidates on `ApiMethod` notifications.
|
||||
- **PLAN-03's `DeployCompileValidator` should adopt PLAN-05's compile-cache abstraction** once it exists (later refactor, not a blocker).
|
||||
- **PLAN-03 Task 8's stuck-thread watchdog pattern is reusable by PLAN-06 Task 4** (inbound orphan counter).
|
||||
|
||||
@@ -187,6 +187,8 @@ Run up to **five plan-lane executor sessions concurrently** (one each for PLAN-0
|
||||
|
||||
## Deferred / Won't-Fix Registry (initiative-level)
|
||||
|
||||
> 2026-07-12 (plan R2-08 T10): the two live items below (SBR oldest-crash gap, wonder-app-vd03 overlay) now have canonical rows in `docs/plans/2026-07-08-deferred-work-register.md` — that register is the single tracking place; the subsections below remain as the historical narrative.
|
||||
|
||||
Consolidated in **PLAN-08 Task 11** → `docs/plans/2026-07-08-deferred-work-register.md` (triages the 23-item inventory from report 08 plus each plan's coverage-table deferrals: serializer swap, hash-chain/Parquet, aggregated live alarm stream, central cert-trust persistence, runtime script sandbox, clustered-GUID benchmark, TLS profile, Dockerfile HEALTHCHECK, etc.). Nothing from the review is silently dropped: every report finding is either a task, an ownership pointer, or a registered deferral with rationale.
|
||||
|
||||
### Follow-up discovered during P0 execution (2026-07-08)
|
||||
@@ -195,7 +197,7 @@ Consolidated in **PLAN-08 Task 11** → `docs/plans/2026-07-08-deferred-work-reg
|
||||
|
||||
### Deferred during Wave 1 PLAN-01 execution (2026-07-08)
|
||||
|
||||
- **`deploy/wonder-app-vd03/` overlay edits not applied — that production deploy artifact is not tracked in this repo.** PLAN-01 Tasks 16, 20, and 23 each specified a small edit to the on-host deployment overlay (`appsettings.Central.json`: add `AllowSingleNodeCluster: true` + drop the phantom seed, add `NodeName: central-a`; `install.ps1`: add `sc.exe failure` service-recovery actions for the down-if-alone restart contract). Only the *deployment-record* doc (`docs/deployment/deployment-records/2026-06-27-wonder-app-vd03.md`) is version-controlled; the `deploy/wonder-app-vd03/` artifact directory itself is not present in the repo (production config kept out of source control). The **code/validator/doc** halves of all three tasks shipped and are what make the features usable (`AllowSingleNodeCluster` flag + validator, the `IHostApplicationLifetime` process-exit watchdog, the oldest-member Traefik/Host doc reconciliation + TLS roadmap note). **Owner: whoever maintains the on-host artifact** — apply the three overlay edits out-of-band on `wonder-app-vd03` (see the referenced task steps); without `NodeName` every audit row from that deployment stamps a NULL `SourceNode`, and without the service-recovery actions a self-downed node will not auto-restart.
|
||||
- **`deploy/wonder-app-vd03/` overlay edits — applied on-disk 2026-07-12 by PLAN-R2-01 Tasks 5-6 (the earlier not-in-repo deferral rationale was factually wrong).** PLAN-01 Tasks 16, 20, and 23 each specified a small edit to the on-host deployment overlay (`appsettings.Central.json`/`appsettings.Site.json`: add `AllowSingleNodeCluster: true` + drop the phantom seed, add `NodeName` central-a/node-a; `install.ps1`: complete the recovery actions with `sc.exe failureflag` for the down-if-alone restart contract). The original deferral claimed the artifact directory "is not present in the repo" — this was **false**: `deploy/wonder-app-vd03/` **is present in the working tree, merely gitignored** (`.gitignore:48: /deploy/` — production config kept out of version control, not absent), and was actively maintained here (PLAN-07 edited its `_comment_Security`). Correcting the record: the *presence* claim was wrong; only the *version-control-tracking* is intentionally absent. PLAN-R2-01 Tasks 5-6 applied the three overlay edits directly on-disk (no `git add` — the artifact stays gitignored; `sc.exe failure` service-recovery actions were already present on disk, so T6 only added the missing `failureflag`). The **code/validator/doc** halves of all three original tasks had already shipped (`AllowSingleNodeCluster` flag + validator, the `IHostApplicationLifetime` process-exit watchdog, the oldest-member Traefik/Host doc reconciliation + TLS roadmap note). **Remaining owner action:** sync the artifact to `wonder-app-vd03` and restart services — appsettings take effect on restart; `failureflag` on re-running `install.ps1` §7 (or running the two `sc.exe` lines manually) — per the RUNBOOK troubleshooting row added by T6. Without `NodeName` every audit row from that deployment stamps a NULL `SourceNode`; without the failure-actions flag a self-downed (non-crash-exit) node will not auto-restart.
|
||||
|
||||
### Plan deviations during Wave 1 PLAN-03 execution (2026-07-09)
|
||||
|
||||
|
||||
@@ -35,7 +35,12 @@
|
||||
"JwtExpiryMinutes": 15,
|
||||
"IdleTimeoutMinutes": 30,
|
||||
"RequireHttpsCookie": false,
|
||||
"CookieName": "ZB.MOM.WW.ScadaBridge.Auth.env2"
|
||||
"CookieName": "ZB.MOM.WW.ScadaBridge.Auth.env2",
|
||||
"ForwardedHeaders": {
|
||||
"_comment": "Traefik fronts this node on the external scadabridge-net docker network. Trust X-Forwarded-For from the docker bridge address pool so LoginThrottle keys on the real client IP (arch-review R2 N2). Narrow to the Traefik container IP if the network is pinned.",
|
||||
"Enabled": true,
|
||||
"KnownNetworks": [ "172.16.0.0/12" ]
|
||||
}
|
||||
},
|
||||
"Communication": {
|
||||
"DeploymentTimeout": "00:02:00",
|
||||
|
||||
@@ -35,7 +35,12 @@
|
||||
"JwtExpiryMinutes": 15,
|
||||
"IdleTimeoutMinutes": 30,
|
||||
"RequireHttpsCookie": false,
|
||||
"CookieName": "ZB.MOM.WW.ScadaBridge.Auth.env2"
|
||||
"CookieName": "ZB.MOM.WW.ScadaBridge.Auth.env2",
|
||||
"ForwardedHeaders": {
|
||||
"_comment": "Traefik fronts this node on the external scadabridge-net docker network. Trust X-Forwarded-For from the docker bridge address pool so LoginThrottle keys on the real client IP (arch-review R2 N2). Narrow to the Traefik container IP if the network is pinned.",
|
||||
"Enabled": true,
|
||||
"KnownNetworks": [ "172.16.0.0/12" ]
|
||||
}
|
||||
},
|
||||
"Communication": {
|
||||
"DeploymentTimeout": "00:02:00",
|
||||
|
||||
+16
-8
@@ -273,19 +273,27 @@ All test passwords are `password`. See `infra/glauth/config.toml` for the full l
|
||||
### Automated failover drill (`failover-drill.sh`)
|
||||
|
||||
```bash
|
||||
bash docker/failover-drill.sh
|
||||
DRILL_MODE=standby bash docker/failover-drill.sh # default — survivable younger-node crash
|
||||
DRILL_MODE=active bash docker/failover-drill.sh # oldest-node crash — measures the registered outage gap
|
||||
```
|
||||
|
||||
The scripted drill is the repeatable version of the manual steps below. It:
|
||||
The scripted drill (`docker kill` = SIGKILL, the hard-crash path — a `docker stop` would take the graceful `CoordinatedShutdown` path and would not prove crash recovery) has **two modes**, because under the unified oldest-member semantics the *active* node IS the oldest, i.e. the one crash two-node keep-oldest cannot survive:
|
||||
|
||||
1. Finds the **active** central node (via each node's `/health/active`).
|
||||
2. **`docker kill`s it — SIGKILL, the hard-crash path.** This is the exact scenario arch-review 01 found had *no* automatic recovery before the SBR downing provider was enabled (`Akka.Cluster.SBR.SplitBrainResolverProvider` is now named explicitly in HOCON); a `docker stop` would take the graceful `CoordinatedShutdown` path instead and would not prove crash recovery.
|
||||
3. Polls Traefik (`http://localhost:9000/health/active`, override with `TRAEFIK_URL`) until the survivor is routable, printing the observed failover time. Fails after `TIMEOUT_S` (default 90s).
|
||||
4. Dumps the survivor's singleton/oldest/downing log evidence, then restarts the victim so it **rejoins as a fresh incarnation** (Task 20's recovery loop).
|
||||
- **`DRILL_MODE=standby` (default) — kills the STANDBY (younger) central node.** The survivable direction: SBR downs the crashed member and the active node keeps its singletons. Expected result: **no routing outage at all** (the active node is never touched, so `/health/active` blips = 0) and member removal on the survivor within **~25s** (10s failure-detection threshold + 15s stable-after; the 2s heartbeat interval is not additive). PASS = the survivor logs the member removal within `TIMEOUT_S` (default 90s) while routing stays up.
|
||||
- **`DRILL_MODE=active` — kills the ACTIVE (oldest) central node.** Expected result: a **total central outage** until the victim container is restarted — this is the registered deferred keep-oldest decision (master tracker 2026-07-08): keep-oldest downs the partition *without* the oldest, so the younger survivor downs itself, and it cannot re-form a cluster alone (see the seed-node constraint below). The drill confirms the dark window, then recovery within ~2 min of restarting the victim. The mode exists to make the registered gap *observable*, not to pretend it is covered.
|
||||
|
||||
Expected failover: **~25s** (2s heartbeat + 10s detection + 15s stable-after) **plus the Traefik ~5s health-check interval**. The drill exercises S1 (SBR downing on hard crash), S3 (single active node routed through Traefik), and the Task 20 restart/rejoin contract. Requires a running cluster (`bash docker/deploy.sh`) and `curl` + `docker` on the host.
|
||||
The drill exercises S1 (SBR downing on hard crash), S3 (single active node routed through Traefik), and the Task 20 restart/rejoin contract. Requires a running cluster (`bash docker/deploy.sh`) and `curl` + `docker` on the host.
|
||||
|
||||
> **Observed failover time:** _run after next deploy and record here_ (the script was validated with `bash -n`; no live cluster was up when it was committed).
|
||||
**Seed-node bootstrap constraint.** Only the FIRST seed in `Cluster:SeedNodes` may self-join to form a *new* cluster. Both central nodes list `scadabridge-central-a` first (`docker/central-node-a/appsettings.Central.json`, `docker/central-node-b/appsettings.Central.json`), so a lone restarted `central-b` (with `central-a` still down) loops on `InitJoin` forever — it never reaches `Up`, and `/health/active` never returns 200. Operator recovery actions: **(1)** restart the dead first-seed node (`central-a`) — preferred; or **(2)** restart the survivor with a self-first seed override (env `ScadaBridge__Cluster__SeedNodes__0=akka.tcp://scadabridge@<self-host>:8081`, `ScadaBridge__Cluster__SeedNodes__1=<peer>`). The repo deliberately does NOT ship self-first ordering per node: with *both* nodes self-first, a simultaneous cold start can let each self-join independently → two one-node clusters that never merge (the cold-start split-brain the identical-seed-order convention exists to prevent). The real remedy is the pending keep-oldest topology/strategy decision (deferred, owner: user).
|
||||
|
||||
> **Observed results** (plan R2-01 T3):
|
||||
>
|
||||
> **PENDING — live drill not yet executed** (requires a dedicated docker cluster rebuild; deferred to avoid disrupting the running environment). The two-mode drill script and its per-direction PASS criteria are in place; the empirical timings below are to be filled by running `DRILL_MODE=standby` then `DRILL_MODE=active` against a freshly-deployed cluster and recording the one-line summaries + the cluster commit SHA.
|
||||
>
|
||||
> | Direction (`DRILL_MODE`) | Outcome | Measured |
|
||||
> |--------------------------|---------|----------|
|
||||
> | `standby` (younger-node crash) | _pending live run_ | _pending live run_ |
|
||||
> | `active` (oldest-node crash) | _pending live run_ | _pending live run_ |
|
||||
|
||||
### Central Failover
|
||||
|
||||
|
||||
@@ -38,7 +38,12 @@
|
||||
"JwtSigningKey": "scadabridge-dev-jwt-signing-key-must-be-at-least-32-characters-long",
|
||||
"JwtExpiryMinutes": 15,
|
||||
"IdleTimeoutMinutes": 30,
|
||||
"RequireHttpsCookie": false
|
||||
"RequireHttpsCookie": false,
|
||||
"ForwardedHeaders": {
|
||||
"_comment": "Traefik fronts this node on the external scadabridge-net docker network. Trust X-Forwarded-For from the docker bridge address pool so LoginThrottle keys on the real client IP (arch-review R2 N2). Narrow to the Traefik container IP if the network is pinned.",
|
||||
"Enabled": true,
|
||||
"KnownNetworks": [ "172.16.0.0/12" ]
|
||||
}
|
||||
},
|
||||
"Communication": {
|
||||
"DeploymentTimeout": "00:02:00",
|
||||
|
||||
@@ -38,7 +38,12 @@
|
||||
"JwtSigningKey": "scadabridge-dev-jwt-signing-key-must-be-at-least-32-characters-long",
|
||||
"JwtExpiryMinutes": 15,
|
||||
"IdleTimeoutMinutes": 30,
|
||||
"RequireHttpsCookie": false
|
||||
"RequireHttpsCookie": false,
|
||||
"ForwardedHeaders": {
|
||||
"_comment": "Traefik fronts this node on the external scadabridge-net docker network. Trust X-Forwarded-For from the docker bridge address pool so LoginThrottle keys on the real client IP (arch-review R2 N2). Narrow to the Traefik container IP if the network is pinned.",
|
||||
"Enabled": true,
|
||||
"KnownNetworks": [ "172.16.0.0/12" ]
|
||||
}
|
||||
},
|
||||
"Communication": {
|
||||
"DeploymentTimeout": "00:02:00",
|
||||
|
||||
+88
-19
@@ -1,44 +1,113 @@
|
||||
#!/usr/bin/env bash
|
||||
# Failover drill against the running docker cluster (bash docker/deploy.sh first).
|
||||
# Kills the ACTIVE central node (docker kill = SIGKILL, the hard-crash path that
|
||||
# review 01 found had NO automatic recovery before the SBR downing provider was
|
||||
# enabled) and measures how long Traefik takes to route to the survivor.
|
||||
#
|
||||
# ROUND-2 REWRITE (arch-review 01 round 2, N1). The original drill killed the
|
||||
# ACTIVE central node — but under the unified oldest-member semantics the
|
||||
# active node IS the oldest, i.e. the one crash two-node keep-oldest CANNOT
|
||||
# survive (registered deferred user decision, master tracker 2026-07-08;
|
||||
# SbrFailoverTests.cs XML doc). Two modes:
|
||||
#
|
||||
# DRILL_MODE=standby (default) — kills the STANDBY (younger) central node.
|
||||
# The survivable direction: SBR downs the crashed member, the active node
|
||||
# keeps its singletons, and Traefik routing never goes dark. PASS = the
|
||||
# survivor logs the member removal within TIMEOUT_S (budget ~25s+: 10s
|
||||
# failure detection + 15s stable-after) while /health/active stays up.
|
||||
#
|
||||
# DRILL_MODE=active — kills the ACTIVE (oldest) central node. THE EXPECTED
|
||||
# OUTCOME IS A TOTAL CENTRAL OUTAGE: keep-oldest downs the partition
|
||||
# without the oldest, so the younger survivor downs ITSELF (down-if-alone
|
||||
# cannot help — the alone-oldest is dead and cannot down itself), and the
|
||||
# self-downed survivor cannot re-form a cluster alone unless it is the
|
||||
# FIRST seed (both nodes list central-a first; only the first seed may
|
||||
# self-join). This mode measures the dark window and PASSes only when
|
||||
# central recovers AFTER the victim container is restarted. It exists to
|
||||
# make the registered gap observable — not to pretend it is covered.
|
||||
set -euo pipefail
|
||||
|
||||
TRAEFIK_URL="${TRAEFIK_URL:-http://localhost:9000}"
|
||||
TIMEOUT_S="${TIMEOUT_S:-90}"
|
||||
DRILL_MODE="${DRILL_MODE:-standby}"
|
||||
OUTAGE_CONFIRM_S="${OUTAGE_CONFIRM_S:-60}"
|
||||
|
||||
active_container() {
|
||||
if curl -sf -o /dev/null "http://localhost:9001/health/active"; then echo scadabridge-central-a
|
||||
elif curl -sf -o /dev/null "http://localhost:9002/health/active"; then echo scadabridge-central-b
|
||||
else echo "ERROR: no active central node found" >&2; exit 1; fi
|
||||
}
|
||||
peer_of() { [ "$1" = scadabridge-central-a ] && echo scadabridge-central-b || echo scadabridge-central-a; }
|
||||
|
||||
VICTIM=$(active_container)
|
||||
echo "Active central node: ${VICTIM} — killing it (SIGKILL, crash path)"
|
||||
case "$DRILL_MODE" in
|
||||
standby|active) ;;
|
||||
*) echo "ERROR: DRILL_MODE must be 'standby' or 'active' (was '$DRILL_MODE')" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
ACTIVE=$(active_container)
|
||||
if [ "$DRILL_MODE" = standby ]; then
|
||||
VICTIM=$(peer_of "$ACTIVE"); SURVIVOR="$ACTIVE"
|
||||
else
|
||||
VICTIM="$ACTIVE"; SURVIVOR=$(peer_of "$ACTIVE")
|
||||
fi
|
||||
|
||||
echo "mode=${DRILL_MODE} active=${ACTIVE} victim=${VICTIM} survivor=${SURVIVOR}"
|
||||
KILL_AT=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
docker kill "${VICTIM}" > /dev/null
|
||||
START=$(date +%s)
|
||||
|
||||
echo "Waiting for the survivor to become active through Traefik (${TRAEFIK_URL}/health/active)..."
|
||||
if [ "$DRILL_MODE" = standby ]; then
|
||||
echo "Standby crash: waiting for ${SURVIVOR} to DOWN+REMOVE the dead member (SBR budget ~25s)..."
|
||||
BLIPS=0
|
||||
while true; do
|
||||
ELAPSED=$(( $(date +%s) - START ))
|
||||
curl -sf -o /dev/null "${TRAEFIK_URL}/health/active" || BLIPS=$((BLIPS + 1))
|
||||
if docker logs --since "${KILL_AT}" "${SURVIVOR}" 2>&1 | grep -Eiq "marking.*node.*down|member removed|is removed"; then
|
||||
echo "PASS: survivor removed the crashed member in ${ELAPSED}s (budget ~25s: 10s detection + 15s stable-after)."
|
||||
echo "Active-node routing blips during the drill: ${BLIPS} (expected 0 — the active node was never touched)."
|
||||
break
|
||||
fi
|
||||
if (( ELAPSED > TIMEOUT_S )); then
|
||||
echo "FAIL: no downing/removal evidence on ${SURVIVOR} after ${ELAPSED}s — SBR did not act" >&2
|
||||
docker start "${VICTIM}" > /dev/null
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
else
|
||||
echo "Active crash: EXPECTING a central outage (registered keep-oldest gap). Watching /health/active..."
|
||||
DARK_STREAK=0
|
||||
while true; do
|
||||
ELAPSED=$(( $(date +%s) - START ))
|
||||
if curl -sf -o /dev/null "${TRAEFIK_URL}/health/active"; then DARK_STREAK=0; else DARK_STREAK=$((DARK_STREAK + 1)); fi
|
||||
if (( DARK_STREAK >= 10 )); then
|
||||
echo "Outage confirmed at ${ELAPSED}s: no active central node — the younger survivor self-downed"
|
||||
echo "(keep-oldest downs the partition WITHOUT the oldest; this is the registered deferred gap)."
|
||||
break
|
||||
fi
|
||||
if (( ELAPSED > OUTAGE_CONFIRM_S )); then
|
||||
echo "NOTE: /health/active stayed reachable ${ELAPSED}s after killing the oldest — better than the"
|
||||
echo "registered gap predicts. Do NOT celebrate: capture both nodes' logs and investigate before trusting it."
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
fi
|
||||
|
||||
echo "Restarting ${VICTIM}..."
|
||||
docker start "${VICTIM}" > /dev/null
|
||||
RESTART_AT=$(date +%s)
|
||||
echo "Waiting for central to be routable again through Traefik (${TRAEFIK_URL}/health/active)..."
|
||||
while true; do
|
||||
ELAPSED=$(( $(date +%s) - START ))
|
||||
ELAPSED=$(( $(date +%s) - RESTART_AT ))
|
||||
if curl -sf -o /dev/null "${TRAEFIK_URL}/health/active"; then
|
||||
echo "PASS: failover complete in ${ELAPSED}s (design budget ~25s + Traefik health interval)"
|
||||
echo "Recovered: an active central node is routable ${ELAPSED}s after the victim restart."
|
||||
break
|
||||
fi
|
||||
if (( ELAPSED > TIMEOUT_S )); then
|
||||
echo "FAIL: no active central node after ${ELAPSED}s — SBR/singleton handover did not recover" >&2
|
||||
docker start "${VICTIM}" > /dev/null
|
||||
if (( ELAPSED > 120 )); then
|
||||
echo "FAIL: central not routable 120s after restarting ${VICTIM}" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
SURVIVOR=$([ "${VICTIM}" = scadabridge-central-a ] && echo scadabridge-central-b || echo scadabridge-central-a)
|
||||
echo "Survivor singleton evidence (last 20 matching log lines from ${SURVIVOR}):"
|
||||
docker logs "${SURVIVOR}" 2>&1 | grep -Ei "singleton|oldest|Downing|Removed" | tail -20 || true
|
||||
|
||||
echo "Restarting ${VICTIM} and waiting for it to rejoin..."
|
||||
docker start "${VICTIM}" > /dev/null
|
||||
sleep 5
|
||||
echo "Drill complete. Verify on the Health dashboard that both nodes show Up and the survivor is Primary."
|
||||
echo "Survivor singleton/downing evidence (last 20 matching log lines from ${SURVIVOR}):"
|
||||
docker logs "${SURVIVOR}" 2>&1 | grep -Ei "singleton|oldest|downing|removed" | tail -20 || true
|
||||
echo "Drill complete (${DRILL_MODE}). Verify on the Health dashboard that both nodes show Up and exactly one is Primary."
|
||||
|
||||
@@ -5,15 +5,8 @@ Single tracked list of consciously-deferred work. Rules: every deferral gets a r
|
||||
them and are removed from this table when that plan's task lands.
|
||||
|
||||
## Fix-now (owned by archreview plans)
|
||||
| # | Item | Owner |
|
||||
|---|------|-------|
|
||||
| 1 | Transport: Area membership doesn't travel in bundles (EntitySerializer.cs:251,518) | PLAN-05 |
|
||||
| 2 | CLI.Tests not in slnx | PLAN-08 Task 1 |
|
||||
| 3 | AuditLog reconciliation dials site NodeA only (SiteEnumerator.cs:32,64) | PLAN-04 |
|
||||
| 4 | Options validation missing on ~12 components | PLAN-08 Tasks 2–5 |
|
||||
| 5 | SmsConfiguration.MaxRetryCount stored but not honored | PLAN-06 |
|
||||
| 6 | Role-check case-sensitivity asymmetry (ManagementActor.cs:1910) | PLAN-07 |
|
||||
| 23 | Vestigial site-side notification-list surface | PLAN-08 Task 6 |
|
||||
|
||||
All 7 fix-now items landed via PLAN-04/05/06/07/08 (verified in review 08 round 2, 2026-07-12) — rows removed per the rule above; see the round-2 report §1 for the per-item evidence.
|
||||
|
||||
## Deferred (with rationale + revisit trigger)
|
||||
| # | Item | Where noted | Rationale for deferral | Revisit trigger |
|
||||
@@ -48,4 +41,12 @@ Rows removed from the Deferred table above once confirmed shipped. Kept here for
|
||||
| Communication → HealthMonitoring layering (ICentralHealthAggregator consumed by CentralCommunicationActor.cs:351) | Moving the interface + SiteHealthState to Commons ripples across 5 projects for a cosmetic inversion | Next breaking change to ICentralHealthAggregator |
|
||||
| docs/components reference docs for ScriptAnalysis, KpiHistory, DelmiaNotifier | Reference docs are substantial (StyleGuide-conformant); README claim scoped instead (PLAN-08 Task 10) | Next doc-writing session touching those components |
|
||||
| Test-coverage backfill: SiteCallAudit.Tests (31 tests/1.6k LOC), DeploymentManager.Tests | No defect identified; coverage partly lives in ManagementService/Host/Integration suites | First regression escaping either component |
|
||||
| Failover-timing + broader perf envelope (S&F drain rate, per-subscriber backpressure) | Needs PLAN-01 two-node rig; placeholder harness shipped (PLAN-08 Task 8) | PLAN-01 rig landing |
|
||||
| Failover-timing + broader perf envelope (S&F drain rate, per-subscriber backpressure) | **Trigger fired 2026-07-08** — the "PLAN-01 rig landing" trigger fired when `tests/ZB.MOM.WW.ScadaBridge.IntegrationTests/Cluster/TwoNodeClusterFixture.cs` landed (before the placeholder harness even shipped), and went unnoticed (NF2). **Owner: PLAN-R2-01** — `archreview/plans/PLAN-R2-01-*.md` (failover-envelope measurement task) owns wiring `FailoverTimingTests` to the fixture rig or recording the true blocker. | Owned by PLAN-R2-01; this register row tracks the handoff only |
|
||||
|
||||
## Deferred — operational risk (from the initiative tracker, folded in 2026-07-12)
|
||||
Two live items previously tracked ONLY in `archreview/plans/00-MASTER-TRACKER.md`'s registry are folded in here (NF5) so this register is the single tracking place. The tracker's narrative subsections remain as the historical evidence.
|
||||
|
||||
| # | Item | Where noted | Rationale for deferral | Revisit trigger |
|
||||
|---|------|-------------|------------------------|-----------------|
|
||||
| SBR | **SBR oldest-crash total-outage gap** — 2-node `keep-oldest` downs the partition *without* the oldest, so a hard crash of the ACTIVE (oldest) central node makes the standby self-down (~10s) → total central outage until the crashed node restarts; only a younger-node crash fails over. | `archreview/plans/00-MASTER-TRACKER.md:194` + auto-memory `sbr-keep-oldest-2node-active-crash-gap` | Remedy is a production SBR topology/strategy decision (keep-majority + a 3rd/lighthouse seed node, static-quorum, or an accepted-risk note) — **owner: user decision**, not silently changeable. | Before the next production deployment that adds a central node, or the first real active-node crash. |
|
||||
| vd03 | **`deploy/wonder-app-vd03/` overlay edits unapplied** — `appsettings.Central.json` needs `AllowSingleNodeCluster: true` + phantom-seed removal + `NodeName: central-a`; `install.ps1` needs `sc.exe failure` recovery actions. The `deploy/wonder-app-vd03/` artifact directory is intentionally untracked (production config out of source control), so the repo cannot ship the fix. | `archreview/plans/00-MASTER-TRACKER.md:198` (PLAN-01 T16/T20/T23) | Needs on-host access; without `NodeName` that deployment's audit rows stamp NULL `SourceNode` — **partially mitigated once PLAN-R2-08 Task 7 lands: the host now FAILS AT BOOT with a key-naming error instead of silently NULLing, so applying the overlay becomes mandatory at the next upgrade.** Owner: whoever maintains the host (user). | Next wonder-app-vd03 deployment/upgrade — **the Task 7 validator makes this row unskippable then.** |
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Script-Artifact Invalidation Contract (`ScriptArtifactsChanged`)
|
||||
|
||||
**Status:** Seam + first producer shipped in PLAN-05 (arch-review 05, Task 14). Consumer + additional producers land in **plan 06**.
|
||||
**Status:** Seam + first producer shipped in PLAN-05 (T14). **Consumer (a) shipped in PLAN-R2-06 T1** (`ScriptArtifactChangeSubscriber`). (b) Management producers remain unwired by design — delete calls `InvalidateMethod` directly; edits self-heal via the revision check. (c) is satisfied by the per-request revision check (PLAN-06 T1).
|
||||
|
||||
**Problem.** When a script-bearing artifact's definition changes at runtime — an Inbound API method's script, a shared script, or a template's scripts — any node that has already **compiled and cached** a handler/delegate for it keeps serving the stale version until a process restart. This is the root of two recorded project-memory gotchas (a DB-direct edit to a broken inbound method keeps returning "Script compilation failed" from `_knownBadMethods`; a non-compiling update leaves the old delegate serving). A bundle import is a third mutation path with the same hazard.
|
||||
|
||||
@@ -45,14 +45,12 @@ public interface IScriptArtifactChangeBus
|
||||
- `InProcessScriptArtifactChangeBus` in the Host, registered as a central-role singleton.
|
||||
- The **Transport bundle importer as the first producer**: `BundleImporter.ApplyAsync` collects the Overwrite/Rename resolutions for `ApiMethod` / `SharedScript` / `Template` and publishes one `ScriptArtifactsChanged` per kind **after** `tx.CommitAsync`. The importer's dependency on the bus is optional (`IScriptArtifactChangeBus?`) so Transport-only hosts (and unit tests) run without it.
|
||||
|
||||
## Handoff to plan 06
|
||||
## Handoff to plan 06 — dispositions
|
||||
|
||||
Plan 06 implements the consumer side and the remaining producers:
|
||||
|
||||
| Item | Responsibility |
|
||||
| Item | Disposition |
|
||||
|---|---|
|
||||
| **(a) Inbound API consumer** | `InboundScriptExecutor` subscribes to the bus and, on an `ApiMethod` notification, evicts the named entries from `_scriptHandlers` **and** `_knownBadMethods` — closing the "DB-direct edit keeps returning compile error" and "non-compiling update keeps old handler" gotchas. |
|
||||
| **(b) Management producers** | Publish `ScriptArtifactsChanged` from the `ManagementActor` `UpdateApiMethod` / shared-script / template-script edit paths (`Source = "Management"`), so a UI/CLI edit invalidates the same caches a bundle import does. |
|
||||
| **(c) Cross-node / failover freshness** | A revision-keyed handler cache (compile keyed by code hash — see PLAN-05 Task 15's `ScriptCompileVerdictCache`) so a node that missed the in-process notification still self-heals on next use, and a newly-active failover node never serves a stale compile. |
|
||||
| **(a) Inbound API consumer** | **Shipped (PLAN-R2-06 T1).** `ScriptArtifactChangeSubscriber` (an InboundAPI `IHostedService` registered by `AddInboundAPI`) subscribes to the bus and, on an `ApiMethod` notification, calls `InboundScriptExecutor.InvalidateMethod` per name — evicting the named entries from `_scriptHandlers` **and** `_knownBadMethods`, closing the "DB-direct edit keeps returning compile error" and "non-compiling update keeps old handler" gotchas. |
|
||||
| **(b) Management producers** | **Unwired by design.** The `ManagementActor` delete path calls `InboundScriptExecutor.InvalidateMethod` directly; the update/edit paths self-heal via the per-request revision check, so no notification is needed for correctness. Left unwired deliberately rather than publishing `Source = "Management"` notifications. |
|
||||
| **(c) Cross-node / failover freshness** | **Satisfied by the per-request revision check (PLAN-06 T1).** The compiled handler is revision-checked against the freshly-fetched `ApiMethod.Script` on every request, so a node that missed the in-process notification still self-heals on next use, and a newly-active failover node never serves a stale compile. |
|
||||
|
||||
Until (a)–(c) land, plan 05's publisher is a no-op in practice (no subscribers), but the seam is stable and the producer is correct, so plan 06 is purely additive.
|
||||
Consumer (a) is now wired, so plan 05's `ApiMethod` publisher reaches a real subscriber. `SharedScript`/`Template` notifications still publish with **no consumer** — that is safe: nothing at central caches compiled artifacts of those kinds, so there is nothing to invalidate. The seam is stable and the producer is correct.
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
BEGIN TRANSACTION;
|
||||
IF NOT EXISTS (
|
||||
SELECT * FROM [__EFMigrationsHistory]
|
||||
WHERE [MigrationId] = N'20260713142234_AddSiteCallsTerminalIndex'
|
||||
)
|
||||
BEGIN
|
||||
EXEC(N'CREATE INDEX [IX_SiteCalls_Terminal] ON [SiteCalls] ([TerminalAtUtc]) WHERE [TerminalAtUtc] IS NOT NULL');
|
||||
END;
|
||||
|
||||
IF NOT EXISTS (
|
||||
SELECT * FROM [__EFMigrationsHistory]
|
||||
WHERE [MigrationId] = N'20260713142234_AddSiteCallsTerminalIndex'
|
||||
)
|
||||
BEGIN
|
||||
INSERT INTO [__EFMigrationsHistory] ([MigrationId], [ProductVersion])
|
||||
VALUES (N'20260713142234_AddSiteCallsTerminalIndex', N'10.0.7');
|
||||
END;
|
||||
|
||||
COMMIT;
|
||||
GO
|
||||
|
||||
@@ -191,7 +191,7 @@ Per-leaf alarm rendering (leaf nodes are individual conditions for native alarms
|
||||
- **Live updates** — the page is driven by a **transient, per-site central live alarm cache** (`ISiteAlarmLiveCache`, owned by the Communication component; see [Component-Communication](Component-Communication.md)). On site select the page subscribes to the cache; the cache runs one shared, reference-counted per-site aggregator that **seeds** from the snapshot fan-out and then stays warm on a single **site-wide, alarm-only** `SubscribeSite` gRPC stream (seed-then-stream, dedup by `(InstanceUniqueName, AlarmName, SourceReference)`). Applied deltas raise an in-process change event (mirroring `IDeploymentStatusNotifier`) that the Blazor circuit pushes to the browser via `StateHasChanged()` — no new SignalR hub. `AlarmSummaryService.BuildFromLiveAlarms` rebuilds the roll-up + rows from the cache's current alarm set. The cache is **purely in-memory on the active central node** — there is still **no persisted central alarm store**; on a NodeA↔NodeB failover the new active node re-seeds from scratch.
|
||||
- **View** — roll-up tiles (total active, worst severity, unacked count, per-`AlarmKind` counts) plus a flat, sortable, filterable table. Filters cover instance, `AlarmKind` (Computed / NativeOpcUa / NativeMxAccess), state, acked/unacked, severity threshold, and name search.
|
||||
- **Read-only** — there are no ack / shelve / suppress controls (native alarms remain read-only by design).
|
||||
- **Refresh** — manual refresh button plus the 15s poll timer (mirroring the Health dashboard), now retained as a **fallback + `NotReporting` authority** behind the live cache: when the cache reports `IsLive`, the page renders live-cache state; when a stream is unhealthy or a site has not yet seeded, the poll keeps the page fresh so a stream failure never blanks it. (Aggregated live stream **delivered 2026-07-10** — see `docs/plans/2026-07-10-aggregated-live-alarm-stream-plan.md`.)
|
||||
- **Refresh** — manual refresh button plus the 15s poll timer (mirroring the Health dashboard), now retained as a **fallback + `NotReporting` authority** behind the live cache: when the cache reports `IsLive`, the page renders live-cache state; when a stream is unhealthy, **the aggregator has died (deathwatch resets `IsLive`)**, or a site has not yet seeded, the poll keeps the page fresh so a stream failure never blanks it. When live, the poll updates only the `NotReporting` list and leaves the row set to the delta path, so a slow fan-out can never momentarily revert a fresher live delta (R2 N5). (Aggregated live stream **delivered 2026-07-10** — see `docs/plans/2026-07-10-aggregated-live-alarm-stream-plan.md`.)
|
||||
- **Reuse** — the alarm badge/formatter markup is factored out of Debug View into a shared `AlarmStateBadges` component consumed by both Debug View and this page.
|
||||
|
||||
### Parked Message Management (Deployment Role)
|
||||
|
||||
@@ -108,9 +108,11 @@ When a node downs itself (via `down-if-alone`, or any other SBR decision), the r
|
||||
1. Self-down ⇒ `CoordinatedShutdown` ⇒ `ActorSystem` termination.
|
||||
2. The Host watches `ActorSystem.WhenTerminated`; a termination that is **not** the host's own `StopAsync` triggers `IHostApplicationLifetime.StopApplication()`, so **the process exits**.
|
||||
3. The service supervisor restarts it — docker `restart: unless-stopped`, or Windows service recovery actions (`sc.exe failure … restart/…`).
|
||||
4. The restarted process rejoins as a **fresh incarnation** (the keep-oldest resolver handles the rejoin cleanly; there is no stale membership to reconcile).
|
||||
4. The restarted process rejoins as a **fresh incarnation** (the keep-oldest resolver handles the rejoin cleanly; there is no stale membership to reconcile) — **but only while a peer still holding cluster state is reachable**. A lone restarted node that is *not* the first seed cannot re-form a cluster on its own (see the seed-node bootstrap constraint below); it waits for its peer.
|
||||
|
||||
The docker failover drill exercises this loop end-to-end (kill the active node, assert singleton migration + clean rejoin).
|
||||
**Seed-node bootstrap constraint.** Only the FIRST seed listed in `Cluster:SeedNodes` may self-join to form a *new* cluster. All nodes list the same first seed (e.g. `scadabridge-central-a`), so a lone restarted non-first-seed node (with the first seed still down) loops on `InitJoin` forever — never `Up`, never routable. This is why the two-node keep-oldest **oldest/active-node crash is a total-outage gap**: after the oldest dies the younger survivor self-downs, and it cannot re-bootstrap alone. Recovery is operator-driven — either restart the dead first-seed node (preferred) or restart the survivor with a self-first seed override (`ScadaBridge__Cluster__SeedNodes__0` = self, `__1` = peer). The repo does not ship self-first ordering per node: with both nodes self-first a simultaneous cold start risks two independent one-node clusters that never merge. Removing the gap itself is the **registered deferred keep-oldest topology/strategy decision** (master tracker 2026-07-08, owner: user).
|
||||
|
||||
The docker failover drill (`docker/failover-drill.sh`) exercises this per direction: `standby` mode proves SBR downing + singleton continuity on the oldest; `active` mode measures the registered total-outage gap and the recovery-on-restart path.
|
||||
|
||||
## Single-Node Operation
|
||||
|
||||
|
||||
@@ -75,8 +75,11 @@ Delivered 2026-07-10 (`docs/plans/2026-07-10-aggregated-live-alarm-stream-plan.m
|
||||
- **Central live cache** (`ISiteAlarmLiveCache`, singleton `SiteAlarmLiveCacheService`): a DI singleton on the active central node. For each site with ≥1 active viewer it runs ONE shared, **reference-counted** per-site aggregator (`SiteAlarmAggregatorActor`); the first `Subscribe(siteId, onChanged)` starts it, the last subscriber leaving stops it after a short **linger** to avoid re-seed thrash. `GetCurrentAlarms(siteId)` returns the current immutable snapshot; `IsLive(siteId)` reports whether the aggregator has seeded and published at least once.
|
||||
- **Seed-then-stream** (copied from `DebugStreamBridgeActor` ordering): open the `SubscribeSite` stream first (buffer live deltas), run the snapshot fan-out once via the existing `DebugViewSnapshot` path (bounded by `LiveAlarmCacheSeedConcurrency`), flush the buffer with **dedup by `(InstanceUniqueName, AlarmName, SourceReference)`**, then live pass-through. Placeholders are seeded from the snapshot and never expected on the live stream.
|
||||
- **Failover & drift**: NodeA↔NodeB reconnect (client via `SiteStreamGrpcClient.SubscribeSiteAsync`, factory endpoint failover) re-seeds rather than serving stale; a periodic **reconcile snapshot** (default 60s) corrects any missed enable/disable so the cache can't drift indefinitely. `[PERM]` (`docs/plans/2026-05-29-native-alarms-design.md`): the cache is **purely in-memory** — no EF entity/table/migration, no persisted central alarm store — so a new active node simply re-seeds from scratch.
|
||||
- **Options** (`Communication` section, `CommunicationOptions`; eagerly validated by `CommunicationOptionsValidator` / `ValidateOnStart`): `LiveAlarmCacheLinger` (default 30s), `LiveAlarmCacheReconcileInterval` (default 60s), `LiveAlarmCacheSeedConcurrency` (default 8), `LiveAlarmCacheMaxSubscribersPerSite` (default 200).
|
||||
- **Options** (`Communication` section, `CommunicationOptions`; eagerly validated by `CommunicationOptionsValidator` / `ValidateOnStart`): `LiveAlarmCacheLinger` (default 30s), `LiveAlarmCacheReconcileInterval` (default 60s), `LiveAlarmCacheSeedConcurrency` (default 8), `LiveAlarmCacheMaxSubscribersPerSite` (default 200), `LiveAlarmCachePublishCoalesce` (default 250ms; `0` = publish per delta — legacy — batches an alarm storm into one snapshot copy + one viewer fan-out per window; arch review 02 round 2, N6).
|
||||
- **Telemetry** (`ScadaBridgeTelemetry` meter): observable gauge `scadabridge.site.alarm_cache.aggregators.active` (running per-site aggregators) and counter `scadabridge.site.alarm_cache.reconnects` (site-wide stream reconnects — a NodeA↔NodeB flip or reconcile-driven reopen; a sustained climb signals a flapping site link).
|
||||
- **Accepted limitations (arch review 02 round 2, N8):**
|
||||
- **Standby-node aggregators**: the live cache is per-node and `SetActorSystem` is wired on every central node, so browsing the standby node directly (diagnostic ports, e.g. 9002) starts a second, fully-functional read-only aggregator + `SubscribeSite` stream there. Accepted — not gated: the aggregator is read-only, bounded (one stream/site, viewer-capped), and torn down by the viewer linger; gating `SetActorSystem` behind the active check would break the diagnostic-browse path and buy nothing. The `[PERM]` "lives only on the active central node" claim is corrected to "per-node; in routine operation only the active node hosts viewers".
|
||||
- **Site deleted while an Alarm Summary viewer is open**: the viewer's aggregator reconciles to an empty snapshot (the deleted site's instances vanish from the fan-out) until its viewers leave, then the linger stop reaps it; the site's cached gRPC channels are disposed at deletion via `SiteStreamGrpcClientFactory.RemoveSiteAsync` (see `ManagementActor.HandleDeleteSite`, arch review 02 round 2, N8).
|
||||
|
||||
#### Site-Side gRPC Streaming Components
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ Site clusters (metric collection and reporting). Central cluster (aggregation an
|
||||
- Each report is a **flat snapshot** containing the current values of all monitored metrics, a **monotonic sequence number**, and the **report timestamp** from the site. Central replaces the previous state for that site only if the incoming sequence number is higher than the last received — this prevents stale reports (e.g., delayed in transit or from a pre-failover node) from overwriting newer state.
|
||||
- Central tracks two **distinct** signals per site — do not conflate them:
|
||||
- **Liveness (online/offline)**: driven by the **last signal of any kind** (a full report *or* a transport heartbeat, the latter arriving every ~5s from any reachable site node). If central receives no signal within `OfflineTimeout` (default: **60 seconds**), the site is marked **offline**. Because heartbeats are frequent, this only fires on genuine total loss of contact — not during a single-node failover, when the surviving node keeps heartbeating. The synthetic *central* self-report site has no heartbeat source and uses the longer `CentralOfflineTimeout` (default: **3 minutes**) so a single missed self-report does not flap it offline.
|
||||
- **Metrics staleness**: an **online** site (heartbeats still arriving) that has produced no full **report** within `MetricsStaleTimeout` (default: **2 minutes**) is flagged **metrics stale**. This surfaces the failure mode where a site's report pipeline (`HealthReportSender`) has died but the node is otherwise reachable — previously such a site showed "online with frozen metrics forever". `MetricsStaleTimeout` must be positive and no shorter than the report interval.
|
||||
- **Metrics staleness**: an **online** site (heartbeats still arriving) that has produced no full **report** within `MetricsStaleTimeout` (default: **2 minutes**) is flagged **metrics stale**. This surfaces the failure mode where a site's report pipeline (`HealthReportSender`) has died but the node is otherwise reachable — previously such a site showed "online with frozen metrics forever". A site that has **never** delivered a report is anchored on its first-contact time (`FirstSeenAt`), so a report pipeline dead from first boot is also flagged after `MetricsStaleTimeout` (previously such a site showed online-with-no-metrics indefinitely). `MetricsStaleTimeout` must be positive and no shorter than the report interval.
|
||||
- **Online recovery**: when central receives a health report from a site that was marked offline, the site is automatically marked **online** and the metrics-stale flag cleared (a fresh report proves the pipeline is alive); a heartbeat alone also restores **online** but leaves the metrics-stale flag untouched. No manual acknowledgment required.
|
||||
- **Status-transition timestamp**: each online↔offline flip stamps `LastStatusChangeAt`, so the UI can answer "since when has this site been offline/online" rather than only "is it offline now". Report/heartbeat traffic that leaves the status unchanged does not move it.
|
||||
|
||||
|
||||
@@ -118,13 +118,13 @@ The `sbk_<keyId>_<secret>` design already supports zero-downtime rotation — a
|
||||
|--------|-------------|---------|
|
||||
| `UNAUTHORIZED` | 401 | Missing/invalid API key (every auth-stage failure maps here — no stage leak). |
|
||||
| `NOT_APPROVED` | 403 | Method not found **or** key not in scope — indistinguishable by design. |
|
||||
| `UNSUPPORTED_MEDIA_TYPE` | 415 | Request carried a body with a non-JSON `Content-Type`. A body with **no** `Content-Type` is still parsed leniently as JSON. |
|
||||
| `UNSUPPORTED_MEDIA_TYPE` | 415 | Request carried a body — declared by `Content-Length` or `Transfer-Encoding` (chunked) — with a non-JSON `Content-Type`. A body with **no** `Content-Type` is still parsed leniently as JSON, and a chunked (no `Content-Length`) body is treated identically to a fixed-length one. |
|
||||
| `INVALID_JSON` | 400 | Request body was not valid JSON. |
|
||||
| `VALIDATION_FAILED` | 400 | Parameter validation failed (missing required field, wrong type, undeclared field). |
|
||||
| `BODY_TOO_LARGE` | 413 | Request body exceeded `MaxRequestBodyBytes`. |
|
||||
| `STANDBY_NODE` | 503 | Request hit a standby central node; the inbound API serves only the active node. |
|
||||
| `TIMEOUT` | 500 | The method script exceeded its execution timeout. |
|
||||
| `SITE_UNREACHABLE` | 500 | A routed `Route.To(...).Call(...)` could not reach the target site. |
|
||||
| `SITE_UNREACHABLE` | 500 | The target site did not respond to the routed request within the routing timeout (no contact / down / partitioned) — typed from the routed Ask expiring (`AskTimeoutException`), never inferred from an error-message substring. A `Success=false` response means the site answered and is classified `SCRIPT_ERROR`. |
|
||||
| `SCRIPT_COMPILE_FAILED` | 500 | The method's script failed to compile (returned consistently on every node until a compiling version is saved). |
|
||||
| `SCRIPT_ERROR` | 500 | Catch-all for any other script-execution failure (details are logged centrally, never leaked to the caller). |
|
||||
|
||||
|
||||
@@ -88,9 +88,9 @@ A timer fires every `SampleInterval` (default 60s; an immediate first tick prime
|
||||
|
||||
**Best-effort, per-source isolation.** Each source call and the write are individually guarded. A throwing source is logged and its samples skipped for that tick; it never aborts the tick, the other sources, or the source component itself. This is the same `IEnumerable<>`-of-adapters decoupling pattern used by `INotificationDeliveryAdapter`.
|
||||
|
||||
**Hourly rollup tick.** A third timer (`kpi-rollup`) fires every `RollupInterval` (default 1h). On each tick the recorder opens a per-tick DI scope and calls `IKpiHistoryRepository.FoldHourlyRollupsAsync` over the window `[TruncateToHour(now) − RollupLookbackHours, TruncateToHour(now))` — an **exclusive** upper bound at the current hour start, so the in-progress hour is never folded. The fold groups raw `KpiSample` rows by `(series, hour)`, applies the per-metric aggregation intent (sum for rate metrics, last-value for gauges — see Query), and **idempotently upserts** each `(Source, Metric, Scope, ScopeKey, HourStartUtc)` row. Re-folding the trailing lookback window (default 3h) rather than only the last hour is the self-heal for a **singleton-failover-missed tick**: because the upsert overwrites the aggregate in place, re-running the same window produces identical values (no double-count), so any complete hour a missed tick skipped is refilled on the next fold. A `_rollupInFlight` guard (mirroring `_sampleInFlight`) skips a tick while a prior fold is still running so two idempotent upserts never race on overlapping rows.
|
||||
**Hourly rollup tick.** A third timer (`kpi-rollup`) fires every `RollupInterval` (default 1h). On each tick the recorder opens a per-tick DI scope and calls `IKpiHistoryRepository.FoldHourlyRollupsAsync` over the window `[TruncateToHour(now) − RollupLookbackHours, TruncateToHour(now))` — an **exclusive** upper bound at the current hour start, so the in-progress hour is never folded. The fold groups raw `KpiSample` rows by `(series, hour)`, applies the per-metric aggregation intent (sum for rate metrics, last-value for gauges — see Query), and **idempotently upserts** each `(Source, Metric, Scope, ScopeKey, HourStartUtc)` row. Re-folding the trailing lookback window (default 3h) rather than only the last hour is the self-heal for a **singleton-failover-missed tick**: because the upsert overwrites the aggregate in place, re-running the same window produces identical values (no double-count), so any complete hour a missed tick skipped is refilled on the next fold. A `_rollupInFlight` guard (mirroring `_sampleInFlight`) skips a tick while a prior fold is still running so two idempotent upserts never race on overlapping rows. During a **failover overlap** — the old singleton incarnation's in-flight fold and the new node's first fold both `Add`-ing the same `(series, hour)` row — the loser of the unique `IX_KpiRollupHourly_Series` upsert race discards its **whole fold pass** (a single `SaveChanges`), logged at Information and repaired by the next idempotent fold; when reading fold logs after a failover, the failure grain is the **pass, not the row** (arch-review 04 round 2, R5).
|
||||
|
||||
**One-shot backfill.** So the 30 d / 90 d charts are not blank until enough wall-clock passes, the recorder runs a **single** backfill on start (`kpi-rollup-backfill` timer) that folds the full retention window once, `[TruncateToHour(now) − RetentionDays, TruncateToHour(now))`, reusing the same `FoldHourlyRollupsAsync` path. A `_backfillDone` latch makes it at-most-once per actor lifetime, and it defers (re-arms) while a periodic fold is in flight; being an idempotent upsert, it is cheap and safe to re-run on a later failover restart.
|
||||
**One-shot backfill.** So the 30 d / 90 d charts are not blank until enough wall-clock passes, the recorder runs a backfill on start (`kpi-rollup-backfill` timer) that folds the retention window, reusing the same `FoldHourlyRollupsAsync` path. The `FoldHourlyRollupsAsync` repository call materializes its whole window in memory, so the backfill folds the window in **≤24 h slices, oldest-first** — one bounded slice per fold — rather than handing the fold the full 90-day window at once (which at fleet volume is tens of millions of rows in one tracked pass). Between slices it lowers its in-flight guard so periodic folds **interleave** instead of stalling behind the historical pass; a periodic fold and a backfill slice strictly alternate through the single `_rollupInFlight`/`_backfillInFlight` gate pair, so the two idempotent upserts never race on overlapping rows. On a **failover restart** the plan first consults the `IKpiHistoryRepository.GetLatestRollupHourAsync` watermark: if the newest folded hour is already within `RollupLookbackHours` of now (the common case) the whole historical pass is **skipped** — the periodic fold's lookback covers the tail — otherwise the window is **shrunk to the un-rolled tail** `[watermark, TruncateToHour(now))` (re-folding the watermark hour itself as an idempotent safety margin) instead of the full `RetentionDays` floor. A `_backfillDone` latch makes the whole pass at-most-once per actor lifetime, and it defers (re-arms) while a periodic fold is in flight. Being an idempotent upsert, it is safe (idempotent) to re-run on a later failover restart; the watermark fast-path makes the failover re-run cheap as well.
|
||||
|
||||
**Retention.** A daily purge timer (`PurgeInterval`, default 24h) now runs **both** purges: raw `KpiSample` rows older than `RetentionDays` (default 90) via `IKpiHistoryRepository.PurgeOlderThanAsync`, **and** hourly rollup rows older than `RollupRetentionDays` (default 365) via `IKpiHistoryRepository.PurgeRollupsOlderThanAsync` (a one-hour-sliced batched DELETE mirroring the raw purge). Rollups deliberately **outlive** raw samples so long-range trend charts have data after the raw rows are purged; the options validator enforces `RollupRetentionDays >= RetentionDays` so a window never falls into a hole where raw is purged but no rollup exists.
|
||||
|
||||
@@ -148,15 +148,17 @@ Reads `ICentralHealthAggregator.GetAllSiteStates()` (in-memory, no DB). Scope: *
|
||||
|
||||
### Bucketed query
|
||||
|
||||
`IKpiHistoryRepository.GetRawSeriesAsync(source, metric, scope, scopeKey, fromUtc, toUtc, …)` returns the raw points for one series over `[fromUtc, toUtc]`. `GetHourlySeriesAsync(…)` returns the same ascending `KpiSeriesPoint` shape from `KpiRollupHourly` (projecting `HourStartUtc` → the point timestamp, `Value`), so the bucketer is agnostic to the source. `KpiSeriesBucketer.Bucket(series, fromUtc, toUtc, maxPoints)` then partitions the window into ≤ `maxPoints` time buckets and returns the **last value per bucket** as `KpiSeriesPoint(BucketStartUtc, Value)` — unchanged, and still applied on top of a rollup series (a 90 d rollup is ~2,160 points > the 200-point cap, so bucketing still thins it). The chart plots the single `Value` series; avg / min / max charting is still deferred, though the rollup's `MinValue` / `MaxValue` / `SampleCount` columns are now stored to unblock it.
|
||||
`IKpiHistoryRepository.GetRawSeriesAsync(source, metric, scope, scopeKey, fromUtc, toUtc, …)` returns the raw points for one series over `[fromUtc, toUtc]`. `GetHourlySeriesAsync(…)` returns the same ascending `KpiSeriesPoint` shape from `KpiRollupHourly` (projecting `HourStartUtc` → the point timestamp, `Value`), so the bucketer is agnostic to the source. `KpiSeriesBucketer.Bucket(series, fromUtc, toUtc, maxPoints, aggregation)` then partitions the window into ≤ `maxPoints` time buckets and reduces each bucket **per-metric**: it keeps the **last value** for a **Gauge** series and **sums per bucket** for a **Rate** series, driven by the same `KpiMetricAggregationCatalog` the hourly fold consults. The previous unconditional last-value-per-bucket kept **1 of ~11 hourly sums** per 90 d bucket on the rollup path — the exact fold error the catalog's own doc warns about, re-introduced one layer up — and is fixed per arch-review 04 round 2, R3. The reduction is still applied on top of a rollup series (a 90 d rollup is ~2,160 points > the 200-point cap, so bucketing still thins it). The chart plots the single `Value` series; avg / min / max charting is still deferred, though the rollup's `MinValue` / `MaxValue` / `SampleCount` columns are now stored to unblock it.
|
||||
|
||||
### Raw-vs-rollup range routing
|
||||
|
||||
`GetRawSeriesAsync` and `GetHourlySeriesAsync` are selected by window width in the query service (below), governed by `RollupThresholdHours` (default 168 = 7 d). Windows **at or below** the threshold read raw samples (preserving intra-minute detail on the 24 h / 7 d charts); windows **strictly wider** than the threshold read the pre-aggregated hourly rollups (≈60× fewer rows scanned on the 30 d / 90 d charts). The boundary is strict — a window of exactly `RollupThresholdHours` stays on the raw path.
|
||||
|
||||
Rate charts keep **one unit (events per chart bucket)** on both sides of the boundary: the catalog-driven per-bucket sum reduction is applied to the raw-minute deltas and the hourly rollup sums alike, so widening 7 d → 30 d no longer changes a Rate chart's magnitude ~60×. Residual: an unbucketed just-over-threshold rollup series plots native per-hour sums vs. the at-threshold raw path's ~50-minute bucket sums (≤~1.2×) — accepted and documented, not hidden.
|
||||
|
||||
### Aggregation intent — gauge vs. rate
|
||||
|
||||
Not every metric folds the same way, and choosing wrong silently corrupts long-range totals. `KpiMetricAggregationCatalog.Resolve(source, metric)` classifies each `(Source, Metric)` pair as `KpiRollupAggregation.Gauge` (fold = **last-value-per-hour**) or `KpiRollupAggregation.Rate` (fold = **sum-per-hour**):
|
||||
Not every metric folds the same way, and choosing wrong silently corrupts long-range totals. `KpiMetricAggregationCatalog.Resolve(source, metric)` classifies each `(Source, Metric)` pair as `KpiRollupAggregation.Gauge` (fold = **last-value-per-hour**) or `KpiRollupAggregation.Rate` (fold = **sum-per-hour**). The catalog now has **two consumers**: the hourly rollup fold (which picks last-value vs. sum per hour) and the chart-time bucket reduction in `KpiSeriesBucketer` (which picks last-value vs. sum per chart bucket) — one classification keeps both layers consistent:
|
||||
|
||||
- **Gauge** (last-value) — instantaneous readings where a per-hour sum is meaningless: `queueDepth`, `buffered`, `stuck`/`stuckCount`, `parked*`, `oldestPendingAgeSeconds`, `connectionsUp`/`connectionsDown`, `deployedInstances`, and every other metric not explicitly listed as a rate. This is the **safe default**: an unknown / newly-added metric resolves to Gauge and never crashes the fold or invents a total.
|
||||
- **Rate** (sum-per-hour) — already-windowed per-interval deltas whose hour total is the sum of its minute samples: `NotificationOutbox/deliveredLastInterval`, `SiteCallAudit/failedLastInterval`, `SiteCallAudit/deliveredLastInterval`, and the per-report `SiteHealth` error counts `scriptErrors` / `alarmEvalErrors` / `deadLetters` / `eventLogWriteFailures`. Folding these as last-value would keep one minute's delta and discard the other 59, under-counting every long-range total.
|
||||
|
||||
@@ -151,6 +151,8 @@ This allows the compile to bind `Attributes["name"]`, `Notify.To("x").Send(...)`
|
||||
|
||||
Mirrors `TriggerExpressionGlobals` in the same way. Used by `ValidationService.CheckExpressionSyntax` in the Template Engine for conditional and expression trigger validation.
|
||||
|
||||
`CheckExpressionSyntax` memoises its verdict in the Template Engine's process-wide `ScriptCompileVerdictCache`, whose key is the pair **(globals surface, SHA-256 of the code)** — not the code alone. The surface discriminator is load-bearing: a trigger expression vetted against `TriggerCompileSurface` is **not** interchangeable with a `ScriptCompileSurface` script-body verdict (different globals resolve different identifiers), so a code-only key could return a stale "clean" for code never compiled against the caller's surface. Keying by surface makes that cross-surface verdict reuse structurally impossible.
|
||||
|
||||
#### Parity guard
|
||||
|
||||
A reflection-based parity test in `SiteRuntime.Tests` compares the public member names on `ScriptCompileSurface` against `ScriptGlobals` (and `TriggerCompileSurface` against `TriggerExpressionGlobals`). Any drift between the stub and the real globals causes this test to fail, ensuring the stubs cannot silently fall out of sync.
|
||||
@@ -170,7 +172,7 @@ All five call sites that previously maintained their own script trust enforcemen
|
||||
| **Site Runtime** `ScriptCompilationService.ValidateTrustModel` | Semantic resolver, no reflection-gateway hardening | Delegates to `FindViolations`; retains `CSharpScript.Compile` against real `ScriptGlobals` for execution |
|
||||
| **Inbound API** `ForbiddenApiChecker.FindViolations` | Syntactic walker, forbade all `System.Diagnostics` | Thin shim delegating to `ScriptTrustValidator.FindViolations`; `System.Diagnostics` loosened to `.Process`-only |
|
||||
| **Central UI** `ScriptAnalysisService` | Semantic + full compile, lenient threading | Delegates forbidden-API verdict and sources editor-marker deny-list from `ScriptTrustPolicy`; retains Test-Run execution host |
|
||||
| **Transport** `BundleImporter.RunSemanticValidationAsync` / `DetectBlockersAsync` | No trust gate — a bundle's forbidden-API scripts imported clean, deferred to runtime (fifth write path) | Runs `ScriptTrustValidator.FindViolations` over every non-Skip template / shared / ApiMethod body at import review — hard error (apply) / Blocker row (preview) for all kinds |
|
||||
| **Transport** `BundleImporter.RunSemanticValidationAsync` / `DetectBlockersAsync` | No trust gate — a bundle's forbidden-API scripts imported clean, deferred to runtime (fifth write path) | Runs `ScriptTrustValidator.FindViolations` over every non-Skip template / shared / ApiMethod body, template script + alarm Expression-trigger bodies, **and instance alarm-override trigger expressions** at import review — hard error (apply) / Blocker row (preview) for all kinds |
|
||||
|
||||
The static enforcement is **defence-in-depth**, not a true runtime sandbox. Scripts execute in-process; the denied API list prevents obvious escapes at compile time but does not provide the isolation guarantees of an out-of-process sandbox or a restricted `AssemblyLoadContext`. This caveat applies to all consumers.
|
||||
|
||||
@@ -190,4 +192,4 @@ No dependency on Akka.NET, ASP.NET Core, Entity Framework, or any other ScadaBri
|
||||
- **Site Runtime (#3)**: `ScriptCompilationService.ValidateTrustModel` delegates the trust verdict to `ScriptTrustValidator.FindViolations`; retains its own `CSharpScript.Compile` against the real `ScriptGlobals` for execution-time compilation.
|
||||
- **Inbound API (#14)**: `ForbiddenApiChecker.FindViolations` is a thin shim over `ScriptTrustValidator.FindViolations`.
|
||||
- **Central UI (#9)**: `ScriptAnalysisService` delegates the run-gate forbidden-API verdict and sources the editor-marker deny-list from `ScriptTrustPolicy`; retains the Test-Run execution host (`SandboxScriptHost`).
|
||||
- **Transport (#24)**: `BundleImporter` runs `ScriptTrustValidator.FindViolations` over every non-Skip template, shared, and ApiMethod script body during import validation (`RunSemanticValidationAsync` Pass 0) and preview (`DetectBlockersAsync`), rejecting forbidden-API scripts at import review rather than at runtime. Trust violations are hard errors for all script kinds (unlike the false-positive-prone name-resolution heuristic, whose template-script findings are advisory).
|
||||
- **Transport (#24)**: `BundleImporter` runs `ScriptTrustValidator.FindViolations` over every non-Skip template, shared, and ApiMethod script body, template script + alarm Expression-trigger bodies, **and instance alarm-override trigger expressions** (`InstanceAlarmOverrideDto.TriggerConfigurationOverride`) during import validation (`RunSemanticValidationAsync` Pass 0) and preview (`DetectBlockersAsync`), rejecting forbidden-API scripts at import review rather than at runtime. Trust violations are hard errors for all script kinds (unlike the false-positive-prone name-resolution heuristic, whose template-script findings are advisory).
|
||||
|
||||
@@ -138,7 +138,7 @@ Set in a local or docker-dev environment via the environment variable `ScadaBrid
|
||||
- Approve or reject a pending secured write from the Secured Writes page — the *approving* half of the two-person write workflow.
|
||||
- **Purpose**: The approving counterpart to **Operator**. Separation of duties is enforced **server-side**: the ManagementActor rejects any approval where the approving user equals the submitting Operator (no self-approval), so the two roles must be held by distinct principals for a write to execute. (See Component-ManagementService.md and Component-CentralUI.md.)
|
||||
|
||||
> **Two-person secured-write workflow.** `Operator` and `Verifier` are deliberately separate global roles so a single principal cannot both initiate and approve a write through the MxAccess Gateway. Both are coarse global roles like the others; any site scoping is layered on at the LDAP-mapping level.
|
||||
> **Two-person secured-write workflow.** `Operator` and `Verifier` are deliberately separate global roles so a single principal cannot both initiate and approve a write through the MxAccess Gateway. Both are coarse global roles like the others. **Site scoping from the LDAP mapping is enforced server-side on every secured-write command** (arch-review R2 N3): submit checks the target site, approve/reject check the row's site (before the TTL/self-approval/CAS chain), and the list constrains a scoped caller to their permitted sites — mirroring the deployment commands' `EnforceSiteScope` rule. Administrators and system-wide principals (empty permitted-site set) are unrestricted.
|
||||
>
|
||||
> **Deployment-configuration hazard — never grant one principal both roles.** The whole control collapses if the same identity (or LDAP group) maps to *both* `Operator` and `Verifier`: server-side no-self-approval blocks approving the *exact* write you submitted, but a dual-role principal can still trivially pair-approve with a second submission, so a single compromised or careless account regains unilateral write. Keep `SCADA-Operators` and `SCADA-Verifiers` group membership disjoint. The dev `DisableLogin` caveat is the extreme of this: with `DisableLogin` on, the auto-login principal holds **all** roles, so the two-person flow cannot be exercised end-to-end by a single identity — which is exactly why `DisableLogin=true` is refused outside Development (see the `DisableLoginGuard` note above). No-self-approval is covered by handler tests; real two-person use requires two real, distinct-role identities.
|
||||
>
|
||||
@@ -205,7 +205,8 @@ Role checks are expressed as named ASP.NET Core authorization policies (in `Auth
|
||||
|
||||
To blunt online password-spray and brute-force against the directory, every LDAP-bind surface consults a shared in-memory throttle (`LoginThrottle`, a `ZB.MOM.WW.ScadaBridge.Security` singleton) before attempting a bind, and records the bind outcome afterwards.
|
||||
|
||||
- **Scope — every LDAP-bind surface**: the Central UI interactive login (`POST /auth/login`) and CLI token issue (`POST /auth/token`), plus the HTTP-Basic management/CLI surfaces fronted by `ManagementAuthenticator` — `POST /management`, the audit REST endpoints, and the debug-stream hub. No bind happens outside these paths, so throttling coverage is complete. The dev/test `DisableLogin` bypass sits **above** the throttle: a login-disabled deployment never binds, so it is never throttled.
|
||||
- **Key and window**: counters are keyed per `{username}|{IP}` pair (lower-cased username; remote IP from the connection). A fixed window of `LoginFailureWindowMinutes` (default 5) opens on the first failure; reaching `MaxLoginFailuresPerWindow` failures (default 5) within it locks that key out for `LoginLockoutMinutes` (default 5). A successful bind clears the key; an expired window resets the count. Setting `MaxLoginFailuresPerWindow` to `0` disables throttling entirely.
|
||||
- **Scope — every LDAP-bind surface**: the Central UI interactive login (`POST /auth/login`) and CLI token issue (`POST /auth/token`), plus the HTTP-Basic management/CLI surfaces fronted by `ManagementAuthenticator` — `POST /management`, the audit REST endpoints, **and the debug-stream hub (`DebugStreamHub.OnConnectedAsync` delegates its whole credential path to `ManagementAuthenticator.AuthenticateAsync`, closing the round-2 N1 gap where the hub bound LDAP directly and bypassed the throttle)**. No bind happens outside these paths. The dev/test `DisableLogin` bypass sits **above** the throttle: a login-disabled deployment never binds, so it is never throttled.
|
||||
- **Key and window**: counters are keyed per `{username}|{IP}` pair (lower-cased username; the IP half is the **forwarded-header-resolved client IP** when `ScadaBridge:Security:ForwardedHeaders` names a trusted proxy, otherwise the raw connection peer). A fixed window of `LoginFailureWindowMinutes` (default 5) opens on the first failure; reaching `MaxLoginFailuresPerWindow` failures (default 5) within it locks that key out for `LoginLockoutMinutes` (default 5). A successful bind clears the key; an expired window resets the count. Setting `MaxLoginFailuresPerWindow` to `0` disables throttling entirely.
|
||||
- **Proxy topologies and the username-lockout DoS.** The key is deliberately `{username}|{IP}`, not `{username}` alone: keying on username alone would let any network-adjacent actor lock any operator out of a SCADA control surface with five wrong passwords, repeatable indefinitely. That isolation only exists when the throttle sees real client IPs — behind a proxy **without** ForwardedHeaders trust, all clients collapse onto the proxy's IP and the DoS returns. Mitigations: (a) the shipped Traefik topologies enable trusted-proxy ForwardedHeaders (so per-IP isolation is real in the documented deployment); (b) the lockout window is short by default (`LoginLockoutMinutes` = 5) and never touches the directory account itself; (c) residual risk — an attacker spraying from their *own* IP locking out a victim username *at that IP only* — is the throttle working as designed. A success-path bypass ("a correct password unlocks") was considered and rejected: it would let an attacker who has the password bypass the lockout entirely, and it converts the throttle into a password oracle during the lockout window.
|
||||
- **Behaviour when locked**: the surface refuses the bind without contacting LDAP. The Basic-Auth surfaces (`ManagementAuthenticator`) and `POST /auth/token` return HTTP `429` with `code = "AUTH_THROTTLED"`; `POST /auth/login` redirects to `/login` with a "Too many failed attempts. Try again later." message.
|
||||
- **Best-effort, per-node**: the throttle is in-memory with no external state. The two central nodes maintain independent counters (a spray hitting both simply burns its budget on each). Memory is bounded — writes prune expired keys and cap the tracked-key count. This is a rate-limit guard, not an account-lockout policy; it never disables the directory account itself.
|
||||
|
||||
@@ -96,7 +96,7 @@ flowchart TD
|
||||
|
||||
> **Startup config load is retried (S5)**: step 1's deployed-config read is best-effort against local SQLite, which can transiently fail (locked/busy). A failed load must **not** leave the site as a silent zero-instance node — it is re-attempted on a fixed interval until it succeeds, then the retry self-cancels.
|
||||
|
||||
> **Per-instance compilation during staggered startup (P6, follow-up)**: the Roslyn compile of each instance's scripts still runs inside Instance Actor start during staggered startup; moving it off-thread is a deferred optimization (it affects failover time-to-recover only, not correctness).
|
||||
> **Per-instance compilation during staggered startup (P6, follow-up)**: the Roslyn compile of each instance's scripts still runs inside Instance Actor start during staggered startup; moving it off-thread is a deferred optimization (it affects failover time-to-recover only, not correctness). As of the round-2 hardening, a process-wide compile cache dedupes identical script bodies within a node's process lifetime (the deploy gate's compile is reused by Instance Actor start), shrinking the recompile cost; the *first* compile after process start still runs inside Instance Actor start, so the deferral stands.
|
||||
|
||||
### Deployment Handling
|
||||
- Receives flattened instance configurations from central via the Communication Layer.
|
||||
@@ -238,6 +238,7 @@ When the Instance Actor is stopped (due to disable, delete, or redeployment), Ak
|
||||
- **Range Violation**: Value is outside the allowed min/max range.
|
||||
- **Rate of Change**: Value change rate exceeds the defined threshold over a configurable time window. Direction filter (rising / falling / either) restricts which side of the rate triggers.
|
||||
- **HiLo**: Multi-setpoint level alarm with up to four configurable setpoints (LoLo, Lo, Hi, HiHi). Any subset may be configured. Each setpoint may carry its own priority that overrides the alarm-level priority for that band.
|
||||
- **Expression** trigger evaluation runs on the bounded script-execution thread pool shared with script bodies (and Expression *script* triggers): scheduler saturation — e.g. stuck scripts occupying all threads — delays Expression alarm transitions site-wide until a thread frees. This is a deliberate trade-off (evaluation no longer blocks dispatcher threads; per-actor coalescing bounds queue growth) and the scheduler gauges + stuck-script watchdog make the saturated state visible.
|
||||
- For binary trigger types (ValueMatch / RangeViolation / RateOfChange), when the condition is met and the alarm is currently in **normal** state, the alarm transitions to **active**:
|
||||
- Updates the alarm state on the parent Instance Actor (which publishes to the Akka stream).
|
||||
- If an on-trigger script is defined, spawns an Alarm Execution Actor to execute it.
|
||||
@@ -486,7 +487,7 @@ Per Akka.NET best practices, internal actor communication uses **Tell** (fire-an
|
||||
### Script Compilation Errors
|
||||
- If script compilation fails when a deployment is received, the entire deployment for that instance is **rejected**. No partial state is applied.
|
||||
- The failure is reported back to central as a failed deployment.
|
||||
- **Enforced by a site-side pre-compile validation gate (S3)**: before the Instance Actor is created or the config persisted, the Deployment Manager compiles **all** of the instance's scripts, alarm on-trigger scripts, and trigger expressions **off-thread**; any compile failure rejects the deployment with the collected compile errors. This makes the deployment result honest even if central's pre-deployment validation was bypassed or skew let a bad artifact through.
|
||||
- **Enforced by a site-side pre-compile validation gate (S3)**: before the Instance Actor is created or the config persisted, the Deployment Manager compiles **all** of the instance's scripts, alarm on-trigger scripts, and trigger expressions **synchronously, as a pure prefix step of the deploy handler on the singleton's actor thread** — deliberately not off-thread, because piping the verdict back to self would reorder concurrent deploys against delete/disable and break the redeploy-supersede mailbox-FIFO ordering (see the in-code rationale in `DeploymentManagerActor.HandleDeploy`); unchanged script bodies hit the process-wide compile cache, so the gate recompiles only genuinely new code. Any compile failure rejects the deployment with the collected compile errors. This makes the deployment result honest even if central's pre-deployment validation was bypassed or skew let a bad artifact through.
|
||||
- Note: Pre-deployment validation at central should still catch compilation errors before they reach the site; the site gate is the authoritative backstop.
|
||||
|
||||
---
|
||||
|
||||
@@ -80,7 +80,7 @@ There is **no maximum buffer size**. Messages accumulate in the buffer until del
|
||||
- The standby node applies the same operations to its own local SQLite database but is **passive**: it never runs the delivery sweep. The retry sweep is **gated to the active node** (the oldest Up member / singleton host, re-evaluated every sweep tick), so only one node delivers at a time. The standby applies replicated operations purely to keep its copy warm for a future failover.
|
||||
- On failover, the new active node has a near-complete copy of the buffer. In rare cases, the most recent operations may not have been replicated (e.g., a message added or removed just before failover). This can result in a few **duplicate deliveries** (message delivered but its `Remove` not yet replicated) or a few **missed retries** (message added but not replicated). Duplicate deliveries are therefore confined to the **failover window** — an in-flight delivery whose `Remove` had not yet replicated — and never occur in steady-state operation (the standby's gate keeps it from delivering the same rows). Both are acceptable trade-offs for the latency benefit.
|
||||
- On failover, the new active node's gate flips to active within one sweep interval and it resumes delivery from its local copy.
|
||||
- **Peer-join anti-entropy resync.** Asynchronous, no-ack replication keeps the standby warm in steady state, but a standby that was **down for an extended period** (a crash, a long maintenance window) misses every operation replicated while it was gone and would otherwise diverge from the active node's buffer forever. To close that gap, whenever a node **(re)tracks its peer**, a **standby** requests a full-buffer snapshot (`RequestSfBufferResync`); the **active** node answers with up to `MaxResyncRows` (10 000) of its oldest rows (`SfBufferSnapshot`), and the standby **replaces its entire local buffer** with that snapshot (`ReplaceAllAsync`, one transaction). Only the active node answers; only a standby applies (each side checks the repo-standard leader+Up active-node predicate, safe-by-default to standby). Because replicated applies are **upserts** (see the replication apply path), any Add/Remove/Park that lands after the resync merges cleanly onto the resynced state — no primary-key conflict, no lost delta. If the buffer exceeds the 10 000-row cap the snapshot is flagged `Truncated` and the standby logs a Warning; the residual divergence beyond the cap drains naturally as the active node delivers.
|
||||
- **Peer-join anti-entropy resync (chunked, ack-confirmed).** Asynchronous, no-ack replication keeps the standby warm in steady state, but a standby that was **down for an extended period** (a crash, a long maintenance window) misses every operation replicated while it was gone and would otherwise diverge from the active node's buffer forever. To close that gap, whenever a node **(re)tracks its peer**, a **standby** requests a full-buffer snapshot (`RequestSfBufferResync`); the **active** node loads up to `MaxResyncRows` (10 000) of its oldest rows and answers with a **sequence of byte-budgeted chunks** (`SfBufferSnapshotChunk`), each carrying a shared `ResyncId`, a 1-based `Sequence`, and the `TotalChunks` count. Chunking is mandatory because the monolithic snapshot exceeds Akka remoting's **default 128 000-byte frame** for any realistic backlog and `BuildHocon` sets no override — a single oversized message is silently undeliverable (review 02 round 2, **N2 High**). Rows accumulate into a chunk until the estimated payload budget (`MaxResyncChunkBytes` = 64 000, ≈50% frame headroom) or the row cap (`MaxResyncChunkRows` = 200) is hit; a single row whose payload alone exceeds the budget ships solo with a Warning. The standby **assembles all chunks of one `ResyncId`** (a new `ResyncId` discards any stale partial assembly; a partial that never completes is dropped after `resyncAssemblyTimeout`, default 30 s, and counted a replication failure), then **replaces its entire local buffer** with the assembled snapshot (`ReplaceAllAsync`, one transaction) and returns a delivery confirmation (`SfBufferResyncAck`). The active node arms an ack window (`resyncAckTimeout`, default 60 s): an acknowledged resync increments `scadabridge.store_and_forward.resync.completed`; an unacknowledged one (lost chunks / dead peer) logs a Warning and increments `scadabridge.store_and_forward.resync.ack_missing` — closing N2's silent-loss mode (previously nothing counted a lost snapshot and nothing retried until the next peer-track). Only the active node answers; only a standby applies, and both sides re-check at apply time (a mid-flight active-flip aborts the wipe) — each side checks the repo-standard **oldest-Up member** active-node predicate (singleton-host semantics via the shared `ActiveNodeEvaluator`, the **same predicate as the S&F delivery gate**; review 02 round 2, **N1 Critical** — using cluster *leadership* here let a rolling restart of the lower-address node make the delivering node wipe its own live buffer). Because replicated applies are **upserts** (see the replication apply path), any Add/Remove/Park that lands after the resync merges cleanly onto the resynced state — no primary-key conflict, no lost delta; the one accepted exception is the rare **N5** orphan-row race (a replicated `Remove` ordered before the snapshot chunks can re-add the removed row, re-delivered once and self-corrected at the next resync — inherent to no-ack replication). If the buffer exceeds the 10 000-row cap the snapshot is flagged `Truncated` and the standby logs a Warning; the residual divergence beyond the cap drains naturally as the active node delivers. The legacy monolithic `SfBufferSnapshot` message + standby handler are **retained** for rolling-upgrade compatibility (an old active node's monolithic snapshot is still applied by a new standby).
|
||||
|
||||
### Operation Tracking Table (lives in Site Runtime, not here)
|
||||
|
||||
|
||||
@@ -184,6 +184,8 @@ Before a deployment is sent to a site, the Template Engine performs comprehensiv
|
||||
- **Script compilation and trust check**: All instance scripts and alarm on-trigger scripts are compiled via `ScriptCompiler.TryCompile`, which performs a real Roslyn type-checking compile (using `ScriptCompileSurface` from the Script Analysis component as the globals type) and an authoritative forbidden-API check (via `ScriptTrustValidator.FindViolations`). Scripts with compilation errors, type errors, or forbidden-API violations are rejected. This gate is authoritative — not advisory — meaning alias, `using static`, and `global::` bypasses are caught by semantic symbol resolution.
|
||||
- **Alarm trigger references**: Alarm trigger definitions reference attributes that exist in the flattened configuration.
|
||||
- **Script trigger references**: Script triggers (value change, conditional) reference attributes that exist in the flattened configuration.
|
||||
- **Expression-trigger syntax/trust check**: Each Expression-triggered script/alarm has its boolean expression run through `ValidationService.CheckExpressionSyntax` — a `ScriptTrustValidator.FindViolations` forbidden-API verdict followed by a real Roslyn compile against `TriggerCompileSurface`. Both stages report **all** findings (every forbidden API, every compile error), not just the first, so an operator fixing a multi-error expression sees the full list in one round-trip. The verdict is memoised in the surface-keyed `ScriptCompileVerdictCache`, so an unchanged expression compiles once per process.
|
||||
- **Read-only skip (`validateScriptCompilation: false`)**: The `Validate` flag that read-only staleness/comparison callers (DeploymentManager's `FlatteningPipeline` on its comparison/probe paths) pass to skip the expensive Roslyn script-body compile now **also** skips the Expression-trigger syntax/compile check — the two compile-bearing stages. The cheap structural checks (blank-expression, attribute-reference scan) still run, and the deploy gate keeps the default (`true`), so it remains the authoritative check for both stages.
|
||||
- **Data connection binding completeness**: Every attribute with a data source reference has a data connection binding assigned on the instance, and the bound data connection name exists as a defined connection at the instance's site.
|
||||
- **Exception**: Validation does **not** verify that data source relative paths resolve to real tags on physical devices — that is a runtime concern.
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ As of M8 (T18), Transport is no longer limited to central-only configuration: it
|
||||
- Validate `manifest.json` on upload: format version gating, SHA-256 content hash verification.
|
||||
- Manage in-memory `BundleSession` objects: 30-minute TTL, 3-strike passphrase lockout per session.
|
||||
- Compute a per-artifact diff between bundle contents and the target environment, classifying each artifact as Identical, Modified, New, a Blocker (import-stopping), or a Warning (advisory, non-blocking).
|
||||
- Apply user-supplied conflict resolutions (Add, Overwrite, Skip, Rename) in a single EF transaction, running multi-pass semantic validation before committing: a **script trust gate** (`ScriptTrustValidator.FindViolations` over every non-Skip template / shared / ApiMethod body — bundle import is the fifth script-trust call site, so forbidden-API scripts are rejected at import review, not deferred to runtime), then a minimal name-resolution scan over the merged target (ApiMethod findings fail fast on unresolved SharedScript / ExternalSystem identifiers; template-script findings are advisory warnings), then the full `SemanticValidator` from `ZB.MOM.WW.ScadaBridge.TemplateEngine` over each imported template's per-template `FlattenedConfiguration`.
|
||||
- Apply user-supplied conflict resolutions (Add, Overwrite, Skip, Rename) in a single EF transaction, running multi-pass semantic validation before committing: a **script trust gate** (`ScriptTrustValidator.FindViolations` over every non-Skip executable C# surface a bundle carries — template / shared / ApiMethod script bodies, template script **and** alarm **Expression-trigger** bodies, **and instance alarm-override trigger expressions** (`InstanceAlarmOverrideDto.TriggerConfigurationOverride`); bundle import is the fifth script-trust call site, so forbidden-API scripts are rejected at import review, not deferred to runtime — all these surfaces hard-block uniformly, since the trust gate's semantic verdict is authoritative with no false-positive channel, and the advisory-vs-hard severity split is a property of the name-resolution heuristic only), then a minimal name-resolution scan over the merged target (ApiMethod findings fail fast on unresolved SharedScript / ExternalSystem identifiers; template-script findings are advisory warnings), then the full `SemanticValidator` from `ZB.MOM.WW.ScadaBridge.TemplateEngine` over each imported template's per-template `FlattenedConfiguration`.
|
||||
- Emit `BundleExported`, `BundleImported`, `BundleImportFailed`, `UnencryptedBundleExport`, `BundleImportUnlockFailed`, `BundleImportAlarmScriptUnresolved`, `BundleImportCompositionUnresolved`, and `BundleImportBaseTemplateUnresolved` audit events via `IAuditService`.
|
||||
- Thread a `BundleImportId` correlation GUID through every per-entity `AuditLogEntry` written during `ApplyAsync` via a scoped `IAuditCorrelationContext`.
|
||||
- Enforce `RequireDesign` on export and `RequireAdmin` on import both at the Razor page layer and inside the service entrypoints (defense in depth).
|
||||
@@ -133,7 +133,7 @@ flowchart TD
|
||||
class T muted
|
||||
```
|
||||
|
||||
The component is central-hosted. It is registered in `ZB.MOM.WW.ScadaBridge.Host` for central roles only, never for site roles — even when a bundle carries site-scoped config (T18), Transport only writes the central configuration database, never the site clusters. All persistence flows through existing audited repository interfaces in `ZB.MOM.WW.ScadaBridge.ConfigurationDatabase` — the component does not call `DbContext.SaveChangesAsync` directly. `BundleSessionStore` is in-process on the active central node (matching Blazor Server circuit affinity): 30-minute TTL, eviction on expiry, 3-strike passphrase lockout per session, and a `MaxConcurrentImportSessions` cap (default 8; arch-review 05 Task 24) bounding the decrypted-content footprint since each open session pins a fully-decrypted bundle in memory.
|
||||
The component is central-hosted. It is registered in `ZB.MOM.WW.ScadaBridge.Host` for central roles only, never for site roles — even when a bundle carries site-scoped config (T18), Transport only writes the central configuration database, never the site clusters. All persistence flows through existing audited repository interfaces in `ZB.MOM.WW.ScadaBridge.ConfigurationDatabase` — the component does not call `DbContext.SaveChangesAsync` directly. `BundleSessionStore` is in-process on the active central node (matching Blazor Server circuit affinity): 30-minute TTL, eviction on expiry, 3-strike passphrase lockout per session, and a `MaxConcurrentImportSessions` cap (default 8; arch-review 05 Task 24 — startup-validated as `> 0` by `TransportOptionsValidator`, since a zero/negative cap would reject every import session forever) bounding the decrypted-content footprint since each open session pins a fully-decrypted bundle in memory.
|
||||
|
||||
**Preview→apply window (recorded decision).** The diff/preview an operator reviews is a point-in-time snapshot; there is **no optimistic concurrency** between preview and apply. If the target config changes in the window between `PreviewAsync` and `ApplyAsync` (another admin edits a template, say), the apply proceeds against the current state — the resolutions are keyed by `(EntityType, Name)`, so they still target the right rows, but the operator may not have seen the latest state in the diff. This is accepted for v1 (single-operator import is the expected workflow, and the terminal Confirm step already gates a typo-resistant environment-name entry). The bundle-format `version` fields are reserved for a future optimistic-concurrency check but are not yet enforced (Underdeveloped 5).
|
||||
|
||||
@@ -233,7 +233,11 @@ Bundle references that cannot be satisfied in either the bundle or the target DB
|
||||
|
||||
**Blocker-scan heuristic boundaries.** The scanner walks `TemplateScript.Code`, `TemplateAttribute.Value`, and `ApiMethod.Script` looking for top-level `Identifier(` or `Identifier.` tokens. To keep the heuristic usable on real script bodies it (a) skips identifiers preceded by `.` (member access — `obj.Method()` does not flag `Method`); (b) does NOT scan `TemplateAttribute.DataSourceReference` (an OPC UA address path, never script source); (c) excludes names **declared locally in the body** — every `LocalFunctionStatement` / `MethodDeclaration` identifier is collected via a Roslyn script-kind parse, so a script's own helper (`decimal ComputeRate(...) => …; return ComputeRate(x);`) is not mistaken for a missing SharedScript; and (d) filters out a `KnownNonReferenceNames` denylist of .NET stdlib types (`Convert`, `DateTimeOffset`, `Regex`, `StringBuilder`, `Parse`, LINQ operators, …), ScadaBridge runtime API roots (`Notify`, `Database`, `ExternalSystem`, `Scripts`, `Instance`, `Parameters`, `Attributes`, `Route`, …), and common SQL keywords that appear inside string literals (`COUNT`, `SELECT`, `FROM`, `HAVING`, `VALUES`, …). Both the diff-step `DetectBlockersAsync` and the Apply-time `RunSemanticValidationAsync` Pass 1 share this filter, so the diff preview and the Apply gate agree.
|
||||
|
||||
**Severity split — template-script findings are advisory, ApiMethod findings block.** A residual unresolved identifier (one the heuristic could not filter) is classified by its **origin**. Findings from **template scripts** and **attribute default expressions** are non-blocking **warnings**: they surface as `ConflictKind.Warning` rows in the preview and ride on `ImportResult.Warnings` after a successful apply — the import proceeds because the authoritative design-time **deploy gate re-validates** the script with a real compile (`ScriptCompiler.TryCompile`), so a heuristic false positive here must never stop a valid import. Findings from **ApiMethod scripts** remain **hard errors** (`ConflictKind.Blocker` in preview; `SemanticValidationException` at apply) because inbound API methods have no downstream design-time gate — the import review is their only pre-runtime check. The split is applied identically in `DetectBlockersAsync` and `RunSemanticValidationAsync` Pass 1 so preview and apply agree.
|
||||
**Severity split — template-script findings are advisory, ApiMethod findings block.** A residual unresolved identifier (one the heuristic could not filter) is classified by its **origin**. Findings from **template scripts** and **attribute default expressions** are non-blocking **warnings**: they surface as `ConflictKind.Warning` rows in the preview and ride on `ImportResult.Warnings` after a successful apply — the import proceeds because the authoritative design-time **deploy gate re-validates** the script with a real compile (`ScriptCompiler.TryCompile`), so a heuristic false positive here must never stop a valid import. Findings from **ApiMethod scripts** remain **hard errors** (`ConflictKind.Blocker` in preview; `SemanticValidationException` at apply) because inbound API methods have no downstream design-time gate — the import review is their only pre-runtime check. The split is applied identically in `DetectBlockersAsync` and `RunSemanticValidationAsync` Pass 1 so preview and apply agree. (This advisory-vs-hard split is a property of the name-resolution **heuristic** only — it is false-positive-prone; the **script trust gate** above has no false-positive channel, so its verdict is a hard error for every gated surface, template trigger expressions and instance alarm-override trigger expressions included.)
|
||||
|
||||
**Locked-member override warnings (advisory).** An instance override — attribute (`FlatteningService` drop), alarm, or native-alarm-source — that targets a **locked** template member is silently inert: the flattener ignores it. To stop that being invisible, a best-effort scan (`CollectLockedOverrideWarnings`) surfaces each such override as a `ConflictKind.Warning` preview row and on `ImportResult.Warnings` after apply, matching locked members by **direct name** (own + inherited placeholder rows both carry the `IsLocked` flag). It never blocks the import and never changes what is written — the override row is still persisted verbatim (bundle fidelity); the flattener remains the behavioural authority, and composed path-qualified members or derived-shadow (`LockedInDerived`) locks a placeholder row does not reflect emit no warning (never a false positive — the ManagementActor and flattener stay the enforcement points).
|
||||
|
||||
**Script-artifact change bus (post-commit).** After a successful apply the importer publishes one `ScriptArtifactsChanged` per script-bearing kind (ApiMethod / SharedScript / Template) for **every non-Skip resolution — Overwrite, Rename, AND Add** — because an artifact deleted on the target and re-imported as `Add` under the same name can still leave a stale compiled-handler/`_knownBadMethods` entry on a node; over-approximation is safe (the bus is advisory, at-least-once, and consumers self-heal). The **subscriber** side (Inbound API cache invalidation wiring) lives with the Inbound API component — see PLAN-R2-06.
|
||||
|
||||
**Step 5 — Confirm.** Final summary plus a "N instances will become stale" warning enumerating affected instances (a real count as of M8 — see "Stale-Instance Signaling"). User types the source environment name to confirm (typo-resistant gate at the prod boundary).
|
||||
|
||||
|
||||
+25
@@ -0,0 +1,25 @@
|
||||
using ZB.MOM.WW.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
|
||||
/// <summary>
|
||||
/// Validates <see cref="AuditLogPartitionMaintenanceOptions"/> at host startup
|
||||
/// (arch-review 08 round 2 NF4). A zero tick interval spins the maintenance
|
||||
/// hosted service; a zero look-ahead leaves <c>pf_AuditLog_Month</c> without a
|
||||
/// future monthly boundary, risking inserts into the unbounded tail partition.
|
||||
/// </summary>
|
||||
public sealed class AuditLogPartitionMaintenanceOptionsValidator
|
||||
: OptionsValidatorBase<AuditLogPartitionMaintenanceOptions>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Validate(ValidationBuilder builder, AuditLogPartitionMaintenanceOptions options)
|
||||
{
|
||||
builder.RequireThat(options.IntervalSeconds > 0,
|
||||
$"AuditLog:PartitionMaintenance:{nameof(AuditLogPartitionMaintenanceOptions.IntervalSeconds)} " +
|
||||
$"({options.IntervalSeconds}) must be > 0.");
|
||||
|
||||
builder.RequireThat(options.LookaheadMonths > 0,
|
||||
$"AuditLog:PartitionMaintenance:{nameof(AuditLogPartitionMaintenanceOptions.LookaheadMonths)} " +
|
||||
$"({options.LookaheadMonths}) must be > 0.");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
using ZB.MOM.WW.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
|
||||
/// <summary>
|
||||
/// Validates <see cref="AuditLogPurgeOptions"/> at host startup (arch-review 08
|
||||
/// round 2 NF4). The class self-clamps at resolve time, but a plainly-wrong
|
||||
/// config (zero cadence, zero batch, zero maintenance-command timeout) should
|
||||
/// still fail the boot so the operator learns immediately.
|
||||
/// <see cref="AuditLogPurgeOptions.IntervalOverride"/> is test-only (per the
|
||||
/// class remarks) and is deliberately not validated.
|
||||
/// </summary>
|
||||
public sealed class AuditLogPurgeOptionsValidator : OptionsValidatorBase<AuditLogPurgeOptions>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Validate(ValidationBuilder builder, AuditLogPurgeOptions options)
|
||||
{
|
||||
builder.RequireThat(options.IntervalHours > 0,
|
||||
$"AuditLog:Purge:{nameof(AuditLogPurgeOptions.IntervalHours)} " +
|
||||
$"({options.IntervalHours}) must be > 0.");
|
||||
|
||||
// Backing field for the documented "ChannelPurgeBatchSize" config key.
|
||||
builder.RequireThat(options.ChannelPurgeBatchSizeConfigured > 0,
|
||||
$"AuditLog:Purge:{nameof(AuditLogPurgeOptions.ChannelPurgeBatchSizeConfigured)} " +
|
||||
$"(config key ChannelPurgeBatchSize = {options.ChannelPurgeBatchSizeConfigured}) must be > 0.");
|
||||
|
||||
builder.RequireThat(options.MaintenanceCommandTimeoutMinutes > 0,
|
||||
$"AuditLog:Purge:{nameof(AuditLogPurgeOptions.MaintenanceCommandTimeoutMinutes)} " +
|
||||
$"({options.MaintenanceCommandTimeoutMinutes}) must be > 0.");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
using ZB.MOM.WW.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Central;
|
||||
|
||||
/// <summary>
|
||||
/// Validates <see cref="SiteAuditReconciliationOptions"/> at host startup
|
||||
/// (arch-review 08 round 2 NF4). A zero tick interval spins the reconciliation
|
||||
/// singleton, a zero batch pulls nothing per RPC, and a zero stalled-cycle
|
||||
/// threshold would fire the stalled signal on the first non-draining cycle.
|
||||
/// <see cref="SiteAuditReconciliationOptions.ReconciliationIntervalOverride"/>
|
||||
/// is test-only (per the class remarks) and is deliberately not validated.
|
||||
/// </summary>
|
||||
public sealed class SiteAuditReconciliationOptionsValidator
|
||||
: OptionsValidatorBase<SiteAuditReconciliationOptions>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Validate(ValidationBuilder builder, SiteAuditReconciliationOptions options)
|
||||
{
|
||||
builder.RequireThat(options.ReconciliationIntervalSeconds > 0,
|
||||
$"AuditLog:Reconciliation:{nameof(SiteAuditReconciliationOptions.ReconciliationIntervalSeconds)} " +
|
||||
$"({options.ReconciliationIntervalSeconds}) must be > 0.");
|
||||
|
||||
builder.RequireThat(options.BatchSize > 0,
|
||||
$"AuditLog:Reconciliation:{nameof(SiteAuditReconciliationOptions.BatchSize)} " +
|
||||
$"({options.BatchSize}) must be > 0.");
|
||||
|
||||
builder.RequireThat(options.StalledAfterNonDrainingCycles > 0,
|
||||
$"AuditLog:Reconciliation:{nameof(SiteAuditReconciliationOptions.StalledAfterNonDrainingCycles)} " +
|
||||
$"({options.StalledAfterNonDrainingCycles}) must be > 0.");
|
||||
}
|
||||
}
|
||||
@@ -92,20 +92,23 @@ public static class ServiceCollectionExtensions
|
||||
// failures as AuditRedactionFailure on the site health report.
|
||||
services.TryAddSingleton<IAuditRedactionFailureCounter, NoOpAuditRedactionFailureCounter>();
|
||||
|
||||
// Site writer + telemetry options bindings.
|
||||
// BindConfiguration is not used because the configuration root supplied
|
||||
// by the caller may not be the application root — we go through the
|
||||
// section explicitly so a partial IConfiguration (e.g. a test stub
|
||||
// anchored on the AuditLog section's parent) still works.
|
||||
services.AddOptions<SqliteAuditWriterOptions>()
|
||||
.Bind(config.GetSection(SiteWriterSectionName));
|
||||
services.AddOptions<SiteAuditTelemetryOptions>()
|
||||
.Bind(config.GetSection(SiteTelemetrySectionName));
|
||||
// Site writer + telemetry options bindings — now bind-and-validate via
|
||||
// the shared helper so a bad site config fails fast at boot (NF4). The
|
||||
// helper's GetSection(sectionPath) still resolves the section explicitly
|
||||
// (NOT BindConfiguration), preserving the partial-config-root rationale:
|
||||
// the caller's IConfiguration may be a test stub anchored on the AuditLog
|
||||
// section's parent rather than the application root.
|
||||
services.AddValidatedOptions<SqliteAuditWriterOptions, SqliteAuditWriterOptionsValidator>(
|
||||
config, SiteWriterSectionName);
|
||||
services.AddValidatedOptions<SiteAuditTelemetryOptions, SiteAuditTelemetryOptionsValidator>(
|
||||
config, SiteTelemetrySectionName);
|
||||
// Retention purge options (the SiteAuditRetentionService hosted job is
|
||||
// registered site-only in AddAuditLogHealthMetricsBridge). Binding here is
|
||||
// inert on central — no hosted service consumes it there.
|
||||
services.AddOptions<SiteAuditRetentionOptions>()
|
||||
.Bind(config.GetSection(SiteAuditRetentionOptions.SectionName));
|
||||
// registered site-only in AddAuditLogHealthMetricsBridge). The binding
|
||||
// here is inert on central — no hosted service consumes it there — but
|
||||
// the defaults validate clean, so eager validation on the central root is
|
||||
// harmless.
|
||||
services.AddValidatedOptions<SiteAuditRetentionOptions, SiteAuditRetentionOptionsValidator>(
|
||||
config, SiteAuditRetentionOptions.SectionName);
|
||||
|
||||
// SqliteAuditWriter is a singleton with a single owned SqliteConnection
|
||||
// and a background writer Task; multiple instances would race on the
|
||||
@@ -365,8 +368,8 @@ public static class ServiceCollectionExtensions
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(config);
|
||||
|
||||
services.AddOptions<AuditLogPartitionMaintenanceOptions>()
|
||||
.Bind(config.GetSection(PartitionMaintenanceSectionName));
|
||||
services.AddValidatedOptions<AuditLogPartitionMaintenanceOptions,
|
||||
AuditLogPartitionMaintenanceOptionsValidator>(config, PartitionMaintenanceSectionName);
|
||||
services.AddHostedService<AuditLogPartitionMaintenanceService>();
|
||||
|
||||
// I1 (review): bind the two central-singleton options HERE rather than in
|
||||
@@ -382,10 +385,10 @@ public static class ServiceCollectionExtensions
|
||||
// 5 min reconciliation tick); production exposes IntervalHours /
|
||||
// ReconciliationIntervalSeconds only — the test-only *Override knobs are
|
||||
// not intended to be set from config (see the options classes' remarks).
|
||||
services.AddOptions<AuditLogPurgeOptions>()
|
||||
.Bind(config.GetSection(PurgeSectionName));
|
||||
services.AddOptions<SiteAuditReconciliationOptions>()
|
||||
.Bind(config.GetSection(ReconciliationSectionName));
|
||||
services.AddValidatedOptions<AuditLogPurgeOptions, AuditLogPurgeOptionsValidator>(
|
||||
config, PurgeSectionName);
|
||||
services.AddValidatedOptions<SiteAuditReconciliationOptions, SiteAuditReconciliationOptionsValidator>(
|
||||
config, ReconciliationSectionName);
|
||||
|
||||
// Central health snapshot — a single object
|
||||
// that owns the CentralAuditWriteFailures + AuditRedactionFailure
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
using ZB.MOM.WW.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
|
||||
/// <summary>
|
||||
/// Validates <see cref="SiteAuditRetentionOptions"/> at host startup
|
||||
/// (arch-review 08 round 2 NF4). The option class self-clamps at resolve time,
|
||||
/// but a plainly-wrong config (zero retention window, zero/negative purge
|
||||
/// period, negative initial delay) should still fail the boot so the operator
|
||||
/// learns immediately rather than trusting a silent clamp.
|
||||
/// <see cref="SiteAuditRetentionOptions.PurgeIntervalOverride"/> is test-only
|
||||
/// (per the class remarks) and is deliberately not validated.
|
||||
/// </summary>
|
||||
public sealed class SiteAuditRetentionOptionsValidator : OptionsValidatorBase<SiteAuditRetentionOptions>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Validate(ValidationBuilder builder, SiteAuditRetentionOptions options)
|
||||
{
|
||||
builder.RequireThat(options.RetentionDays > 0,
|
||||
$"AuditLog:SiteRetention:{nameof(SiteAuditRetentionOptions.RetentionDays)} " +
|
||||
$"({options.RetentionDays}) must be > 0.");
|
||||
|
||||
builder.RequireThat(options.PurgeInterval > TimeSpan.Zero,
|
||||
$"AuditLog:SiteRetention:{nameof(SiteAuditRetentionOptions.PurgeInterval)} " +
|
||||
$"({options.PurgeInterval}) must be > 0.");
|
||||
|
||||
builder.RequireThat(options.InitialDelay >= TimeSpan.Zero,
|
||||
$"AuditLog:SiteRetention:{nameof(SiteAuditRetentionOptions.InitialDelay)} " +
|
||||
$"({options.InitialDelay}) must be >= 0.");
|
||||
}
|
||||
}
|
||||
@@ -61,32 +61,42 @@ public sealed class SiteAuditRetentionService : IHostedService, IDisposable
|
||||
|
||||
private async Task RunLoopAsync(CancellationToken ct)
|
||||
{
|
||||
// InitialDelay before the first purge — short so a daily-recycled node
|
||||
// still purges soon after start rather than waiting a full interval it may
|
||||
// never reach.
|
||||
try
|
||||
{
|
||||
await Task.Delay(_options.InitialDelay, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await SafePurgeAsync(ct).ConfigureAwait(false);
|
||||
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
// InitialDelay before the first purge — short so a daily-recycled node
|
||||
// still purges soon after start rather than waiting a full interval it may
|
||||
// never reach.
|
||||
try
|
||||
{
|
||||
await Task.Delay(_options.ResolvedPurgeInterval, ct).ConfigureAwait(false);
|
||||
await Task.Delay(_options.InitialDelay, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
return;
|
||||
}
|
||||
|
||||
await SafePurgeAsync(ct).ConfigureAwait(false);
|
||||
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
await Task.Delay(_options.ResolvedPurgeInterval, ct).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
await SafePurgeAsync(ct).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Shutdown landed mid-purge: SafePurgeAsync rethrows OCE by design so the purge
|
||||
// aborts promptly, but the loop task must complete CLEANLY — StopAsync hands
|
||||
// _loop straight to the host, and a canceled task there is shutdown-log noise
|
||||
// (arch-review 04 round 2, R7). Cancellation here IS the clean exit.
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
using ZB.MOM.WW.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site;
|
||||
|
||||
/// <summary>
|
||||
/// Validates <see cref="SqliteAuditWriterOptions"/> at host startup
|
||||
/// (arch-review 08 round 2 NF4). The channel/batch/flush knobs feed the
|
||||
/// background writer task; a zero anywhere stalls it (nothing drains, nothing
|
||||
/// flushes), and an empty <c>DatabasePath</c> leaves the SQLite writer with no
|
||||
/// backing store. <see cref="SqliteAuditWriterOptions.BacklogPollIntervalSeconds"/>
|
||||
/// is intentionally NOT validated — a non-positive value has a documented
|
||||
/// fall-back-to-30s contract in <c>SiteAuditBacklogReporter</c>.
|
||||
/// </summary>
|
||||
public sealed class SqliteAuditWriterOptionsValidator : OptionsValidatorBase<SqliteAuditWriterOptions>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Validate(ValidationBuilder builder, SqliteAuditWriterOptions options)
|
||||
{
|
||||
builder.RequireThat(!string.IsNullOrWhiteSpace(options.DatabasePath),
|
||||
$"AuditLog:SiteWriter:{nameof(SqliteAuditWriterOptions.DatabasePath)} must be a non-empty path.");
|
||||
|
||||
builder.RequireThat(options.ChannelCapacity > 0,
|
||||
$"AuditLog:SiteWriter:{nameof(SqliteAuditWriterOptions.ChannelCapacity)} " +
|
||||
$"({options.ChannelCapacity}) must be > 0.");
|
||||
|
||||
builder.RequireThat(options.BatchSize > 0,
|
||||
$"AuditLog:SiteWriter:{nameof(SqliteAuditWriterOptions.BatchSize)} " +
|
||||
$"({options.BatchSize}) must be > 0.");
|
||||
|
||||
builder.RequireThat(options.FlushIntervalMs > 0,
|
||||
$"AuditLog:SiteWriter:{nameof(SqliteAuditWriterOptions.FlushIntervalMs)} " +
|
||||
$"({options.FlushIntervalMs}) must be > 0.");
|
||||
|
||||
// BacklogPollIntervalSeconds is deliberately unchecked: a non-positive
|
||||
// value falls back to the reporter's 30s default (see the option's
|
||||
// remarks + register row 21 resolution).
|
||||
}
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
using ZB.MOM.WW.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry;
|
||||
|
||||
/// <summary>
|
||||
/// Validates <see cref="SiteAuditTelemetryOptions"/> at host startup
|
||||
/// (arch-review 08 round 2 NF4). <c>SiteAuditTelemetryActor</c> uses the two
|
||||
/// interval knobs as raw scheduling delays (busy after a productive/faulted
|
||||
/// drain, idle after an empty one), so a zero interval spins the drain loop and
|
||||
/// a zero batch drains nothing. Idle must be >= busy: a shorter idle interval
|
||||
/// inverts the actor's back-off-when-empty intent.
|
||||
/// </summary>
|
||||
public sealed class SiteAuditTelemetryOptionsValidator : OptionsValidatorBase<SiteAuditTelemetryOptions>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Validate(ValidationBuilder builder, SiteAuditTelemetryOptions options)
|
||||
{
|
||||
builder.RequireThat(options.BatchSize > 0,
|
||||
$"AuditLog:SiteTelemetry:{nameof(SiteAuditTelemetryOptions.BatchSize)} " +
|
||||
$"({options.BatchSize}) must be > 0.");
|
||||
|
||||
builder.RequireThat(options.BusyIntervalSeconds > 0,
|
||||
$"AuditLog:SiteTelemetry:{nameof(SiteAuditTelemetryOptions.BusyIntervalSeconds)} " +
|
||||
$"({options.BusyIntervalSeconds}) must be > 0.");
|
||||
|
||||
builder.RequireThat(options.IdleIntervalSeconds > 0,
|
||||
$"AuditLog:SiteTelemetry:{nameof(SiteAuditTelemetryOptions.IdleIntervalSeconds)} " +
|
||||
$"({options.IdleIntervalSeconds}) must be > 0.");
|
||||
|
||||
builder.RequireThat(options.IdleIntervalSeconds >= options.BusyIntervalSeconds,
|
||||
$"AuditLog:SiteTelemetry:{nameof(SiteAuditTelemetryOptions.IdleIntervalSeconds)} " +
|
||||
$"({options.IdleIntervalSeconds}) must be >= " +
|
||||
$"{nameof(SiteAuditTelemetryOptions.BusyIntervalSeconds)} ({options.BusyIntervalSeconds}) " +
|
||||
"— the idle drain must back off at least as far as the busy drain.");
|
||||
}
|
||||
}
|
||||
@@ -291,9 +291,27 @@
|
||||
try
|
||||
{
|
||||
var result = await AlarmSummaryService.GetSiteAlarmsAsync(siteId);
|
||||
_rows = result.Alarms;
|
||||
// Stale-site guard (arch-review R2 N4): the operator may have switched sites
|
||||
// while this fan-out was in flight — drop the result rather than labeling
|
||||
// site A's alarms under site B's picker. Mirrors OnLiveAlarmsChanged (:374).
|
||||
if (_selectedSiteId != siteId)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// _notReporting is the poll's unique authority — the alarm-only live cache
|
||||
// cannot compute it — so it is always refreshed.
|
||||
_notReporting = result.NotReportingInstances;
|
||||
_rollup = AlarmSummaryService.ComputeRollup(_rows);
|
||||
|
||||
// While the cache is live, the live deltas own the row set: a poll whose fan-out
|
||||
// started BEFORE a delta must not land after it and momentarily revert the alarm
|
||||
// state (arch-review R2 N5). When not live (pre-seed / degraded stream / dead
|
||||
// aggregator — see R2 N6), the poll remains the full-rebuild safety net.
|
||||
if (!LiveAlarmCache.IsLive(siteId))
|
||||
{
|
||||
_rows = result.Alarms;
|
||||
_rollup = AlarmSummaryService.ComputeRollup(_rows);
|
||||
}
|
||||
RecomputeVisibleRows();
|
||||
}
|
||||
catch
|
||||
@@ -341,10 +359,11 @@
|
||||
// aggregated alarm set changes. We rebuild _rows/_rollup/_visibleRows from the
|
||||
// immutable live snapshot — but deliberately DO NOT touch _notReporting, since
|
||||
// the alarm-only live cache can't compute it.
|
||||
// • The 15s poll (RefreshAsync) still runs untouched: it is the authority for
|
||||
// _notReporting and the safety net when the cache is not live (pre-seed or a
|
||||
// degraded/failed stream — IsLive == false). When live, the poll simply
|
||||
// re-affirms the same snapshot; the live path just makes updates arrive sooner.
|
||||
// • The 15s poll (RefreshAsync) is the authority for _notReporting and the
|
||||
// full-rebuild safety net ONLY while the cache is not live (pre-seed or a
|
||||
// degraded/failed stream — IsLive == false); when live it deliberately leaves
|
||||
// _rows to the delta path, so a slow fan-out can never revert a fresher live
|
||||
// delta (arch-review R2 N5).
|
||||
// • Both paths mutate shared state only via the Blazor dispatcher (the poll via
|
||||
// its InvokeAsync callback, the live delta via OnLiveChanged's InvokeAsync), so
|
||||
// they are serialized and never race. Each rebuild is an idempotent snapshot, so
|
||||
@@ -363,15 +382,20 @@
|
||||
_liveSubscription = null;
|
||||
}
|
||||
|
||||
// N7: set BEFORE teardown so a live callback racing Dispose is dropped both
|
||||
// before the InvokeAsync marshal and inside it (mirrors DebugView.razor).
|
||||
private volatile bool _disposed;
|
||||
|
||||
// Raised on the aggregator's thread — marshal onto the circuit before touching state.
|
||||
private void OnLiveAlarmsChanged(int siteId)
|
||||
{
|
||||
if (_disposed) return;
|
||||
_ = InvokeAsync(() =>
|
||||
{
|
||||
// Drop stale callbacks for a site we've since navigated away from, and
|
||||
// let the poll drive until the aggregator has actually seeded (so we never
|
||||
// clobber a good poll snapshot with an empty pre-seed list).
|
||||
if (_selectedSiteId != siteId || !LiveAlarmCache.IsLive(siteId))
|
||||
if (_disposed || _selectedSiteId != siteId || !LiveAlarmCache.IsLive(siteId))
|
||||
{
|
||||
return;
|
||||
}
|
||||
@@ -506,6 +530,7 @@
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_disposed = true;
|
||||
StopTimer();
|
||||
DisposeLiveSubscription();
|
||||
}
|
||||
|
||||
@@ -51,6 +51,14 @@ public partial class KpiTrendChart
|
||||
/// The series to plot, assumed ordered ascending by
|
||||
/// <see cref="KpiSeriesPoint.BucketStartUtc"/>. <c>null</c> or empty renders
|
||||
/// the unavailable placeholder rather than an empty chart.
|
||||
/// <para>
|
||||
/// The per-point value's meaning depends on the metric's
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi.KpiMetricAggregationCatalog"/>
|
||||
/// classification: a Rate-classified series carries <b>per-bucket totals</b> (the
|
||||
/// bucketer sums each bucket's deltas), while a Gauge series carries
|
||||
/// last-value-per-bucket readings. The chart is agnostic to which — it plots the
|
||||
/// supplied values verbatim (arch-review 04 round 2, R3).
|
||||
/// </para>
|
||||
/// </summary>
|
||||
[Parameter] public IReadOnlyList<KpiSeriesPoint>? Points { get; set; }
|
||||
|
||||
@@ -58,7 +66,10 @@ public partial class KpiTrendChart
|
||||
/// <c>data-test</c> slug. Required-ish; defaults to "Trend" if blank.</summary>
|
||||
[Parameter] public string Title { get; set; } = "Trend";
|
||||
|
||||
/// <summary>Optional unit suffix appended to value labels (e.g. "s").</summary>
|
||||
/// <summary>Optional unit suffix appended to value labels (e.g. "s"). For a
|
||||
/// Rate-classified metric the supplied text must state the per-bucket-total
|
||||
/// semantics (e.g. "delivered/bucket"), not a per-minute/per-interval reading,
|
||||
/// so the label matches the summed values in <see cref="Points"/> (R3).</summary>
|
||||
[Parameter] public string? Unit { get; set; }
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -76,12 +76,18 @@ public sealed class KpiHistoryQueryService : IKpiHistoryQueryService
|
||||
{
|
||||
var effectiveMax = maxPoints ?? _options.DefaultMaxSeriesPoints;
|
||||
|
||||
// Per-metric reduction intent: the same catalog that drives the hourly fold drives
|
||||
// the chart-time bucket reduction, so a Rate series is summed per bucket on BOTH the
|
||||
// raw-minute and hourly-rollup routes — one unit ("events per chart bucket") on both
|
||||
// sides of the RollupThresholdHours routing boundary (arch-review 04 round 2, R3).
|
||||
var aggregation = KpiMetricAggregationCatalog.Resolve(source, metric);
|
||||
|
||||
// Test-seam ctor: use the injected repository directly.
|
||||
if (_injectedRepository is not null)
|
||||
{
|
||||
var injectedSeries = await FetchSeriesAsync(
|
||||
_injectedRepository, source, metric, scope, scopeKey, fromUtc, toUtc, cancellationToken);
|
||||
return KpiSeriesBucketer.Bucket(injectedSeries, fromUtc, toUtc, effectiveMax);
|
||||
return KpiSeriesBucketer.Bucket(injectedSeries, fromUtc, toUtc, effectiveMax, aggregation);
|
||||
}
|
||||
|
||||
// Production: a fresh scope (and thus a fresh DbContext) per query so a
|
||||
@@ -90,7 +96,7 @@ public sealed class KpiHistoryQueryService : IKpiHistoryQueryService
|
||||
var repository = serviceScope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();
|
||||
var series = await FetchSeriesAsync(
|
||||
repository, source, metric, scope, scopeKey, fromUtc, toUtc, cancellationToken);
|
||||
return KpiSeriesBucketer.Bucket(series, fromUtc, toUtc, effectiveMax);
|
||||
return KpiSeriesBucketer.Bucket(series, fromUtc, toUtc, effectiveMax, aggregation);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -103,6 +109,13 @@ public sealed class KpiHistoryQueryService : IKpiHistoryQueryService
|
||||
/// the caller's <see cref="KpiSeriesBucketer.Bucket"/> step is agnostic to the source
|
||||
/// (a 90 d rollup can still exceed the point ceiling, so bucketing still applies).
|
||||
/// The boundary is strict: exactly <c>RollupThresholdHours</c> stays on the raw path.
|
||||
/// <para>
|
||||
/// The routing boundary no longer changes a Rate chart's magnitude: both paths reduce to
|
||||
/// per-bucket sums via the catalog-driven aggregation applied in
|
||||
/// <see cref="GetSeriesAsync"/>. A residual ≤~1.2× native-resolution difference exists only
|
||||
/// for an unbucketed just-over-threshold rollup series (plotted at native per-hour sums) vs.
|
||||
/// an at-threshold bucketed raw series — documented, not hidden (arch-review 04 round 2, R3).
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private Task<IReadOnlyList<KpiSeriesPoint>> FetchSeriesAsync(
|
||||
IKpiHistoryRepository repository,
|
||||
|
||||
@@ -109,6 +109,16 @@ public interface IKpiHistoryRepository
|
||||
string source, string metric, string scope, string? scopeKey,
|
||||
DateTime fromUtc, DateTime toUtc, CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Returns the newest <c>KpiRollupHourly.HourStartUtc</c> across all series, or
|
||||
/// <c>null</c> when no rollups exist. The recorder's one-shot backfill consults
|
||||
/// this watermark to skip (or shrink to) the un-rolled tail instead of re-folding
|
||||
/// the whole raw-retention window on every failover (arch-review 04 round 2, R1).
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A task resolving to the newest folded hour start, or <c>null</c> when empty.</returns>
|
||||
Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Bulk-deletes hourly rollup rows whose
|
||||
/// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Entities.Kpi.KpiRollupHourly.HourStartUtc"/> is
|
||||
|
||||
+16
-1
@@ -46,6 +46,11 @@ public interface ISecuredWriteRepository
|
||||
/// <param name="siteId">Site id filter; <c>null</c> matches every site.</param>
|
||||
/// <param name="skip">Number of rows to skip (offset paging).</param>
|
||||
/// <param name="take">Maximum number of rows to return.</param>
|
||||
/// <param name="permittedSiteIds">When non-null, only rows whose <c>SiteId</c> (site
|
||||
/// identifier) is in the set match; <c>null</c> matches every site. Applied IN ADDITION
|
||||
/// to <paramref name="siteId"/>. Used by the ManagementActor to constrain an unfiltered
|
||||
/// list to a site-scoped caller's permitted sites at the query level (arch-review R2 N3),
|
||||
/// so paging and totals stay correct.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that resolves to a page of matching rows, newest submission first.</returns>
|
||||
Task<IReadOnlyList<PendingSecuredWrite>> QueryAsync(
|
||||
@@ -53,6 +58,7 @@ public interface ISecuredWriteRepository
|
||||
string? siteId,
|
||||
int skip,
|
||||
int take,
|
||||
IReadOnlyCollection<string>? permittedSiteIds = null,
|
||||
CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
@@ -106,7 +112,16 @@ public interface ISecuredWriteRepository
|
||||
/// </summary>
|
||||
/// <param name="status">Status filter; <c>null</c> matches every status.</param>
|
||||
/// <param name="siteId">Site id filter; <c>null</c> matches every site.</param>
|
||||
/// <param name="permittedSiteIds">When non-null, only rows whose <c>SiteId</c> (site
|
||||
/// identifier) is in the set match; <c>null</c> matches every site. Applied IN ADDITION
|
||||
/// to <paramref name="siteId"/>. Used by the ManagementActor to constrain an unfiltered
|
||||
/// list to a site-scoped caller's permitted sites at the query level (arch-review R2 N3),
|
||||
/// so paging and totals stay correct.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>A task that resolves to the count of matching rows.</returns>
|
||||
Task<int> CountAsync(string? status, string? siteId, CancellationToken ct = default);
|
||||
Task<int> CountAsync(
|
||||
string? status,
|
||||
string? siteId,
|
||||
IReadOnlyCollection<string>? permittedSiteIds = null,
|
||||
CancellationToken ct = default);
|
||||
}
|
||||
|
||||
@@ -47,6 +47,14 @@ public static class ScadaBridgeTelemetry
|
||||
Meter.CreateCounter<long>("scadabridge.store_and_forward.replication.failures", unit: "1",
|
||||
description: "S&F buffer replication operations that failed to dispatch/deliver to the peer node");
|
||||
|
||||
private static readonly Counter<long> _sfResyncCompleted =
|
||||
Meter.CreateCounter<long>("scadabridge.store_and_forward.resync.completed", unit: "1",
|
||||
description: "S&F anti-entropy resyncs the standby acknowledged as applied");
|
||||
|
||||
private static readonly Counter<long> _sfResyncAckMissing =
|
||||
Meter.CreateCounter<long>("scadabridge.store_and_forward.resync.ack_missing", unit: "1",
|
||||
description: "S&F resyncs answered by the active node but never acknowledged by the standby within the ack window (lost chunks / dead peer)");
|
||||
|
||||
/// <summary>
|
||||
/// Incremented each time a per-site live-alarm aggregator re-establishes its site-wide
|
||||
/// gRPC stream — a NodeA↔NodeB failover flip or a reconcile-driven reopen after the
|
||||
@@ -107,6 +115,12 @@ public static class ScadaBridgeTelemetry
|
||||
/// <summary>Records one failed S&F replication dispatch.</summary>
|
||||
public static void RecordReplicationFailure() => _replicationFailures.Add(1);
|
||||
|
||||
/// <summary>Records one acknowledged (applied) S&F buffer resync.</summary>
|
||||
public static void RecordSfResyncCompleted() => _sfResyncCompleted.Add(1);
|
||||
|
||||
/// <summary>Records one resync whose ack window expired.</summary>
|
||||
public static void RecordSfResyncAckMissing() => _sfResyncAckMissing.Add(1);
|
||||
|
||||
/// <summary>Records that a site connection opened (increments the up-count gauge).</summary>
|
||||
public static void SiteConnectionOpened() => Interlocked.Increment(ref _siteConnectionsUp);
|
||||
|
||||
|
||||
@@ -42,10 +42,10 @@ public enum KpiRollupAggregation
|
||||
/// Keying by <c>(source, metric)</c> — not metric alone — is deliberate:
|
||||
/// <c>deliveredLastInterval</c> is emitted by both
|
||||
/// <see cref="KpiSources.NotificationOutbox"/> and <see cref="KpiSources.SiteCallAudit"/>,
|
||||
/// so a metric name is not globally unique. Every metric string is taken from the
|
||||
/// emitting source verbatim (shared <see cref="KpiMetrics"/> constants where they exist,
|
||||
/// otherwise the source's own private literal), so this catalog stays a faithful mirror
|
||||
/// of what actually lands in <c>KpiSample.Metric</c>.
|
||||
/// so a metric name is not globally unique. Every catalogued metric string is now a shared
|
||||
/// <see cref="KpiMetrics"/> constant — the emitting source consumes the same constant — so an
|
||||
/// emitter rename is a <strong>compile error</strong>, not a silent Gauge downgrade (R4), and
|
||||
/// this catalog stays a faithful mirror of what actually lands in <c>KpiSample.Metric</c>.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The catalog stores only the <see cref="KpiRollupAggregation.Rate"/> pairs; anything
|
||||
@@ -57,12 +57,6 @@ public enum KpiRollupAggregation
|
||||
/// </remarks>
|
||||
public static class KpiMetricAggregationCatalog
|
||||
{
|
||||
// Metric literals for the sources that keep their names private (no shared
|
||||
// KpiMetrics constant exists for these). Charted metrics use the KpiMetrics catalog.
|
||||
private const string SiteCallAuditDeliveredLastInterval = "deliveredLastInterval";
|
||||
private const string SiteHealthAlarmEvalErrors = "alarmEvalErrors";
|
||||
private const string SiteHealthEventLogWriteFailures = "eventLogWriteFailures";
|
||||
|
||||
/// <summary>
|
||||
/// The <c>(source, metric)</c> pairs classified as <see cref="KpiRollupAggregation.Rate"/>
|
||||
/// (sum-per-hour). Every other emitted metric is a <see cref="KpiRollupAggregation.Gauge"/>.
|
||||
@@ -99,11 +93,11 @@ public static class KpiMetricAggregationCatalog
|
||||
{
|
||||
(KpiSources.NotificationOutbox, KpiMetrics.NotificationOutbox.DeliveredLastInterval),
|
||||
(KpiSources.SiteCallAudit, KpiMetrics.SiteCallAudit.FailedLastInterval),
|
||||
(KpiSources.SiteCallAudit, SiteCallAuditDeliveredLastInterval),
|
||||
(KpiSources.SiteCallAudit, KpiMetrics.SiteCallAudit.DeliveredLastInterval),
|
||||
(KpiSources.SiteHealth, KpiMetrics.SiteHealth.ScriptErrors),
|
||||
(KpiSources.SiteHealth, SiteHealthAlarmEvalErrors),
|
||||
(KpiSources.SiteHealth, KpiMetrics.SiteHealth.AlarmEvalErrors),
|
||||
(KpiSources.SiteHealth, KpiMetrics.SiteHealth.DeadLetters),
|
||||
(KpiSources.SiteHealth, SiteHealthEventLogWriteFailures),
|
||||
(KpiSources.SiteHealth, KpiMetrics.SiteHealth.EventLogWriteFailures),
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -18,11 +18,16 @@ namespace ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi;
|
||||
/// <strong>not</strong> a rename — changing any value would orphan historical samples.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Only the metrics a trend page actually charts are catalogued here. Sources may emit
|
||||
/// additional internal metrics (e.g. the Notification Outbox <c>stuckCount</c> /
|
||||
/// <c>oldestPendingAgeSeconds</c>, the Site Call Audit <c>deliveredLastInterval</c> /
|
||||
/// <c>stuck</c> / <c>oldestPendingAgeSeconds</c>, and the bulk of the Site Health catalog)
|
||||
/// that no page references; those stay private to their source until a page charts them.
|
||||
/// This catalog now carries two kinds of metric: those a trend page charts, <em>and</em>
|
||||
/// those the <see cref="KpiMetricAggregationCatalog"/> classifies for the hourly rollup fold
|
||||
/// (e.g. <c>SiteCallAudit/deliveredLastInterval</c>, <c>SiteHealth/alarmEvalErrors</c>,
|
||||
/// <c>SiteHealth/eventLogWriteFailures</c>) even if no page charts them yet. The C4/R4 lesson
|
||||
/// is that <strong>any metric named in two places needs one shared symbol</strong>: a private
|
||||
/// literal on the emitter plus a matching private literal in the catalog drifts silently on a
|
||||
/// rename. Sources may still keep genuinely single-use internal metrics (e.g. the Notification
|
||||
/// Outbox <c>stuckCount</c> / <c>oldestPendingAgeSeconds</c>, the Site Call Audit <c>stuck</c> /
|
||||
/// <c>oldestPendingAgeSeconds</c>, and the bulk of the Site Health catalog) private until a page
|
||||
/// or the catalog references them.
|
||||
/// Constants are grouped by source via nested static classes mirroring
|
||||
/// <see cref="KpiSources"/>.
|
||||
/// </para>
|
||||
@@ -65,6 +70,11 @@ public static class KpiMetrics
|
||||
|
||||
/// <summary>Cached calls that failed permanently in the last KPI interval.</summary>
|
||||
public const string FailedLastInterval = "failedLastInterval";
|
||||
|
||||
/// <summary>Cached calls delivered in the last KPI interval. Not charted by a trend
|
||||
/// page today, but named in the rollup aggregation catalog as a Rate metric — so it
|
||||
/// is a shared symbol to keep the emitter and catalog in lock-step (R4).</summary>
|
||||
public const string DeliveredLastInterval = "deliveredLastInterval";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -98,6 +108,16 @@ public static class KpiMetrics
|
||||
/// <summary>Script-execution error count reported by the site.</summary>
|
||||
public const string ScriptErrors = "scriptErrors";
|
||||
|
||||
/// <summary>Alarm-evaluation error count reported by the site. Named in the rollup
|
||||
/// aggregation catalog as a Rate metric — shared symbol so an emitter rename is a
|
||||
/// compile error, not a silent Gauge downgrade (R4).</summary>
|
||||
public const string AlarmEvalErrors = "alarmEvalErrors";
|
||||
|
||||
/// <summary>Event-log write-failure count reported by the site. Named in the rollup
|
||||
/// aggregation catalog as a Rate metric — shared symbol so an emitter rename is a
|
||||
/// compile error, not a silent Gauge downgrade (R4).</summary>
|
||||
public const string EventLogWriteFailures = "eventLogWriteFailures";
|
||||
|
||||
/// <summary>Summed store-and-forward buffer depth across all buffers.</summary>
|
||||
public const string SfBufferDepth = "sfBufferDepth";
|
||||
}
|
||||
|
||||
@@ -3,8 +3,14 @@ namespace ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi;
|
||||
/// <summary>
|
||||
/// Pure, deterministic downsampling helper for KPI series charting ("KPI History & Trends").
|
||||
/// Reduces a raw <see cref="KpiSeriesPoint"/> series to at most <c>maxPoints</c> points using
|
||||
/// last-value-per-bucket / gauge semantics — suitable for step/area charts where the most
|
||||
/// recent value in a window best represents that window.
|
||||
/// <em>per-metric</em> bucket reduction driven by <see cref="KpiRollupAggregation"/>:
|
||||
/// <see cref="KpiRollupAggregation.Gauge"/> (the default) keeps the <strong>last value</strong>
|
||||
/// in each bucket — suitable for step/area charts where the most recent reading best represents
|
||||
/// the window; <see cref="KpiRollupAggregation.Rate"/> <strong>sums per bucket</strong> — each
|
||||
/// raw point of a Rate series is a per-interval delta, so the bucket's true total is the sum of
|
||||
/// its deltas. Summing on both the raw-minute and hourly-rollup paths charts the same unit
|
||||
/// ("events per chart bucket") on both sides of the raw/rollup routing boundary, eliminating the
|
||||
/// ~60× magnitude jump at that boundary (arch-review 04 round 2, R3).
|
||||
/// </summary>
|
||||
public static class KpiSeriesBucketer
|
||||
{
|
||||
@@ -27,12 +33,20 @@ public static class KpiSeriesBucketer
|
||||
/// <param name="fromUtc">UTC start of the query window (inclusive).</param>
|
||||
/// <param name="toUtc">UTC end of the query window (inclusive on the right edge).</param>
|
||||
/// <param name="maxPoints">Maximum number of output points. Must be ≥ 2.</param>
|
||||
/// <param name="aggregation">
|
||||
/// How each bucket is reduced. <see cref="KpiRollupAggregation.Gauge"/> (default) keeps the
|
||||
/// last value in the bucket — the historical behavior. <see cref="KpiRollupAggregation.Rate"/>
|
||||
/// sums the values in the bucket — each raw point of a Rate series is a per-interval delta, so
|
||||
/// the bucket total is the sum of its deltas (arch-review 04 round 2, R3).
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// An <see cref="IReadOnlyList{T}"/> of at most <paramref name="maxPoints"/> bucketed points,
|
||||
/// ordered by <see cref="KpiSeriesPoint.BucketStartUtc"/> ascending.
|
||||
/// Returns <paramref name="raw"/> unchanged (same reference) when
|
||||
/// <c>raw.Count <= maxPoints</c>; callers must not mutate the underlying
|
||||
/// collection in that case, as it is the same object passed in.
|
||||
/// For <see cref="KpiRollupAggregation.Gauge"/> returns <paramref name="raw"/> unchanged
|
||||
/// (same reference) when <c>raw.Count <= maxPoints</c>; callers must not mutate the
|
||||
/// underlying collection in that case. <see cref="KpiRollupAggregation.Rate"/> always runs
|
||||
/// the bucketing pass — two deltas landing in one bucket must sum even in a short series —
|
||||
/// so it never returns the input reference.
|
||||
/// </returns>
|
||||
/// <exception cref="ArgumentOutOfRangeException">
|
||||
/// Thrown when <paramref name="maxPoints"/> < 2 or
|
||||
@@ -44,7 +58,8 @@ public static class KpiSeriesBucketer
|
||||
IReadOnlyList<KpiSeriesPoint> raw,
|
||||
DateTime fromUtc,
|
||||
DateTime toUtc,
|
||||
int maxPoints)
|
||||
int maxPoints,
|
||||
KpiRollupAggregation aggregation = KpiRollupAggregation.Gauge)
|
||||
{
|
||||
if (maxPoints < 2)
|
||||
throw new ArgumentOutOfRangeException(nameof(maxPoints),
|
||||
@@ -54,11 +69,15 @@ public static class KpiSeriesBucketer
|
||||
throw new ArgumentOutOfRangeException(nameof(toUtc),
|
||||
toUtc, "toUtc must be strictly greater than fromUtc.");
|
||||
|
||||
// Normal runtime case — empty or short series: return as-is.
|
||||
// Normal runtime case — empty series: return as-is.
|
||||
if (raw is null || raw.Count == 0)
|
||||
return Array.Empty<KpiSeriesPoint>();
|
||||
|
||||
if (raw.Count <= maxPoints)
|
||||
// Short-series early return is Gauge-only: for Rate, two deltas landing in one bucket
|
||||
// must still sum, so a short Rate series cannot be returned verbatim (arch-review 04
|
||||
// round 2, R3). For well-spaced short Rate series the per-point buckets make the sum an
|
||||
// identity, so the output still equals the input anyway.
|
||||
if (aggregation == KpiRollupAggregation.Gauge && raw.Count <= maxPoints)
|
||||
return raw;
|
||||
|
||||
// Divide the window into maxPoints equal-width buckets.
|
||||
@@ -67,12 +86,11 @@ public static class KpiSeriesBucketer
|
||||
double windowTicks = (double)(toUtc.Ticks - fromUtc.Ticks);
|
||||
double bucketWidthTicks = windowTicks / maxPoints;
|
||||
|
||||
// For each bucket, track the candidate point: the one with the
|
||||
// maximum BucketStartUtc (last value within the bucket).
|
||||
// We use a fixed-size array indexed by bucket number.
|
||||
// Nullable KpiSeriesPoint[] with 'hasValue' flags is fine since the
|
||||
// struct is small.
|
||||
// For each bucket, track the candidate point: for Gauge the one with the maximum
|
||||
// BucketStartUtc (last value within the bucket); for Rate a running sum of the bucket's
|
||||
// deltas. Fixed-size arrays indexed by bucket number.
|
||||
var best = new KpiSeriesPoint[maxPoints];
|
||||
var sums = new double[maxPoints];
|
||||
var occupied = new bool[maxPoints];
|
||||
|
||||
foreach (var point in raw)
|
||||
@@ -89,13 +107,19 @@ public static class KpiSeriesBucketer
|
||||
if (bucketIndex >= maxPoints)
|
||||
bucketIndex = maxPoints - 1;
|
||||
|
||||
if (aggregation == KpiRollupAggregation.Rate)
|
||||
{
|
||||
// Rate: accumulate the bucket's per-interval deltas.
|
||||
sums[bucketIndex] += point.Value;
|
||||
occupied[bucketIndex] = true;
|
||||
}
|
||||
// Keep the last point in iteration order within this bucket. Because the stored
|
||||
// candidate's BucketStartUtc is the bucket-START timestamp (not the raw point's
|
||||
// capture time), the comparison below is true for essentially any in-bucket point,
|
||||
// so each later-in-iteration point overwrites the previous one. For the ascending-
|
||||
// sorted input this method requires, last-in-iteration IS the largest-timestamp
|
||||
// point — i.e. last-value-per-bucket semantics.
|
||||
if (!occupied[bucketIndex] ||
|
||||
else if (!occupied[bucketIndex] ||
|
||||
point.BucketStartUtc > best[bucketIndex].BucketStartUtc)
|
||||
{
|
||||
best[bucketIndex] = new KpiSeriesPoint(
|
||||
@@ -109,8 +133,13 @@ public static class KpiSeriesBucketer
|
||||
var result = new List<KpiSeriesPoint>(maxPoints);
|
||||
for (int i = 0; i < maxPoints; i++)
|
||||
{
|
||||
if (occupied[i])
|
||||
result.Add(best[i]);
|
||||
if (!occupied[i])
|
||||
continue;
|
||||
|
||||
result.Add(aggregation == KpiRollupAggregation.Rate
|
||||
? new KpiSeriesPoint(
|
||||
fromUtc + TimeSpan.FromTicks((long)(i * bucketWidthTicks)), sums[i])
|
||||
: best[i]);
|
||||
}
|
||||
|
||||
return result;
|
||||
|
||||
@@ -52,21 +52,26 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
private readonly string _grpcNodeAAddress;
|
||||
private readonly string _grpcNodeBAddress;
|
||||
private readonly TimeSpan _reconcileInterval;
|
||||
private readonly TimeSpan _publishCoalesce;
|
||||
|
||||
private const int MaxRetries = 3;
|
||||
private const string ReconnectTimerKey = "alarm-grpc-reconnect";
|
||||
private const string StabilityTimerKey = "alarm-grpc-stability";
|
||||
private const string ReconcileTimerKey = "alarm-reconcile";
|
||||
private const string PublishTimerKey = "alarm-publish-coalesce";
|
||||
|
||||
/// <summary>Delay between gRPC reconnection attempts. Settable for tests.</summary>
|
||||
internal static TimeSpan ReconnectDelay { get; set; } = TimeSpan.FromSeconds(5);
|
||||
/// <summary>True while a coalesced publish is armed (dirty deltas awaiting one tick). Actor-thread only.</summary>
|
||||
private bool _publishPending;
|
||||
|
||||
/// <summary>Delay between gRPC reconnection attempts (ctor-injected; production default 5s).</summary>
|
||||
private readonly TimeSpan _reconnectDelay;
|
||||
|
||||
/// <summary>
|
||||
/// How long a freshly-opened gRPC stream must stay up before its retry budget is
|
||||
/// considered recovered (mirrors <see cref="DebugStreamBridgeActor.StabilityWindow"/>).
|
||||
/// Settable for tests.
|
||||
/// considered recovered (mirrors <see cref="DebugStreamBridgeActor.StabilityWindow"/>);
|
||||
/// ctor-injected (production default 60s).
|
||||
/// </summary>
|
||||
internal static TimeSpan StabilityWindow { get; set; } = TimeSpan.FromSeconds(60);
|
||||
private readonly TimeSpan _stabilityWindow;
|
||||
|
||||
private int _retryCount;
|
||||
private bool _useNodeA = true;
|
||||
@@ -94,6 +99,21 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
/// <summary>Ordered buffer of live deltas that arrived while a fan-out was in flight. Actor-thread only.</summary>
|
||||
private readonly List<AlarmStateChanged> _buffer = new();
|
||||
|
||||
/// <summary>
|
||||
/// A failover re-seed was requested while a fan-out was already in flight; it must run
|
||||
/// right after the in-flight one completes rather than being silently dropped (N7.1) —
|
||||
/// the in-flight snapshot's read-time predates the stream death. Actor-thread only.
|
||||
/// </summary>
|
||||
private bool _reseedQueued;
|
||||
|
||||
/// <summary>
|
||||
/// Monotonic stream generation stamped on each opened gRPC stream and echoed back on its
|
||||
/// error callback: a late error raced out of a previous (cancelled) stream carries a stale
|
||||
/// generation and is ignored so it never burns retry budget or double-flips (N7.2).
|
||||
/// Actor-thread only.
|
||||
/// </summary>
|
||||
private int _streamGeneration;
|
||||
|
||||
private const int BufferWarnThreshold = 10_000;
|
||||
private bool _bufferWarned;
|
||||
|
||||
@@ -118,6 +138,15 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
/// <param name="grpcNodeAAddress">gRPC address of the site's node A.</param>
|
||||
/// <param name="grpcNodeBAddress">gRPC address of the site's node B.</param>
|
||||
/// <param name="reconcileInterval">Periodic reconcile snapshot cadence.</param>
|
||||
/// <param name="publishCoalesce">
|
||||
/// Publish-coalescing window for live deltas: a positive value batches a delta storm
|
||||
/// into one publish per window (review 02 round 2, N6); <see cref="TimeSpan.Zero"/>
|
||||
/// restores per-delta publishing (legacy). Seed/reconcile publishes stay immediate.
|
||||
/// </param>
|
||||
/// <param name="reconnectDelay">Delay between gRPC reconnection attempts (production 5s).</param>
|
||||
/// <param name="stabilityWindow">
|
||||
/// How long a fresh gRPC stream must stay up before its retry budget recovers (production 60s).
|
||||
/// </param>
|
||||
public SiteAlarmAggregatorActor(
|
||||
string siteIdentifier,
|
||||
string correlationId,
|
||||
@@ -126,7 +155,10 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
SiteStreamGrpcClientFactory grpcFactory,
|
||||
string grpcNodeAAddress,
|
||||
string grpcNodeBAddress,
|
||||
TimeSpan reconcileInterval)
|
||||
TimeSpan reconcileInterval,
|
||||
TimeSpan publishCoalesce,
|
||||
TimeSpan reconnectDelay,
|
||||
TimeSpan stabilityWindow)
|
||||
{
|
||||
_siteIdentifier = siteIdentifier;
|
||||
_correlationId = correlationId;
|
||||
@@ -136,6 +168,9 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
_grpcNodeAAddress = grpcNodeAAddress;
|
||||
_grpcNodeBAddress = grpcNodeBAddress;
|
||||
_reconcileInterval = reconcileInterval;
|
||||
_publishCoalesce = publishCoalesce;
|
||||
_reconnectDelay = reconnectDelay;
|
||||
_stabilityWindow = stabilityWindow;
|
||||
|
||||
// Live delta from the site-wide alarm stream (marshalled in via Self.Tell).
|
||||
// A received delta must NOT reset the retry budget (a flapping stream that
|
||||
@@ -152,6 +187,13 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
// Periodic reconcile tick (and the re-seed kicked after a reconnect).
|
||||
Receive<RunReconcile>(_ => OnReconcileTick());
|
||||
|
||||
// Coalesced-publish tick: one publish for a batch of dirtying deltas (N6).
|
||||
Receive<PublishCoalesced>(_ =>
|
||||
{
|
||||
_publishPending = false;
|
||||
if (!_stopped) Publish();
|
||||
});
|
||||
|
||||
// Stream stayed up for StabilityWindow — recover the retry budget.
|
||||
Receive<GrpcAlarmStreamStable>(_ =>
|
||||
{
|
||||
@@ -163,6 +205,15 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
// gRPC stream error — flip node + reconnect + re-seed.
|
||||
Receive<GrpcAlarmStreamError>(msg =>
|
||||
{
|
||||
// Ignore a late error raced out of a previous (cancelled) stream — the
|
||||
// RpcException(Cancelled) filter at SiteStreamGrpcClient.cs covers the normal
|
||||
// path, but a genuine socket fault can beat the cancel (N7.2).
|
||||
if (msg.Generation != _streamGeneration)
|
||||
{
|
||||
_log.Debug("Ignoring stale gRPC error from stream generation {0} (current {1})",
|
||||
msg.Generation, _streamGeneration);
|
||||
return;
|
||||
}
|
||||
_log.Warning("Site-alarm gRPC stream error for {0}: {1}", _siteIdentifier, msg.Exception.Message);
|
||||
HandleGrpcError();
|
||||
});
|
||||
@@ -247,7 +298,11 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
if (_stopped) return;
|
||||
if (_fanoutInFlight)
|
||||
{
|
||||
// Already seeding/reconciling — don't stack a second fan-out.
|
||||
// A failover re-seed requested mid-fan-out must run right after the in-flight
|
||||
// one — its snapshot read-time predates the stream death, so skipping it would
|
||||
// serve stale up to the next 60s reconcile (N7.1). An initial-seed collision
|
||||
// never queues (there is only ever one).
|
||||
if (!isInitial) _reseedQueued = true;
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -300,7 +355,18 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
_log.Debug("Site-alarm {0} {1} complete: {2} alarm row(s)",
|
||||
_siteIdentifier, msg.IsInitial ? "seed" : "reconcile", _cache.Count);
|
||||
|
||||
// The fresh snapshot already carries the buffered deltas; drop any armed coalesce
|
||||
// tick so we publish once, immediately.
|
||||
Timers.Cancel(PublishTimerKey);
|
||||
_publishPending = false;
|
||||
Publish();
|
||||
|
||||
// A failover re-seed requested while this fan-out was in flight runs now (N7.1).
|
||||
if (_reseedQueued)
|
||||
{
|
||||
_reseedQueued = false;
|
||||
StartFanout(isInitial: false);
|
||||
}
|
||||
}
|
||||
|
||||
private void OnSeedFailed(SeedFailed msg)
|
||||
@@ -319,7 +385,18 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
// Only publish if we already had a seed (so IsLive doesn't flip true on a
|
||||
// failed initial seed — the page keeps its poll fallback until we truly seed).
|
||||
if (_seeded)
|
||||
{
|
||||
Timers.Cancel(PublishTimerKey);
|
||||
_publishPending = false;
|
||||
Publish();
|
||||
}
|
||||
|
||||
// A failover re-seed requested while this fan-out was in flight runs now (N7.1).
|
||||
if (_reseedQueued)
|
||||
{
|
||||
_reseedQueued = false;
|
||||
StartFanout(isInitial: false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -362,9 +439,23 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
return;
|
||||
}
|
||||
|
||||
// Pass-through: apply and publish only if the cache actually changed.
|
||||
// Pass-through: apply and (coalesced) publish only if the cache actually changed.
|
||||
if (ApplyDelta(delta, requireStrictlyNewer: false))
|
||||
Publish();
|
||||
SchedulePublish();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Coalesced publish: with a positive window, the first dirtying delta arms a
|
||||
/// single-shot timer and further deltas ride the same tick — one snapshot copy and
|
||||
/// one viewer fan-out per window instead of per transition (N6). Zero = legacy
|
||||
/// immediate publish. Last write wins, so batching never changes final state.
|
||||
/// </summary>
|
||||
private void SchedulePublish()
|
||||
{
|
||||
if (_publishCoalesce <= TimeSpan.Zero) { Publish(); return; }
|
||||
if (_publishPending) return;
|
||||
_publishPending = true;
|
||||
Timers.StartSingleTimer(PublishTimerKey, new PublishCoalesced(), _publishCoalesce);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -416,8 +507,9 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
_grpcCts?.Dispose();
|
||||
_grpcCts = new CancellationTokenSource();
|
||||
|
||||
Timers.StartSingleTimer(StabilityTimerKey, new GrpcAlarmStreamStable(), StabilityWindow);
|
||||
Timers.StartSingleTimer(StabilityTimerKey, new GrpcAlarmStreamStable(), _stabilityWindow);
|
||||
|
||||
var generation = ++_streamGeneration;
|
||||
var client = _grpcFactory.GetOrCreate(_siteIdentifier, endpoint);
|
||||
var self = Self;
|
||||
var ct = _grpcCts.Token;
|
||||
@@ -427,7 +519,7 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
await client.SubscribeSiteAsync(
|
||||
_correlationId,
|
||||
alarm => self.Tell(alarm),
|
||||
ex => self.Tell(new GrpcAlarmStreamError(ex)),
|
||||
ex => self.Tell(new GrpcAlarmStreamError(ex, generation)),
|
||||
ct);
|
||||
}, ct);
|
||||
}
|
||||
@@ -474,7 +566,7 @@ public sealed class SiteAlarmAggregatorActor : ReceiveActor, IWithTimers
|
||||
if (_retryCount == 1)
|
||||
Self.Tell(new ReconnectAlarmStream());
|
||||
else
|
||||
Timers.StartSingleTimer(ReconnectTimerKey, new ReconnectAlarmStream(), ReconnectDelay);
|
||||
Timers.StartSingleTimer(ReconnectTimerKey, new ReconnectAlarmStream(), _reconnectDelay);
|
||||
}
|
||||
|
||||
private void CleanupGrpc()
|
||||
@@ -520,8 +612,12 @@ internal sealed record SeedFailed(Exception Exception, bool IsInitial);
|
||||
/// <summary>Internal: periodic reconcile tick (and the re-seed kicked after a reconnect).</summary>
|
||||
internal sealed record RunReconcile;
|
||||
|
||||
/// <summary>Internal: site-alarm gRPC stream error occurred.</summary>
|
||||
internal sealed record GrpcAlarmStreamError(Exception Exception);
|
||||
/// <summary>Internal: coalesced-publish tick — flush the dirty cache to viewers once (N6).</summary>
|
||||
internal sealed record PublishCoalesced;
|
||||
|
||||
/// <summary>Internal: site-alarm gRPC stream error occurred, stamped with the stream
|
||||
/// generation it came from so a late error from a cancelled stream can be ignored (N7.2).</summary>
|
||||
internal sealed record GrpcAlarmStreamError(Exception Exception, int Generation);
|
||||
|
||||
/// <summary>Internal: reconnect the site-alarm gRPC stream (flip node).</summary>
|
||||
internal sealed record ReconnectAlarmStream;
|
||||
|
||||
@@ -67,10 +67,10 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
|
||||
/// <param name="deploymentManagerProxy">Local reference to the Deployment Manager singleton proxy.</param>
|
||||
/// <param name="isActiveCheck">
|
||||
/// Optional override returning <c>true</c> when this node
|
||||
/// is the active member of the site cluster. <c>null</c> uses the real
|
||||
/// Akka <see cref="Cluster"/> leader check (the default for production
|
||||
/// wiring); tests pass a stub so they do not need to load Akka.Cluster
|
||||
/// into the <c>TestKit</c> ActorSystem.
|
||||
/// is the active member of the site cluster. <c>null</c> uses the shared oldest-Up
|
||||
/// evaluator (production wiring passes the Host's singleton-host delegate); tests
|
||||
/// pass a stub so they do not need to load Akka.Cluster into the <c>TestKit</c>
|
||||
/// ActorSystem.
|
||||
/// </param>
|
||||
public SiteCommunicationActor(
|
||||
string siteId,
|
||||
@@ -506,24 +506,16 @@ public class SiteCommunicationActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default active-node check used when no override is
|
||||
/// supplied. Mirrors <c>ActiveNodeGate</c> in the Host (and
|
||||
/// <c>ActiveNodeHealthCheck</c>): the node is the active member of the
|
||||
/// site cluster when it is the current cluster leader AND its own
|
||||
/// <see cref="MemberStatus"/> is <see cref="MemberStatus.Up"/>. Any other
|
||||
/// state (still joining, leaving, no leader yet) reports standby —
|
||||
/// safe-by-default, matching the standby case.
|
||||
/// Default active-node check used when no override is supplied: oldest-Up member
|
||||
/// semantics via the shared <see cref="ClusterState.ActiveNodeEvaluator"/> — the
|
||||
/// same predicate as the S&F delivery gate and the replication resync authority
|
||||
/// (review 02 round 2, N1). Unscoped by role: a site cluster's members all carry
|
||||
/// the site role, so role scoping is a no-op here; production wiring passes the
|
||||
/// Host's role-scoped IClusterNodeProvider delegate anyway. Any other state
|
||||
/// (still joining, leaving) reports standby — safe-by-default.
|
||||
/// </summary>
|
||||
private bool DefaultIsActiveCheck()
|
||||
{
|
||||
var cluster = Cluster.Get(Context.System);
|
||||
var self = cluster.SelfMember;
|
||||
if (self.Status != MemberStatus.Up)
|
||||
return false;
|
||||
|
||||
var leader = cluster.State.Leader;
|
||||
return leader != null && leader == self.Address;
|
||||
}
|
||||
private bool DefaultIsActiveCheck() =>
|
||||
ClusterState.ActiveNodeEvaluator.SelfIsOldestUp(Cluster.Get(Context.System));
|
||||
|
||||
// ── Internal messages ──
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
using Akka.Cluster;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.ClusterState;
|
||||
|
||||
/// <summary>
|
||||
/// THE single definition of "active node" (review 01 [High]; review 02 round 2 [Critical] N1):
|
||||
/// a node is active when it is the OLDEST Up member (optionally within a role scope) — i.e.
|
||||
/// the member the ClusterSingletonManager places singletons on. Cluster LEADERSHIP (lowest
|
||||
/// address) is an Akka-internal concept that diverges from singleton placement permanently
|
||||
/// once the original first node restarts and rejoins; every product-level active/standby
|
||||
/// decision must use this evaluator, never <c>cluster.State.Leader</c>.
|
||||
/// <para>
|
||||
/// Lives in Communication (not Host) so BOTH <c>SiteCommunicationActor</c> and
|
||||
/// <c>SiteReplicationActor</c> can default to it — Host cannot be referenced from either.
|
||||
/// The Host's <c>ClusterActivityEvaluator.SelfIsOldest</c> delegates here, so the S&F
|
||||
/// delivery gate (<c>IClusterNodeProvider.SelfIsPrimary</c>), the resync authority checks,
|
||||
/// and the heartbeat IsActive stamp all share one implementation.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public static class ActiveNodeEvaluator
|
||||
{
|
||||
/// <summary>True when self is Up and no other Up member (in the role scope) is older.</summary>
|
||||
/// <param name="cluster">The Akka cluster to evaluate.</param>
|
||||
/// <param name="role">Optional role scope; when set, only members with this role are considered.</param>
|
||||
/// <returns><c>true</c> when self is Up and the oldest Up member in the role scope.</returns>
|
||||
public static bool SelfIsOldestUp(Cluster cluster, string? role = null)
|
||||
{
|
||||
var self = cluster.SelfMember;
|
||||
if (self.Status != MemberStatus.Up)
|
||||
return false;
|
||||
if (role != null && !self.HasRole(role))
|
||||
return false;
|
||||
|
||||
return cluster.State.Members
|
||||
.Where(m => m.Status == MemberStatus.Up)
|
||||
.Where(m => role == null || m.HasRole(role))
|
||||
.All(m => m.UniqueAddress.Equals(self.UniqueAddress) || self.IsOlderThan(m));
|
||||
}
|
||||
}
|
||||
@@ -122,4 +122,13 @@ public class CommunicationOptions
|
||||
/// count; this just bounds the subscriber list. Default 200.
|
||||
/// </summary>
|
||||
public int LiveAlarmCacheMaxSubscribersPerSite { get; set; } = 200;
|
||||
|
||||
/// <summary>
|
||||
/// Publish-coalescing window for live alarm deltas: an applied delta marks the cache
|
||||
/// dirty and one publish (fresh snapshot + per-viewer onChanged fan-out) fires after
|
||||
/// this window, batching an alarm storm into ~4 publishes/second instead of one per
|
||||
/// transition (review 02 round 2, N6). Zero = publish per delta (legacy). Seed and
|
||||
/// reconcile publishes are always immediate. Default 250 ms.
|
||||
/// </summary>
|
||||
public TimeSpan LiveAlarmCachePublishCoalesce { get; set; } = TimeSpan.FromMilliseconds(250);
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication;
|
||||
|
||||
/// <summary>
|
||||
/// Validates <see cref="CommunicationOptions"/> at startup (ValidateOnStart) so a
|
||||
/// malformed "Communication" appsettings section fails fast at boot with a
|
||||
/// malformed "ScadaBridge:Communication" appsettings section fails fast at boot with a
|
||||
/// key-naming message instead of surfacing later at first Ask/gRPC use. Every
|
||||
/// timeout feeds a per-pattern Ask deadline or a gRPC keepalive/stream-lifetime
|
||||
/// setting, and a zero/negative value there produces an opaque runtime failure
|
||||
@@ -16,74 +16,79 @@ public sealed class CommunicationOptionsValidator : OptionsValidatorBase<Communi
|
||||
protected override void Validate(ValidationBuilder builder, CommunicationOptions options)
|
||||
{
|
||||
builder.RequireThat(options.DeploymentTimeout > TimeSpan.Zero,
|
||||
$"Communication:DeploymentTimeout must be a positive duration (was {options.DeploymentTimeout}).");
|
||||
$"ScadaBridge:Communication:DeploymentTimeout must be a positive duration (was {options.DeploymentTimeout}).");
|
||||
|
||||
builder.RequireThat(options.LifecycleTimeout > TimeSpan.Zero,
|
||||
$"Communication:LifecycleTimeout must be a positive duration (was {options.LifecycleTimeout}).");
|
||||
$"ScadaBridge:Communication:LifecycleTimeout must be a positive duration (was {options.LifecycleTimeout}).");
|
||||
|
||||
builder.RequireThat(options.ArtifactDeploymentTimeout > TimeSpan.Zero,
|
||||
$"Communication:ArtifactDeploymentTimeout must be a positive duration (was {options.ArtifactDeploymentTimeout}).");
|
||||
$"ScadaBridge:Communication:ArtifactDeploymentTimeout must be a positive duration (was {options.ArtifactDeploymentTimeout}).");
|
||||
|
||||
builder.RequireThat(options.QueryTimeout > TimeSpan.Zero,
|
||||
$"Communication:QueryTimeout must be a positive duration (was {options.QueryTimeout}).");
|
||||
$"ScadaBridge:Communication:QueryTimeout must be a positive duration (was {options.QueryTimeout}).");
|
||||
|
||||
builder.RequireThat(options.IntegrationTimeout > TimeSpan.Zero,
|
||||
$"Communication:IntegrationTimeout must be a positive duration (was {options.IntegrationTimeout}).");
|
||||
$"ScadaBridge:Communication:IntegrationTimeout must be a positive duration (was {options.IntegrationTimeout}).");
|
||||
|
||||
builder.RequireThat(options.DebugViewTimeout > TimeSpan.Zero,
|
||||
$"Communication:DebugViewTimeout must be a positive duration (was {options.DebugViewTimeout}).");
|
||||
$"ScadaBridge:Communication:DebugViewTimeout must be a positive duration (was {options.DebugViewTimeout}).");
|
||||
|
||||
builder.RequireThat(options.HealthReportTimeout > TimeSpan.Zero,
|
||||
$"Communication:HealthReportTimeout must be a positive duration (was {options.HealthReportTimeout}).");
|
||||
$"ScadaBridge:Communication:HealthReportTimeout must be a positive duration (was {options.HealthReportTimeout}).");
|
||||
|
||||
builder.RequireThat(options.NotificationForwardTimeout > TimeSpan.Zero,
|
||||
$"Communication:NotificationForwardTimeout must be a positive duration (was {options.NotificationForwardTimeout}).");
|
||||
$"ScadaBridge:Communication:NotificationForwardTimeout must be a positive duration (was {options.NotificationForwardTimeout}).");
|
||||
|
||||
builder.RequireThat(options.GrpcKeepAlivePingDelay > TimeSpan.Zero,
|
||||
$"Communication:GrpcKeepAlivePingDelay must be a positive duration (was {options.GrpcKeepAlivePingDelay}).");
|
||||
$"ScadaBridge:Communication:GrpcKeepAlivePingDelay must be a positive duration (was {options.GrpcKeepAlivePingDelay}).");
|
||||
|
||||
builder.RequireThat(options.GrpcKeepAlivePingTimeout > TimeSpan.Zero,
|
||||
$"Communication:GrpcKeepAlivePingTimeout must be a positive duration (was {options.GrpcKeepAlivePingTimeout}).");
|
||||
$"ScadaBridge:Communication:GrpcKeepAlivePingTimeout must be a positive duration (was {options.GrpcKeepAlivePingTimeout}).");
|
||||
|
||||
builder.RequireThat(options.GrpcMaxStreamLifetime > TimeSpan.Zero,
|
||||
$"Communication:GrpcMaxStreamLifetime must be a positive duration (was {options.GrpcMaxStreamLifetime}).");
|
||||
$"ScadaBridge:Communication:GrpcMaxStreamLifetime must be a positive duration (was {options.GrpcMaxStreamLifetime}).");
|
||||
|
||||
builder.RequireThat(options.TransportHeartbeatInterval > TimeSpan.Zero,
|
||||
$"Communication:TransportHeartbeatInterval must be a positive duration (was {options.TransportHeartbeatInterval}).");
|
||||
$"ScadaBridge:Communication:TransportHeartbeatInterval must be a positive duration (was {options.TransportHeartbeatInterval}).");
|
||||
|
||||
builder.RequireThat(options.ApplicationHeartbeatInterval > TimeSpan.Zero,
|
||||
$"Communication:ApplicationHeartbeatInterval must be a positive duration (was {options.ApplicationHeartbeatInterval}).");
|
||||
$"ScadaBridge:Communication:ApplicationHeartbeatInterval must be a positive duration (was {options.ApplicationHeartbeatInterval}).");
|
||||
|
||||
builder.RequireThat(options.TransportFailureThreshold > TimeSpan.Zero,
|
||||
$"Communication:TransportFailureThreshold must be a positive duration (was {options.TransportFailureThreshold}).");
|
||||
$"ScadaBridge:Communication:TransportFailureThreshold must be a positive duration (was {options.TransportFailureThreshold}).");
|
||||
|
||||
builder.RequireThat(options.PendingDeploymentTtl > TimeSpan.Zero,
|
||||
$"Communication:PendingDeploymentTtl must be a positive duration (was {options.PendingDeploymentTtl}).");
|
||||
$"ScadaBridge:Communication:PendingDeploymentTtl must be a positive duration (was {options.PendingDeploymentTtl}).");
|
||||
|
||||
builder.RequireThat(options.PendingDeploymentPurgeInterval > TimeSpan.Zero,
|
||||
$"Communication:PendingDeploymentPurgeInterval must be a positive duration (was {options.PendingDeploymentPurgeInterval}).");
|
||||
$"ScadaBridge:Communication:PendingDeploymentPurgeInterval must be a positive duration (was {options.PendingDeploymentPurgeInterval}).");
|
||||
|
||||
builder.RequireThat(options.GrpcMaxConcurrentStreams > 0,
|
||||
$"Communication:GrpcMaxConcurrentStreams must be positive (was {options.GrpcMaxConcurrentStreams}).");
|
||||
$"ScadaBridge:Communication:GrpcMaxConcurrentStreams must be positive (was {options.GrpcMaxConcurrentStreams}).");
|
||||
|
||||
// ── Aggregated live alarm cache (plan #10, Task 6) ───────────────────────
|
||||
// Linger drives a Timer dueTime; TimeSpan.Zero is valid (stop the aggregator
|
||||
// immediately when the last viewer leaves), only a negative value is invalid.
|
||||
builder.RequireThat(options.LiveAlarmCacheLinger >= TimeSpan.Zero,
|
||||
$"Communication:LiveAlarmCacheLinger must be zero or a positive duration (was {options.LiveAlarmCacheLinger}).");
|
||||
$"ScadaBridge:Communication:LiveAlarmCacheLinger must be zero or a positive duration (was {options.LiveAlarmCacheLinger}).");
|
||||
|
||||
// Reconcile interval is a periodic timer cadence AND the start-retry cadence; a
|
||||
// zero/negative value would spin or never fire.
|
||||
builder.RequireThat(options.LiveAlarmCacheReconcileInterval > TimeSpan.Zero,
|
||||
$"Communication:LiveAlarmCacheReconcileInterval must be a positive duration (was {options.LiveAlarmCacheReconcileInterval}).");
|
||||
$"ScadaBridge:Communication:LiveAlarmCacheReconcileInterval must be a positive duration (was {options.LiveAlarmCacheReconcileInterval}).");
|
||||
|
||||
// Seed fan-out concurrency gates a SemaphoreSlim; at least 1, capped so a
|
||||
// fat-fingered config can't open an unbounded burst of cross-cluster Asks.
|
||||
builder.RequireThat(options.LiveAlarmCacheSeedConcurrency is >= 1 and <= 64,
|
||||
$"Communication:LiveAlarmCacheSeedConcurrency must be between 1 and 64 (was {options.LiveAlarmCacheSeedConcurrency}).");
|
||||
$"ScadaBridge:Communication:LiveAlarmCacheSeedConcurrency must be between 1 and 64 (was {options.LiveAlarmCacheSeedConcurrency}).");
|
||||
|
||||
// Per-site viewer cap must admit at least one viewer, else the page could never go live.
|
||||
builder.RequireThat(options.LiveAlarmCacheMaxSubscribersPerSite >= 1,
|
||||
$"Communication:LiveAlarmCacheMaxSubscribersPerSite must be at least 1 (was {options.LiveAlarmCacheMaxSubscribersPerSite}).");
|
||||
$"ScadaBridge:Communication:LiveAlarmCacheMaxSubscribersPerSite must be at least 1 (was {options.LiveAlarmCacheMaxSubscribersPerSite}).");
|
||||
|
||||
// Publish-coalescing window drives a single-shot timer; TimeSpan.Zero is valid
|
||||
// (publish per delta — legacy), only a negative value is invalid.
|
||||
builder.RequireThat(options.LiveAlarmCachePublishCoalesce >= TimeSpan.Zero,
|
||||
$"ScadaBridge:Communication:LiveAlarmCachePublishCoalesce must be zero or a positive duration (was {options.LiveAlarmCachePublishCoalesce}).");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,9 +11,12 @@ namespace ZB.MOM.WW.ScadaBridge.Communication;
|
||||
/// reflects alarm transitions in near-real-time instead of re-polling every 15s.
|
||||
/// <para>
|
||||
/// <b>Hard constraint (locked <c>[PERM]</c>):</b> there is NO persisted central alarm
|
||||
/// store. This cache is purely in-memory and lives only on the active central node —
|
||||
/// no EF entity/table/migration backs it. On a NodeA↔NodeB failover the new active
|
||||
/// node re-seeds from scratch.
|
||||
/// store. This cache is purely in-memory, per-node — no EF entity/table/migration backs
|
||||
/// it. In routine operation only the active node (behind Traefik) hosts viewers, so only
|
||||
/// it runs aggregators; browsing the standby node directly (diagnostic ports) starts an
|
||||
/// independent read-only aggregator there — accepted (arch review 02 round 2, N8):
|
||||
/// bounded (one stream/site, viewer-capped), read-only, and torn down by the viewer
|
||||
/// linger. On a NodeA↔NodeB failover the new active node re-seeds from scratch.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Reference-counted lifecycle:</b> the first <see cref="Subscribe"/> for a site
|
||||
@@ -57,6 +60,6 @@ public interface ISiteAlarmLiveCache
|
||||
/// cache is authoritative.
|
||||
/// </summary>
|
||||
/// <param name="siteId">The numeric site id.</param>
|
||||
/// <returns><c>true</c> once the site's aggregator has seeded and published at least once.</returns>
|
||||
/// <returns><c>true</c> while a <b>living</b> aggregator has seeded and published; reverts to false if the aggregator terminates (R2 N6) — a frozen snapshot is never reported as live.</returns>
|
||||
bool IsLive(int siteId);
|
||||
}
|
||||
|
||||
@@ -12,8 +12,12 @@ public static class ServiceCollectionExtensions
|
||||
/// <returns>The same <see cref="IServiceCollection"/> to allow chaining.</returns>
|
||||
public static IServiceCollection AddCommunication(this IServiceCollection services)
|
||||
{
|
||||
// Canonical section is ScadaBridge:Communication — the section every real
|
||||
// appsettings actually writes (arch-review 08 round 2 NF8). The Host used to
|
||||
// supply the real values via a duplicate Configure<> binding; that duplicate is
|
||||
// gone, so AddCommunication must bind the canonical section itself.
|
||||
services.AddOptions<CommunicationOptions>()
|
||||
.BindConfiguration("Communication")
|
||||
.BindConfiguration("ScadaBridge:Communication")
|
||||
.ValidateOnStart();
|
||||
services.TryAddEnumerable(
|
||||
ServiceDescriptor.Singleton<IValidateOptions<CommunicationOptions>, CommunicationOptionsValidator>());
|
||||
|
||||
@@ -233,9 +233,20 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
|
||||
_grpcFactory,
|
||||
grpcA,
|
||||
grpcB,
|
||||
_options.LiveAlarmCacheReconcileInterval));
|
||||
_options.LiveAlarmCacheReconcileInterval,
|
||||
_options.LiveAlarmCachePublishCoalesce,
|
||||
TimeSpan.FromSeconds(5), // reconnect delay — former ReconnectDelay static default
|
||||
TimeSpan.FromSeconds(60))); // stability window — former StabilityWindow static default
|
||||
|
||||
entry.Actor = system.ActorOf(props, $"site-alarm-aggregator-{entry.SiteId}-{Guid.NewGuid():N}");
|
||||
|
||||
var aggregator = entry.Actor;
|
||||
// Deathwatch (arch-review R2 N6): if the aggregator terminates for any reason,
|
||||
// liveness must reset — a dead feed must never keep IsLive == true, or the page
|
||||
// grafts a frozen snapshot over fresh poll data for up to 15 s.
|
||||
system.ActorOf(Props.Create(() => new AggregatorTerminationWatcher(
|
||||
aggregator!, () => OnAggregatorTerminated(entry.SiteId, aggregator!))));
|
||||
|
||||
entry.StartRetryTimer?.Dispose();
|
||||
entry.StartRetryTimer = null;
|
||||
_logger.LogInformation("Started live alarm aggregator for site {SiteId} ({SiteIdentifier})",
|
||||
@@ -291,7 +302,9 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
|
||||
return null;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(site.GrpcNodeAAddress) || string.IsNullOrWhiteSpace(site.GrpcNodeBAddress))
|
||||
var nodeA = site.GrpcNodeAAddress;
|
||||
var nodeB = site.GrpcNodeBAddress;
|
||||
if (string.IsNullOrWhiteSpace(nodeA) && string.IsNullOrWhiteSpace(nodeB))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Live alarm cache: site {SiteId} ({SiteIdentifier}) has no gRPC node addresses; " +
|
||||
@@ -299,7 +312,13 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
|
||||
return null;
|
||||
}
|
||||
|
||||
return (site.SiteIdentifier, site.GrpcNodeAAddress!, site.GrpcNodeBAddress!);
|
||||
// Single-endpoint site (review 02 round 2, N9): reuse the sole configured
|
||||
// endpoint for both slots — the aggregator's NodeA↔NodeB failover flip
|
||||
// degenerates to a reconnect against the same node, which is exactly the
|
||||
// right behavior for a one-node site.
|
||||
var grpcA = !string.IsNullOrWhiteSpace(nodeA) ? nodeA! : nodeB!;
|
||||
var grpcB = !string.IsNullOrWhiteSpace(nodeB) ? nodeB! : nodeA!;
|
||||
return (site.SiteIdentifier, grpcA, grpcB);
|
||||
}
|
||||
|
||||
// ── Snapshot fan-out (the seed / reconcile) ─────────────────────────────────
|
||||
@@ -410,6 +429,57 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
|
||||
}
|
||||
}
|
||||
|
||||
// ── Deathwatch (R2 N6) ──────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>Watches one aggregator and fires a callback exactly once on Terminated.</summary>
|
||||
private sealed class AggregatorTerminationWatcher : ReceiveActor
|
||||
{
|
||||
public AggregatorTerminationWatcher(IActorRef watched, Action onTerminated)
|
||||
{
|
||||
Context.Watch(watched);
|
||||
Receive<Terminated>(_ =>
|
||||
{
|
||||
onTerminated();
|
||||
Context.Stop(Self);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Terminated hook (R2 N6). A DELIBERATE stop (linger fired, entry already removed
|
||||
/// from _sites) is a no-op — the ReferenceEquals guard also ignores a stale watcher
|
||||
/// firing after a newer aggregator replaced the dead one. A crash-death with live
|
||||
/// viewers resets liveness (IsLive → false, snapshot cleared, so the page's poll is
|
||||
/// authoritative again) and arms the existing bounded start-retry so the feature
|
||||
/// self-heals, mirroring StartAggregatorAsync's transient-failure path.
|
||||
/// </summary>
|
||||
private void OnAggregatorTerminated(int siteId, IActorRef deadActor)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
if (!_sites.TryGetValue(siteId, out var entry) || !ReferenceEquals(entry.Actor, deadActor))
|
||||
return;
|
||||
|
||||
entry.Actor = null;
|
||||
entry.HasPublished = false;
|
||||
entry.Current = Empty;
|
||||
|
||||
if (entry.Subscribers.Count > 0 && !entry.Starting)
|
||||
{
|
||||
entry.Starting = true;
|
||||
entry.StartRetryTimer?.Dispose();
|
||||
entry.StartRetryTimer = new Timer(
|
||||
_ => { _ = StartAggregatorAsync(entry); },
|
||||
state: null,
|
||||
dueTime: _options.LiveAlarmCacheReconcileInterval,
|
||||
period: Timeout.InfiniteTimeSpan);
|
||||
}
|
||||
}
|
||||
_logger.LogWarning(
|
||||
"Live alarm aggregator for site {SiteId} terminated unexpectedly; liveness reset " +
|
||||
"(page falls back to polling) and a restart is armed while viewers remain", siteId);
|
||||
}
|
||||
|
||||
// ── Nested types ────────────────────────────────────────────────────────────
|
||||
|
||||
/// <summary>Per-site bookkeeping. All mutable fields are guarded by the service's <c>_lock</c>.</summary>
|
||||
|
||||
+9
@@ -97,5 +97,14 @@ public class SiteCallEntityTypeConfiguration : IEntityTypeConfiguration<SiteCall
|
||||
.HasDatabaseName("IX_SiteCalls_NonTerminal")
|
||||
.HasFilter("[TerminalAtUtc] IS NULL")
|
||||
.IncludeProperties(nameof(SiteCall.SourceSite), nameof(SiteCall.SourceNode), nameof(SiteCall.Status));
|
||||
|
||||
// Terminal backs the daily retention purge's predicate (TerminalAtUtc IS NOT NULL
|
||||
// AND TerminalAtUtc < cutoff) and Task 11's MIN() slice anchor: filtered to the
|
||||
// terminal population so the purge seeks the expired tail instead of full-scanning
|
||||
// the 365-day table (arch-review 04 round 2, R6). Complements IX_SiteCalls_NonTerminal,
|
||||
// which is filtered to IS NULL and unusable for this predicate.
|
||||
builder.HasIndex(s => s.TerminalAtUtc)
|
||||
.HasDatabaseName("IX_SiteCalls_Terminal")
|
||||
.HasFilter("[TerminalAtUtc] IS NOT NULL");
|
||||
}
|
||||
}
|
||||
|
||||
+2090
File diff suppressed because it is too large
Load Diff
+28
@@ -0,0 +1,28 @@
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
#nullable disable
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public partial class AddSiteCallsTerminalIndex : Migration
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "IX_SiteCalls_Terminal",
|
||||
table: "SiteCalls",
|
||||
column: "TerminalAtUtc",
|
||||
filter: "[TerminalAtUtc] IS NOT NULL");
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.DropIndex(
|
||||
name: "IX_SiteCalls_Terminal",
|
||||
table: "SiteCalls");
|
||||
}
|
||||
}
|
||||
}
|
||||
+4
@@ -167,6 +167,10 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Migrations
|
||||
|
||||
SqlServerIndexBuilderExtensions.IncludeProperties(b.HasIndex("CreatedAtUtc"), new[] { "SourceSite", "SourceNode", "Status" });
|
||||
|
||||
b.HasIndex("TerminalAtUtc")
|
||||
.HasDatabaseName("IX_SiteCalls_Terminal")
|
||||
.HasFilter("[TerminalAtUtc] IS NOT NULL");
|
||||
|
||||
b.HasIndex("SourceSite", "CreatedAtUtc")
|
||||
.IsDescending(false, true)
|
||||
.HasDatabaseName("IX_SiteCalls_Source_Created");
|
||||
|
||||
+46
-2
@@ -1,4 +1,6 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Entities.Kpi;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types.Kpi;
|
||||
@@ -13,14 +15,22 @@ namespace ZB.MOM.WW.ScadaBridge.ConfigurationDatabase.Repositories;
|
||||
public sealed class KpiHistoryRepository : IKpiHistoryRepository
|
||||
{
|
||||
private readonly ScadaBridgeDbContext _context;
|
||||
private readonly ILogger<KpiHistoryRepository> _logger;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="KpiHistoryRepository"/> class.
|
||||
/// </summary>
|
||||
/// <param name="context">The EF Core database context.</param>
|
||||
public KpiHistoryRepository(ScadaBridgeDbContext context)
|
||||
/// <param name="logger">
|
||||
/// Optional logger; defaults to <see cref="NullLogger{T}"/> (mirrors
|
||||
/// <c>SiteCallAuditRepository</c> — MS.DI resolves <see cref="ILogger{T}"/> automatically,
|
||||
/// so no registration churn). Used to classify the self-healing failover fold-race.
|
||||
/// </param>
|
||||
public KpiHistoryRepository(
|
||||
ScadaBridgeDbContext context, ILogger<KpiHistoryRepository>? logger = null)
|
||||
{
|
||||
_context = context ?? throw new ArgumentNullException(nameof(context));
|
||||
_logger = logger ?? NullLogger<KpiHistoryRepository>.Instance;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -101,8 +111,14 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
|
||||
// fetch it and group IN MEMORY. This deliberately avoids provider-specific
|
||||
// DATEADD/DATEPART hour-truncation translation (SQLite in tests, SQL Server in prod),
|
||||
// keeping the hour-bucket arithmetic identical across providers and trivially correct.
|
||||
// Projected, untracked fetch: the fold only reads these six fields and never
|
||||
// mutates a KpiSample. A tracked ToListAsync registered ~5k-90k read-only
|
||||
// entities in the change tracker per healthy 3h fold — all re-scanned by
|
||||
// DetectChanges on the final SaveChanges (arch-review 04 round 2, R2). A
|
||||
// projection is inherently untracked and materializes no entity at all.
|
||||
var samples = await _context.KpiSamples
|
||||
.Where(s => s.CapturedAtUtc >= from && s.CapturedAtUtc < to)
|
||||
.Select(s => new FoldSample(s.Source, s.Metric, s.Scope, s.ScopeKey, s.CapturedAtUtc, s.Value))
|
||||
.ToListAsync(cancellationToken);
|
||||
if (samples.Count == 0)
|
||||
{
|
||||
@@ -164,7 +180,22 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
|
||||
}
|
||||
}
|
||||
|
||||
await _context.SaveChangesAsync(cancellationToken);
|
||||
try
|
||||
{
|
||||
await _context.SaveChangesAsync(cancellationToken);
|
||||
}
|
||||
catch (DbUpdateException ex)
|
||||
{
|
||||
// Failover-overlap race: the old singleton incarnation's in-flight fold and the
|
||||
// new node's first fold can both Add the same (series, hour) row; the unique
|
||||
// IX_KpiRollupHourly_Series then faults the loser's ENTIRE SaveChanges — the
|
||||
// failure grain is the PASS, not the row (arch-review 04 round 2, R5). The fold
|
||||
// only ever writes KpiRollupHourly rows, so any save fault here is a lost upsert
|
||||
// race or a transient the next idempotent re-fold repairs identically; classify
|
||||
// at Information instead of surfacing a scary error for a self-healing no-op.
|
||||
_logger.LogInformation(ex,
|
||||
"KPI rollup fold lost a failover-overlap upsert race; pass discarded, next fold self-heals.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -186,6 +217,15 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
|
||||
.ToListAsync(cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<DateTime?> GetLatestRollupHourAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
// Single MAX over the unique IX_KpiRollupHourly_Series population (the rollup
|
||||
// table is small — one row per series-hour). Null when no rollups exist.
|
||||
return await _context.KpiRollupHourly
|
||||
.MaxAsync(r => (DateTime?)r.HourStartUtc, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task PurgeRollupsOlderThanAsync(DateTime before, CancellationToken cancellationToken = default)
|
||||
{
|
||||
@@ -212,4 +252,8 @@ public sealed class KpiHistoryRepository : IKpiHistoryRepository
|
||||
/// <summary>In-memory grouping key: one KPI series (four-tuple) within one UTC hour.</summary>
|
||||
private readonly record struct SeriesHourKey(
|
||||
string Source, string Metric, string Scope, string? ScopeKey, DateTime HourStartUtc);
|
||||
|
||||
/// <summary>Narrow, untracked projection of one KpiSample row for the fold (R2).</summary>
|
||||
private readonly record struct FoldSample(
|
||||
string Source, string Metric, string Scope, string? ScopeKey, DateTime CapturedAtUtc, double Value);
|
||||
}
|
||||
|
||||
+16
-1
@@ -57,6 +57,7 @@ public class SecuredWriteRepository : ISecuredWriteRepository
|
||||
string? siteId,
|
||||
int skip,
|
||||
int take,
|
||||
IReadOnlyCollection<string>? permittedSiteIds = null,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
IQueryable<PendingSecuredWrite> query = _context.Set<PendingSecuredWrite>()
|
||||
@@ -72,6 +73,11 @@ public class SecuredWriteRepository : ISecuredWriteRepository
|
||||
query = query.Where(p => p.SiteId == siteId);
|
||||
}
|
||||
|
||||
if (permittedSiteIds is not null)
|
||||
{
|
||||
query = query.Where(p => permittedSiteIds.Contains(p.SiteId));
|
||||
}
|
||||
|
||||
return await query
|
||||
.OrderByDescending(p => p.SubmittedAtUtc)
|
||||
.ThenByDescending(p => p.Id)
|
||||
@@ -128,7 +134,11 @@ WHERE Id = {id}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<int> CountAsync(string? status, string? siteId, CancellationToken ct = default)
|
||||
public async Task<int> CountAsync(
|
||||
string? status,
|
||||
string? siteId,
|
||||
IReadOnlyCollection<string>? permittedSiteIds = null,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
IQueryable<PendingSecuredWrite> query = _context.Set<PendingSecuredWrite>()
|
||||
.AsNoTracking();
|
||||
@@ -143,6 +153,11 @@ WHERE Id = {id}
|
||||
query = query.Where(p => p.SiteId == siteId);
|
||||
}
|
||||
|
||||
if (permittedSiteIds is not null)
|
||||
{
|
||||
query = query.Where(p => permittedSiteIds.Contains(p.SiteId));
|
||||
}
|
||||
|
||||
return await query.CountAsync(ct);
|
||||
}
|
||||
}
|
||||
|
||||
+23
-3
@@ -246,9 +246,29 @@ OPTION (RECOMPILE);";
|
||||
/// <inheritdoc />
|
||||
public async Task<int> PurgeTerminalAsync(DateTime olderThanUtc, CancellationToken ct = default)
|
||||
{
|
||||
return await _context.Database.ExecuteSqlInterpolatedAsync(
|
||||
$"DELETE FROM dbo.SiteCalls WHERE TerminalAtUtc IS NOT NULL AND TerminalAtUtc < {olderThanUtc};",
|
||||
ct);
|
||||
// Time-sliced batches (arch-review 04 round 2, R6 — the one maintenance DELETE
|
||||
// that missed round 1's batching pass): each DELETE covers at most one DAY of
|
||||
// terminal rows, capping the lock/log footprint per statement. Steady state
|
||||
// (daily purge, 365-day retention) is a single slice; only catch-up after an
|
||||
// outage runs several. One-day (not one-hour) slices are proportionate to
|
||||
// SiteCalls volume, which is far below KpiSample's. The MIN() anchor and the
|
||||
// DELETE predicate both seek IX_SiteCalls_Terminal (filtered IS NOT NULL).
|
||||
var total = 0;
|
||||
var floor = await _context.SiteCalls
|
||||
.Where(s => s.TerminalAtUtc != null && s.TerminalAtUtc < olderThanUtc)
|
||||
.MinAsync(s => s.TerminalAtUtc, ct);
|
||||
while (floor is not null && floor < olderThanUtc)
|
||||
{
|
||||
var ceiling = floor.Value.AddDays(1) < olderThanUtc ? floor.Value.AddDays(1) : olderThanUtc;
|
||||
total += await _context.Database.ExecuteSqlInterpolatedAsync(
|
||||
$"DELETE FROM dbo.SiteCalls WHERE TerminalAtUtc IS NOT NULL AND TerminalAtUtc < {ceiling};",
|
||||
ct);
|
||||
floor = await _context.SiteCalls
|
||||
.Where(s => s.TerminalAtUtc != null && s.TerminalAtUtc < olderThanUtc)
|
||||
.MinAsync(s => s.TerminalAtUtc, ct);
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
// Terminal status string literals for the interval-throughput KPIs. The
|
||||
|
||||
@@ -105,14 +105,16 @@ public class MxGatewayDataConnection : IDataConnection, IBrowsableDataConnection
|
||||
|
||||
// Background event loop: route each value change to the matching subscription callback.
|
||||
_eventLoopCts = new CancellationTokenSource();
|
||||
_ = Task.Run(() => RunEventLoopAsync(_eventLoopCts.Token));
|
||||
var loopToken = _eventLoopCts.Token; // capture NOW: the field may be replaced by a
|
||||
var loopClient = _client; // subsequent reconnect before Task.Run executes (N1)
|
||||
_ = Task.Run(() => RunEventLoopAsync(loopClient, loopToken));
|
||||
}
|
||||
|
||||
private async Task RunEventLoopAsync(CancellationToken ct)
|
||||
private async Task RunEventLoopAsync(IMxGatewayClient client, CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _client!.RunEventLoopAsync(update =>
|
||||
await client.RunEventLoopAsync(update =>
|
||||
{
|
||||
if (_tagToSub.TryGetValue(update.TagPath, out var subId) && _subs.TryGetValue(subId, out var s))
|
||||
s.Callback(update.TagPath, new TagValue(update.Value, update.Quality, update.Timestamp));
|
||||
@@ -122,6 +124,18 @@ public class MxGatewayDataConnection : IDataConnection, IBrowsableDataConnection
|
||||
{
|
||||
// Normal shutdown (DisconnectAsync / DisposeAsync cancelled the loop).
|
||||
}
|
||||
catch (Exception ex) when (ct.IsCancellationRequested)
|
||||
{
|
||||
// Teardown fault of a superseded loop (N1): Cancel() is not synchronous with
|
||||
// the loop's exit — a cancelled/disposed gRPC stream commonly surfaces as
|
||||
// RpcException(StatusCode.Cancelled) (Grpc.Net default without
|
||||
// ThrowOperationCanceledOnCancellation) or ObjectDisposedException from the
|
||||
// concurrent DisposeAsync, milliseconds AFTER ConnectAsync has already reset
|
||||
// _disconnectFired. Any fault on a cancelled token is normal shutdown;
|
||||
// raising Disconnected here would flap the NEW healthy session.
|
||||
_logger.LogDebug(ex,
|
||||
"MxGateway event loop faulted after cancellation — superseded loop teardown; not signalling disconnect");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "MxGateway event stream faulted; signalling disconnect");
|
||||
|
||||
@@ -4,7 +4,7 @@ namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer;
|
||||
|
||||
/// <summary>
|
||||
/// Validates <see cref="DataConnectionOptions"/> at startup (ValidateOnStart) so a
|
||||
/// malformed "DataConnectionLayer" appsettings section fails fast at boot with a
|
||||
/// malformed "ScadaBridge:DataConnection" appsettings section fails fast at boot with a
|
||||
/// key-naming message instead of surfacing later at runtime. The intervals drive
|
||||
/// the reconnect/tag-resolution timers and the seed-read retry loop; a
|
||||
/// zero/negative duration (or a non-positive <c>SeedReadMaxAttempts</c>) would
|
||||
@@ -16,24 +16,24 @@ public sealed class DataConnectionOptionsValidator : OptionsValidatorBase<DataCo
|
||||
protected override void Validate(ValidationBuilder builder, DataConnectionOptions options)
|
||||
{
|
||||
builder.RequireThat(options.ReconnectInterval > TimeSpan.Zero,
|
||||
$"DataConnectionLayer:ReconnectInterval must be a positive duration (was {options.ReconnectInterval}).");
|
||||
$"ScadaBridge:DataConnection:ReconnectInterval must be a positive duration (was {options.ReconnectInterval}).");
|
||||
|
||||
builder.RequireThat(options.TagResolutionRetryInterval > TimeSpan.Zero,
|
||||
$"DataConnectionLayer:TagResolutionRetryInterval must be a positive duration (was {options.TagResolutionRetryInterval}).");
|
||||
$"ScadaBridge:DataConnection:TagResolutionRetryInterval must be a positive duration (was {options.TagResolutionRetryInterval}).");
|
||||
|
||||
builder.RequireThat(options.WriteTimeout > TimeSpan.Zero,
|
||||
$"DataConnectionLayer:WriteTimeout must be a positive duration (was {options.WriteTimeout}).");
|
||||
$"ScadaBridge:DataConnection:WriteTimeout must be a positive duration (was {options.WriteTimeout}).");
|
||||
|
||||
builder.RequireThat(options.SeedReadTimeout > TimeSpan.Zero,
|
||||
$"DataConnectionLayer:SeedReadTimeout must be a positive duration (was {options.SeedReadTimeout}).");
|
||||
$"ScadaBridge:DataConnection:SeedReadTimeout must be a positive duration (was {options.SeedReadTimeout}).");
|
||||
|
||||
builder.RequireThat(options.StableConnectionThreshold > TimeSpan.Zero,
|
||||
$"DataConnectionLayer:StableConnectionThreshold must be a positive duration (was {options.StableConnectionThreshold}).");
|
||||
$"ScadaBridge:DataConnection:StableConnectionThreshold must be a positive duration (was {options.StableConnectionThreshold}).");
|
||||
|
||||
builder.RequireThat(options.SeedReadRetryDelay > TimeSpan.Zero,
|
||||
$"DataConnectionLayer:SeedReadRetryDelay must be a positive duration (was {options.SeedReadRetryDelay}).");
|
||||
$"ScadaBridge:DataConnection:SeedReadRetryDelay must be a positive duration (was {options.SeedReadRetryDelay}).");
|
||||
|
||||
builder.RequireThat(options.SeedReadMaxAttempts > 0,
|
||||
$"DataConnectionLayer:SeedReadMaxAttempts must be positive (was {options.SeedReadMaxAttempts}).");
|
||||
$"ScadaBridge:DataConnection:SeedReadMaxAttempts must be positive (was {options.SeedReadMaxAttempts}).");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer;
|
||||
|
||||
/// <summary>
|
||||
/// Deployment-wide MxGateway defaults, bound from the "MxGateway" section of
|
||||
/// appsettings.json. Per-endpoint behavior lives on MxGatewayEndpointConfig.
|
||||
/// Deployment-wide MxGateway defaults, bound from the "ScadaBridge:MxGateway" section
|
||||
/// of appsettings.json. Per-endpoint behavior lives on MxGatewayEndpointConfig.
|
||||
/// </summary>
|
||||
public class MxGatewayGlobalOptions
|
||||
{
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.DataConnectionLayer;
|
||||
|
||||
/// <summary>
|
||||
/// Deployment-wide OPC UA application identity. Bound from the "OpcUa" section
|
||||
/// of appsettings.json. Per-endpoint behavior lives on OpcUaEndpointConfig.
|
||||
/// Deployment-wide OPC UA application identity. Bound from the "ScadaBridge:OpcUa"
|
||||
/// section of appsettings.json. Per-endpoint behavior lives on OpcUaEndpointConfig.
|
||||
/// Empty paths fall back to a default under Path.GetTempPath() so dev runs
|
||||
/// work without explicit configuration.
|
||||
/// </summary>
|
||||
|
||||
@@ -14,20 +14,26 @@ public static class ServiceCollectionExtensions
|
||||
/// <returns>The same <paramref name="services"/> instance for chaining.</returns>
|
||||
public static IServiceCollection AddDataConnectionLayer(this IServiceCollection services)
|
||||
{
|
||||
// Canonical section is ScadaBridge:DataConnection — the section every real
|
||||
// appsettings actually writes (arch-review 08 round 2 NF8). The Host used to
|
||||
// supply the real values via a duplicate Configure<> binding; that duplicate is
|
||||
// gone, so AddDataConnectionLayer must bind the canonical section itself.
|
||||
services.AddOptions<DataConnectionOptions>()
|
||||
.BindConfiguration("DataConnectionLayer")
|
||||
.BindConfiguration("ScadaBridge:DataConnection")
|
||||
.ValidateOnStart();
|
||||
services.TryAddEnumerable(
|
||||
ServiceDescriptor.Singleton<IValidateOptions<DataConnectionOptions>, DataConnectionOptionsValidator>());
|
||||
|
||||
// OpcUaGlobalOptions / MxGatewayGlobalOptions carry only string identity/path
|
||||
// fields (no numeric or duration knobs), so there is nothing to range-validate
|
||||
// — no dedicated validator is registered for them.
|
||||
// — no dedicated validator is registered for them. Canonical sections are
|
||||
// ScadaBridge:OpcUa / ScadaBridge:MxGateway (NF8): the bare "OpcUa"/"MxGateway"
|
||||
// sections exist in no appsettings, so binding them read nothing.
|
||||
services.AddOptions<OpcUaGlobalOptions>()
|
||||
.BindConfiguration("OpcUa");
|
||||
.BindConfiguration("ScadaBridge:OpcUa");
|
||||
|
||||
services.AddOptions<MxGatewayGlobalOptions>()
|
||||
.BindConfiguration("MxGateway");
|
||||
.BindConfiguration("ScadaBridge:MxGateway");
|
||||
|
||||
// Register the factory for protocol extensibility
|
||||
services.AddSingleton<IDataConnectionFactory, DataConnectionFactory>();
|
||||
|
||||
@@ -166,7 +166,11 @@ public class ExternalSystemClient : IExternalSystemClient
|
||||
/// Delivers a buffered ExternalSystem call during a store-and-forward
|
||||
/// retry sweep. Returns true on success, false on permanent failure (the message
|
||||
/// is parked); throws <see cref="TransientExternalSystemException"/> on a
|
||||
/// transient failure so the engine retries.
|
||||
/// transient failure so the engine retries. Deterministic-poison failures against
|
||||
/// the CURRENT method definition — malformed JSON, or an
|
||||
/// <see cref="ArgumentException"/> from an unresolvable <c>{param}</c> path
|
||||
/// template or an unsupported HTTP verb — return false (park immediately) rather
|
||||
/// than retry, because re-running the same stored payload can never succeed.
|
||||
/// </summary>
|
||||
/// <param name="message">The buffered message to deliver.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
@@ -221,6 +225,21 @@ public class ExternalSystemClient : IExternalSystemClient
|
||||
_logger.LogError(ex, "Buffered call to '{System}' failed permanently; parking.", payload.SystemName);
|
||||
return false;
|
||||
}
|
||||
catch (ArgumentException ex)
|
||||
{
|
||||
// N2: same poison-message shape as the JsonException catch above. BuildUrl's
|
||||
// {param} path-template substitution (placeholder with no matching/non-null
|
||||
// parameter) and ValidateHttpMethod both throw ArgumentException
|
||||
// deterministically for the SAME stored payload — the method definition
|
||||
// changed underneath the buffered call, and retrying can never succeed.
|
||||
// Return false so the S&F engine parks the message instead of counting the
|
||||
// throw as transient and burning MaxRetries.
|
||||
_logger.LogError(
|
||||
ex,
|
||||
"Buffered call to '{System}'/'{Method}' fails deterministically against the current method definition; parking.",
|
||||
payload.SystemName, payload.MethodName);
|
||||
return false;
|
||||
}
|
||||
// TransientExternalSystemException propagates — the S&F engine retries.
|
||||
}
|
||||
|
||||
@@ -447,6 +466,12 @@ public class ExternalSystemClient : IExternalSystemClient
|
||||
/// into store-and-forward would defeat the cap. Cancellation
|
||||
/// (<see cref="OperationCanceledException"/>) is allowed to propagate so the
|
||||
/// caller's ordered timeout/cancellation catch filters classify it.
|
||||
/// <para>
|
||||
/// N4: the accumulated bytes are decoded using the response's declared charset
|
||||
/// (<c>Content-Type: …; charset=…</c>) via <see cref="ResolveEncoding"/> — parity
|
||||
/// with the pre-bounded-read <c>ReadAsStringAsync</c>. A null, empty, or
|
||||
/// unrecognized charset falls back to UTF-8.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
private static async Task<string> ReadBodyBoundedAsync(
|
||||
HttpContent content, string systemName, long maxBytes, CancellationToken token)
|
||||
@@ -479,7 +504,33 @@ public class ExternalSystemClient : IExternalSystemClient
|
||||
}
|
||||
}
|
||||
|
||||
return Encoding.UTF8.GetString(buffer.GetBuffer(), 0, (int)buffer.Length);
|
||||
// N4: honor the response's declared charset (parity with the pre-bounded-read
|
||||
// ReadAsStringAsync). Unknown/invalid charset → UTF-8 fallback: mojibake beats
|
||||
// failing the call over a cosmetic header.
|
||||
return ResolveEncoding(content.Headers.ContentType?.CharSet)
|
||||
.GetString(buffer.GetBuffer(), 0, (int)buffer.Length);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolves the declared response charset to an <see cref="Encoding"/>, tolerating
|
||||
/// the quoted form some servers emit (<c>charset="iso-8859-1"</c>). Null, empty,
|
||||
/// or unrecognized values fall back to UTF-8.
|
||||
/// </summary>
|
||||
private static Encoding ResolveEncoding(string? charset)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(charset))
|
||||
{
|
||||
return Encoding.UTF8;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return Encoding.GetEncoding(charset.Trim().Trim('"'));
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
return Encoding.UTF8;
|
||||
}
|
||||
}
|
||||
|
||||
private static string Truncate(string value, int maxChars)
|
||||
|
||||
@@ -47,6 +47,7 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
|
||||
LatestReport = report,
|
||||
LastReportReceivedAt = now,
|
||||
LastHeartbeatAt = now,
|
||||
FirstSeenAt = now,
|
||||
LastSequenceNumber = report.SequenceNumber,
|
||||
IsOnline = true,
|
||||
IsMetricsStale = false
|
||||
@@ -117,6 +118,7 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
|
||||
LatestReport = null,
|
||||
LastReportReceivedAt = null,
|
||||
LastHeartbeatAt = receivedAt,
|
||||
FirstSeenAt = receivedAt,
|
||||
LastSequenceNumber = 0,
|
||||
IsOnline = true
|
||||
};
|
||||
@@ -297,16 +299,22 @@ public class CentralHealthAggregator : BackgroundService, ICentralHealthAggregat
|
||||
// when no report has arrived within MetricsStaleTimeout. Distinct from
|
||||
// offline — a site whose HealthReportSender crashed keeps heartbeating
|
||||
// and would otherwise show "online with frozen metrics forever".
|
||||
if (state.LastReportReceivedAt is { } lastReport
|
||||
&& now - lastReport > _options.MetricsStaleTimeout
|
||||
// A site that has NEVER reported (heartbeat-only registration,
|
||||
// LastReportReceivedAt == null) anchors on FirstSeenAt instead —
|
||||
// otherwise a pipeline dead from first boot is never flagged (N3).
|
||||
var reportAnchor = state.LastReportReceivedAt ?? state.FirstSeenAt;
|
||||
if (reportAnchor is { } anchor
|
||||
&& now - anchor > _options.MetricsStaleTimeout
|
||||
&& !state.IsMetricsStale)
|
||||
{
|
||||
var stale = state with { IsMetricsStale = true };
|
||||
if (_siteStates.TryUpdate(kvp.Key, stale, state))
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Site {SiteId} metrics are stale — online (heartbeats) but no report for {Elapsed}s (timeout: {Timeout}s)",
|
||||
state.SiteId, (now - lastReport).TotalSeconds, _options.MetricsStaleTimeout.TotalSeconds);
|
||||
"Site {SiteId} metrics are stale — online (heartbeats) but no report for {Elapsed}s{NeverNote} (timeout: {Timeout}s)",
|
||||
state.SiteId, (now - anchor).TotalSeconds,
|
||||
state.LastReportReceivedAt is null ? " (never reported since first contact)" : string.Empty,
|
||||
_options.MetricsStaleTimeout.TotalSeconds);
|
||||
}
|
||||
// CAS loss ⇒ a fresh report/heartbeat swapped in ⇒ correct to skip.
|
||||
}
|
||||
|
||||
@@ -9,9 +9,11 @@ namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
/// Periodically builds a SiteHealthReport for the central cluster itself
|
||||
/// (siteId = <see cref="CentralSiteId"/>) and feeds it into the local
|
||||
/// CentralHealthAggregator so the UI can render central as another card
|
||||
/// on /monitoring/health. Only the cluster leader (Primary) generates
|
||||
/// reports — the standby's aggregator catches up on failover when it
|
||||
/// becomes Primary and starts its own loop.
|
||||
/// on /monitoring/health. Only the active node (Primary = the **oldest Up
|
||||
/// member**, i.e. the singleton host — <c>IClusterNodeProvider.SelfIsPrimary</c>,
|
||||
/// never Akka cluster leadership) generates reports; the standby's aggregator
|
||||
/// catches up on failover when it becomes the oldest and its loop starts
|
||||
/// reporting.
|
||||
/// </summary>
|
||||
public class CentralHealthReportLoop : BackgroundService
|
||||
{
|
||||
@@ -41,8 +43,9 @@ public class CentralHealthReportLoop : BackgroundService
|
||||
private readonly HealthMonitoringOptions _options;
|
||||
private readonly ILogger<CentralHealthReportLoop> _logger;
|
||||
|
||||
// Seeded with Unix-ms so reports from a newly-elected central leader
|
||||
// always sort after reports from any prior leader for siteId="central".
|
||||
// Seeded with Unix-ms so reports from a node that has newly become active
|
||||
// (oldest Up member) always sort after reports from any previously-active
|
||||
// node for siteId="central".
|
||||
// The clock is read through the injected TimeProvider so the seeding is
|
||||
// deterministically testable.
|
||||
private long _sequenceNumber;
|
||||
|
||||
@@ -40,7 +40,7 @@ public sealed class SiteHealthKpiSampleSource : IKpiSampleSource
|
||||
private const string MetricConnectionsUp = "connectionsUp";
|
||||
private const string MetricConnectionsDown = KpiMetrics.SiteHealth.ConnectionsDown;
|
||||
private const string MetricScriptErrors = KpiMetrics.SiteHealth.ScriptErrors;
|
||||
private const string MetricAlarmEvalErrors = "alarmEvalErrors";
|
||||
private const string MetricAlarmEvalErrors = KpiMetrics.SiteHealth.AlarmEvalErrors;
|
||||
private const string MetricSfBufferDepth = KpiMetrics.SiteHealth.SfBufferDepth;
|
||||
private const string MetricDeadLetters = KpiMetrics.SiteHealth.DeadLetters;
|
||||
private const string MetricParkedMessages = "parkedMessages";
|
||||
@@ -48,7 +48,7 @@ public sealed class SiteHealthKpiSampleSource : IKpiSampleSource
|
||||
private const string MetricEnabledInstances = "enabledInstances";
|
||||
private const string MetricDisabledInstances = "disabledInstances";
|
||||
private const string MetricAuditBacklogPending = "auditBacklogPending";
|
||||
private const string MetricEventLogWriteFailures = "eventLogWriteFailures";
|
||||
private const string MetricEventLogWriteFailures = KpiMetrics.SiteHealth.EventLogWriteFailures;
|
||||
|
||||
private readonly ICentralHealthAggregator _aggregator;
|
||||
|
||||
|
||||
@@ -40,6 +40,17 @@ public sealed record SiteHealthState
|
||||
/// </summary>
|
||||
public DateTimeOffset LastHeartbeatAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the instant this site was first observed by the aggregator (via a
|
||||
/// report or a heartbeat); <c>null</c> only for states not built by the
|
||||
/// aggregator (hand-constructed fixtures). Anchors metrics-staleness for a
|
||||
/// site that has NEVER delivered a report (review 01 round-2 N3): with
|
||||
/// <see cref="LastReportReceivedAt"/> null the staleness check previously
|
||||
/// skipped the site forever. <see cref="LastHeartbeatAt"/> cannot anchor
|
||||
/// this — every heartbeat advances it.
|
||||
/// </summary>
|
||||
public DateTimeOffset? FirstSeenAt { get; init; }
|
||||
|
||||
/// <summary>Gets the sequence number of the last accepted health report, used to reject out-of-order duplicates.</summary>
|
||||
public long LastSequenceNumber { get; init; }
|
||||
/// <summary>Gets a value indicating whether the site is currently considered online.</summary>
|
||||
|
||||
@@ -456,14 +456,14 @@ akka {{
|
||||
var outboxAuditWriter = _serviceProvider
|
||||
.GetRequiredService<ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services.ICentralAuditWriter>();
|
||||
|
||||
// All central singletons below are created through CentralSingletonRegistrar.Start,
|
||||
// All central singletons below are created through SingletonRegistrar.Start,
|
||||
// which pins the actor to the active central node (ClusterSingletonManager), exposes a
|
||||
// stable address (ClusterSingletonProxy), and — crucially — registers a PhaseClusterLeave
|
||||
// GracefulStop drain task so an in-flight EF write drains before handover (review 01
|
||||
// [Medium]: notification-outbox and audit-log-ingest previously had NO drain task). Names
|
||||
// are unchanged ({name}-singleton / {name}-proxy). The two SITE singletons stay
|
||||
// hand-rolled below because they are role-scoped (.WithRole(siteRole)).
|
||||
var outbox = CentralSingletonRegistrar.Start(
|
||||
// are unchanged ({name}-singleton / {name}-proxy). Site singletons use the same
|
||||
// registrar with `role: siteRole` (Task 11 / round-2 N5).
|
||||
var outbox = SingletonRegistrar.Start(
|
||||
_actorSystem!, "notification-outbox",
|
||||
Props.Create(() => new ZB.MOM.WW.ScadaBridge.NotificationOutbox.NotificationOutboxActor(
|
||||
_serviceProvider,
|
||||
@@ -496,7 +496,7 @@ akka {{
|
||||
var auditIngestLogger = _serviceProvider.GetRequiredService<ILoggerFactory>()
|
||||
.CreateLogger<ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogIngestActor>();
|
||||
|
||||
var auditIngest = CentralSingletonRegistrar.Start(
|
||||
var auditIngest = SingletonRegistrar.Start(
|
||||
_actorSystem!, "audit-log-ingest",
|
||||
Props.Create(() => new ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogIngestActor(
|
||||
_serviceProvider,
|
||||
@@ -561,7 +561,7 @@ akka {{
|
||||
var siteCallAuditOptions = _serviceProvider
|
||||
.GetRequiredService<IOptions<ZB.MOM.WW.ScadaBridge.SiteCallAudit.SiteCallAuditOptions>>().Value;
|
||||
|
||||
var siteCallAudit = CentralSingletonRegistrar.Start(
|
||||
var siteCallAudit = SingletonRegistrar.Start(
|
||||
_actorSystem!, "site-call-audit",
|
||||
Props.Create(() => new ZB.MOM.WW.ScadaBridge.SiteCallAudit.SiteCallAuditActor(
|
||||
_serviceProvider,
|
||||
@@ -606,7 +606,7 @@ akka {{
|
||||
var auditLogOptions = _serviceProvider
|
||||
.GetRequiredService<IOptions<ZB.MOM.WW.ScadaBridge.AuditLog.Configuration.AuditLogOptions>>();
|
||||
|
||||
CentralSingletonRegistrar.Start(
|
||||
SingletonRegistrar.Start(
|
||||
_actorSystem!, "audit-log-purge",
|
||||
Props.Create(() => new ZB.MOM.WW.ScadaBridge.AuditLog.Central.AuditLogPurgeActor(
|
||||
_serviceProvider,
|
||||
@@ -629,7 +629,7 @@ akka {{
|
||||
var auditReconClient = _serviceProvider
|
||||
.GetRequiredService<ZB.MOM.WW.ScadaBridge.AuditLog.Central.IPullAuditEventsClient>();
|
||||
|
||||
CentralSingletonRegistrar.Start(
|
||||
SingletonRegistrar.Start(
|
||||
_actorSystem!, "site-audit-reconciliation",
|
||||
Props.Create(() => new ZB.MOM.WW.ScadaBridge.AuditLog.Central.SiteAuditReconciliationActor(
|
||||
auditReconSites,
|
||||
@@ -660,7 +660,7 @@ akka {{
|
||||
var kpiHistoryLogger = _serviceProvider.GetRequiredService<ILoggerFactory>()
|
||||
.CreateLogger<ZB.MOM.WW.ScadaBridge.KpiHistory.KpiHistoryRecorderActor>();
|
||||
|
||||
CentralSingletonRegistrar.Start(
|
||||
SingletonRegistrar.Start(
|
||||
_actorSystem!, "kpi-history-recorder",
|
||||
Props.Create(() => new ZB.MOM.WW.ScadaBridge.KpiHistory.KpiHistoryRecorderActor(
|
||||
_serviceProvider,
|
||||
@@ -686,7 +686,7 @@ akka {{
|
||||
var pendingPurgeCommunicationOptions =
|
||||
_serviceProvider.GetRequiredService<IOptions<CommunicationOptions>>();
|
||||
|
||||
CentralSingletonRegistrar.Start(
|
||||
SingletonRegistrar.Start(
|
||||
_actorSystem!, "pending-deployment-purge",
|
||||
Props.Create(() => new PendingDeploymentPurgeActor(
|
||||
_serviceProvider,
|
||||
@@ -772,10 +772,20 @@ akka {{
|
||||
var replicationLogger = _serviceProvider.GetRequiredService<ILoggerFactory>()
|
||||
.CreateLogger<SiteReplicationActor>();
|
||||
|
||||
// ONE active-node predicate instance governs the S&F delivery gate, the resync
|
||||
// authority checks (SiteReplicationActor), and the heartbeat IsActive stamp
|
||||
// (SiteCommunicationActor, wired below) — review 02 round 2, N1. Null in
|
||||
// non-clustered test hosts: the actors fall back to the shared oldest-Up
|
||||
// evaluator, never to a leader check.
|
||||
var clusterNodeProvider = _serviceProvider.GetService<ZB.MOM.WW.ScadaBridge.HealthMonitoring.IClusterNodeProvider>();
|
||||
Func<bool>? activeNodeCheck = clusterNodeProvider != null
|
||||
? () => clusterNodeProvider.SelfIsPrimary
|
||||
: null;
|
||||
|
||||
var replicationActor = _actorSystem!.ActorOf(
|
||||
Props.Create(() => new SiteReplicationActor(
|
||||
storage, sfStorage, replicationService, siteRole, replicationLogger,
|
||||
deploymentConfigFetcher, null, siteRuntimeOptionsValue, null)),
|
||||
deploymentConfigFetcher, activeNodeCheck, siteRuntimeOptionsValue, null)),
|
||||
"site-replication");
|
||||
|
||||
// Wire S&F replication handler to forward operations via the replication actor
|
||||
@@ -787,43 +797,28 @@ akka {{
|
||||
|
||||
_logger.LogInformation("SiteReplicationActor created and S&F replication handler wired");
|
||||
|
||||
// Create the Deployment Manager as a cluster singleton
|
||||
var singletonProps = ClusterSingletonManager.Props(
|
||||
singletonProps: Props.Create(() => new DeploymentManagerActor(
|
||||
storage,
|
||||
compilationService,
|
||||
sharedScriptLibrary,
|
||||
streamManager,
|
||||
siteRuntimeOptionsValue,
|
||||
dmLogger,
|
||||
dclManager,
|
||||
replicationActor,
|
||||
siteHealthCollector,
|
||||
_serviceProvider,
|
||||
null,
|
||||
deploymentConfigFetcher)),
|
||||
terminationMessage: PoisonPill.Instance,
|
||||
settings: ClusterSingletonManagerSettings.Create(_actorSystem!)
|
||||
.WithRole(siteRole)
|
||||
.WithSingletonName("deployment-manager"));
|
||||
|
||||
_actorSystem!.ActorOf(singletonProps, "deployment-manager-singleton");
|
||||
|
||||
// Create a proxy for other actors to communicate with the singleton
|
||||
var proxyProps = ClusterSingletonProxy.Props(
|
||||
singletonManagerPath: "/user/deployment-manager-singleton",
|
||||
settings: ClusterSingletonProxySettings.Create(_actorSystem)
|
||||
.WithRole(siteRole)
|
||||
.WithSingletonName("deployment-manager"));
|
||||
|
||||
var dmProxy = _actorSystem.ActorOf(proxyProps, "deployment-manager-proxy");
|
||||
// Deployment Manager — role-scoped singleton via SingletonRegistrar
|
||||
// (review 01 round-2 N5): previously hand-rolled with bare PoisonPill
|
||||
// termination and NO PhaseClusterLeave drain, so in-flight SQLite
|
||||
// writes (static overrides, native_alarm_state) could be cut off on
|
||||
// graceful failover. Names unchanged: deployment-manager-singleton /
|
||||
// deployment-manager-proxy.
|
||||
var dm = SingletonRegistrar.Start(
|
||||
_actorSystem!, "deployment-manager",
|
||||
Props.Create(() => new DeploymentManagerActor(
|
||||
storage, compilationService, sharedScriptLibrary, streamManager,
|
||||
siteRuntimeOptionsValue, dmLogger, dclManager, replicationActor,
|
||||
siteHealthCollector, _serviceProvider, null, deploymentConfigFetcher)),
|
||||
_logger, role: siteRole);
|
||||
var dmProxy = dm.Proxy;
|
||||
|
||||
// Create SiteCommunicationActor for receiving messages from central
|
||||
var siteCommActor = _actorSystem.ActorOf(
|
||||
Props.Create(() => new SiteCommunicationActor(
|
||||
_nodeOptions.SiteId!,
|
||||
_communicationOptions,
|
||||
dmProxy)),
|
||||
dmProxy,
|
||||
activeNodeCheck)),
|
||||
"site-communication");
|
||||
|
||||
// Register local handlers with SiteCommunicationActor
|
||||
@@ -836,22 +831,11 @@ akka {{
|
||||
var eventLogQueryService = _serviceProvider.GetService<SiteEventLogging.IEventLogQueryService>();
|
||||
if (eventLogQueryService != null)
|
||||
{
|
||||
var eventLogSingletonProps = ClusterSingletonManager.Props(
|
||||
singletonProps: Props.Create(() => new SiteEventLogging.EventLogHandlerActor(eventLogQueryService)),
|
||||
terminationMessage: PoisonPill.Instance,
|
||||
settings: ClusterSingletonManagerSettings.Create(_actorSystem)
|
||||
.WithRole(siteRole)
|
||||
.WithSingletonName("event-log-handler"));
|
||||
_actorSystem.ActorOf(eventLogSingletonProps, "event-log-handler-singleton");
|
||||
|
||||
var eventLogProxyProps = ClusterSingletonProxy.Props(
|
||||
singletonManagerPath: "/user/event-log-handler-singleton",
|
||||
settings: ClusterSingletonProxySettings.Create(_actorSystem)
|
||||
.WithRole(siteRole)
|
||||
.WithSingletonName("event-log-handler"));
|
||||
var eventLogProxy = _actorSystem.ActorOf(eventLogProxyProps, "event-log-handler-proxy");
|
||||
|
||||
siteCommActor.Tell(new RegisterLocalHandler(LocalHandlerType.EventLog, eventLogProxy));
|
||||
var eventLog = SingletonRegistrar.Start(
|
||||
_actorSystem, "event-log-handler",
|
||||
Props.Create(() => new SiteEventLogging.EventLogHandlerActor(eventLogQueryService)),
|
||||
_logger, role: siteRole);
|
||||
siteCommActor.Tell(new RegisterLocalHandler(LocalHandlerType.EventLog, eventLog.Proxy));
|
||||
}
|
||||
|
||||
// Parked message handler — bridges Akka to StoreAndForwardService
|
||||
@@ -872,10 +856,11 @@ akka {{
|
||||
// tick so failover resumes delivery within one RetryTimerInterval.
|
||||
// IClusterNodeProvider.SelfIsPrimary is the canonical "this node is the
|
||||
// oldest Up member (singleton host)" check from the cluster-infrastructure
|
||||
// fix plan — the shared helper this seam was designed to accept. In a
|
||||
// non-clustered test host the provider is unregistered, so the gate stays
|
||||
// unset and the sweep is ungated (legacy behaviour, preserved).
|
||||
var clusterNodeProvider = _serviceProvider.GetService<ZB.MOM.WW.ScadaBridge.HealthMonitoring.IClusterNodeProvider>();
|
||||
// fix plan — the shared helper this seam was designed to accept. The provider
|
||||
// is resolved once above (the same instance gating the resync authority and
|
||||
// heartbeat IsActive stamp — N1). In a non-clustered test host the provider is
|
||||
// unregistered, so the gate stays unset and the sweep is ungated (legacy
|
||||
// behaviour, preserved).
|
||||
if (clusterNodeProvider != null)
|
||||
{
|
||||
storeAndForwardService.SetDeliveryGate(() => clusterNodeProvider.SelfIsPrimary);
|
||||
|
||||
+22
-11
@@ -5,21 +5,24 @@ using Microsoft.Extensions.Logging;
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host.Actors;
|
||||
|
||||
/// <summary>
|
||||
/// Registers a central cluster singleton with the canonical naming scheme
|
||||
/// Registers a cluster singleton with the canonical naming scheme
|
||||
/// (<c>{name}-singleton</c> / <c>{name}-proxy</c>), a PoisonPill termination
|
||||
/// message, and a PhaseClusterLeave drain task that GracefulStops the manager
|
||||
/// so in-flight EF work completes before handover. Extracted from five
|
||||
/// copy-pasted ~60-line blocks (review 01 [Low]) whose drift left the two
|
||||
/// busiest singletons (notification-outbox, audit-log-ingest) without drain
|
||||
/// tasks (review 01 [Medium]).
|
||||
/// so in-flight EF (central) or SQLite (site) work completes before handover.
|
||||
/// Extracted from copy-pasted ~60-line blocks (review 01 [Low]) whose drift
|
||||
/// left the two busiest singletons (notification-outbox, audit-log-ingest)
|
||||
/// without drain tasks (review 01 [Medium]). The optional <c>role</c> maps to
|
||||
/// <see cref="ClusterSingletonManagerSettings"/>/<see cref="ClusterSingletonProxySettings"/>
|
||||
/// <c>.WithRole</c>, so role-scoped site singletons get the same drain
|
||||
/// (review 01 round-2 N5).
|
||||
/// </summary>
|
||||
internal static class CentralSingletonRegistrar
|
||||
internal static class SingletonRegistrar
|
||||
{
|
||||
internal sealed record Handle(IActorRef Manager, IActorRef Proxy);
|
||||
|
||||
/// <summary>
|
||||
/// Creates the <see cref="ClusterSingletonManager"/> and proxy for a central
|
||||
/// cluster singleton under the canonical naming scheme, and registers a
|
||||
/// Creates the <see cref="ClusterSingletonManager"/> and proxy for a cluster
|
||||
/// singleton under the canonical naming scheme, and registers a
|
||||
/// PhaseClusterLeave drain task that GracefulStops the manager on shutdown.
|
||||
/// </summary>
|
||||
/// <param name="system">The actor system to register the singleton on.</param>
|
||||
@@ -27,19 +30,24 @@ internal static class CentralSingletonRegistrar
|
||||
/// <param name="singletonProps">The <see cref="Props"/> used to create the singleton's managed actor.</param>
|
||||
/// <param name="logger">Logger used to report drain progress/failures.</param>
|
||||
/// <param name="drainTimeout">Maximum time to wait for the drain task to complete; defaults to 10 seconds.</param>
|
||||
/// <param name="role">Optional cluster role to scope the singleton to (applied to both the manager and proxy settings).</param>
|
||||
/// <returns>A <see cref="Handle"/> carrying the singleton manager and proxy actor refs.</returns>
|
||||
internal static Handle Start(
|
||||
ActorSystem system,
|
||||
string name,
|
||||
Props singletonProps,
|
||||
ILogger logger,
|
||||
TimeSpan? drainTimeout = null)
|
||||
TimeSpan? drainTimeout = null,
|
||||
string? role = null)
|
||||
{
|
||||
var managerSettings = ClusterSingletonManagerSettings.Create(system).WithSingletonName(name);
|
||||
if (role is not null) managerSettings = managerSettings.WithRole(role);
|
||||
|
||||
var manager = system.ActorOf(
|
||||
ClusterSingletonManager.Props(
|
||||
singletonProps,
|
||||
PoisonPill.Instance,
|
||||
ClusterSingletonManagerSettings.Create(system).WithSingletonName(name)),
|
||||
managerSettings),
|
||||
$"{name}-singleton");
|
||||
|
||||
var timeout = drainTimeout ?? TimeSpan.FromSeconds(10);
|
||||
@@ -61,10 +69,13 @@ internal static class CentralSingletonRegistrar
|
||||
return Akka.Done.Instance;
|
||||
});
|
||||
|
||||
var proxySettings = ClusterSingletonProxySettings.Create(system).WithSingletonName(name);
|
||||
if (role is not null) proxySettings = proxySettings.WithRole(role);
|
||||
|
||||
var proxy = system.ActorOf(
|
||||
ClusterSingletonProxy.Props(
|
||||
$"/user/{name}-singleton",
|
||||
ClusterSingletonProxySettings.Create(system).WithSingletonName(name)),
|
||||
proxySettings),
|
||||
$"{name}-proxy");
|
||||
|
||||
return new Handle(manager, proxy);
|
||||
@@ -0,0 +1,28 @@
|
||||
using ZB.MOM.WW.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host;
|
||||
|
||||
/// <summary>
|
||||
/// Validates <see cref="DatabaseOptions"/> at host startup (arch-review 08
|
||||
/// round 2 NF4). All three connection settings are nullable and role-dependent
|
||||
/// (central needs the SQL Server strings, a site needs only the SQLite path),
|
||||
/// so a <c>null</c> value is valid. A present-but-whitespace value is not — it
|
||||
/// signals a mis-set config key that would fail opaquely at first DB use.
|
||||
/// </summary>
|
||||
public sealed class DatabaseOptionsValidator : OptionsValidatorBase<DatabaseOptions>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Validate(ValidationBuilder builder, DatabaseOptions options)
|
||||
{
|
||||
RequireNotWhitespace(builder, options.ConfigurationDb, nameof(DatabaseOptions.ConfigurationDb));
|
||||
RequireNotWhitespace(builder, options.MachineDataDb, nameof(DatabaseOptions.MachineDataDb));
|
||||
RequireNotWhitespace(builder, options.SiteDbPath, nameof(DatabaseOptions.SiteDbPath));
|
||||
}
|
||||
|
||||
// null is valid (role-dependent); reject only a present-but-blank value.
|
||||
private static void RequireNotWhitespace(ValidationBuilder builder, string? value, string field)
|
||||
{
|
||||
builder.RequireThat(value is null || !string.IsNullOrWhiteSpace(value),
|
||||
$"ScadaBridge:Database:{field} is set but blank — either remove it or give it a real value.");
|
||||
}
|
||||
}
|
||||
@@ -13,23 +13,15 @@ namespace ZB.MOM.WW.ScadaBridge.Host.Health;
|
||||
/// </summary>
|
||||
public static class ClusterActivityEvaluator
|
||||
{
|
||||
/// <summary>True when self is Up and no other Up member (in the role scope) is older.</summary>
|
||||
/// <summary>True when self is Up and no other Up member (in the role scope) is older.
|
||||
/// Delegates to the shared <see cref="Communication.ClusterState.ActiveNodeEvaluator"/> —
|
||||
/// one implementation for the delivery gate, the resync authority checks, and the
|
||||
/// heartbeat IsActive stamp (review 02 round 2, N1).</summary>
|
||||
/// <param name="cluster">The Akka cluster to evaluate.</param>
|
||||
/// <param name="role">Optional role scope; when set, only members with this role are considered.</param>
|
||||
/// <returns><c>true</c> when self is Up and the oldest Up member in the role scope.</returns>
|
||||
public static bool SelfIsOldest(Cluster cluster, string? role = null)
|
||||
{
|
||||
var self = cluster.SelfMember;
|
||||
if (self.Status != MemberStatus.Up)
|
||||
return false;
|
||||
if (role != null && !self.HasRole(role))
|
||||
return false;
|
||||
|
||||
return cluster.State.Members
|
||||
.Where(m => m.Status == MemberStatus.Up)
|
||||
.Where(m => role == null || m.HasRole(role))
|
||||
.All(m => m.UniqueAddress.Equals(self.UniqueAddress) || self.IsOlderThan(m));
|
||||
}
|
||||
public static bool SelfIsOldest(Cluster cluster, string? role = null) =>
|
||||
Communication.ClusterState.ActiveNodeEvaluator.SelfIsOldestUp(cluster, role);
|
||||
|
||||
/// <summary>The oldest Up member in the role scope, or null while none is Up. Used for Primary/Standby labelling.</summary>
|
||||
/// <param name="cluster">The Akka cluster to evaluate.</param>
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
using Serilog.Events;
|
||||
using ZB.MOM.WW.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host;
|
||||
|
||||
/// <summary>
|
||||
/// Validates <see cref="LoggingOptions"/> at host startup (arch-review 08 round
|
||||
/// 2 NF4). <see cref="LoggingOptions.MinimumLevel"/> is parsed into a Serilog
|
||||
/// <see cref="LogEventLevel"/> by <c>LoggerConfigurationFactory.ParseLevel</c>,
|
||||
/// which silently falls back to <c>Information</c> for an unrecognised value.
|
||||
/// This validator surfaces that misconfiguration at boot instead: a present,
|
||||
/// non-blank value must name a real level (Verbose/Debug/Information/Warning/
|
||||
/// Error/Fatal, case-insensitive). A null/blank value is treated as "unset"
|
||||
/// (defaults to Information), mirroring the parser.
|
||||
/// </summary>
|
||||
public sealed class LoggingOptionsValidator : OptionsValidatorBase<LoggingOptions>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Validate(ValidationBuilder builder, LoggingOptions options)
|
||||
{
|
||||
builder.RequireThat(
|
||||
string.IsNullOrWhiteSpace(options.MinimumLevel)
|
||||
|| Enum.TryParse<LogEventLevel>(options.MinimumLevel, ignoreCase: true, out _),
|
||||
$"ScadaBridge:Logging:{nameof(LoggingOptions.MinimumLevel)} ('{options.MinimumLevel}') " +
|
||||
$"is not a recognised Serilog level. Valid: {string.Join(", ", Enum.GetNames<LogEventLevel>())}.");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
using ZB.MOM.WW.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Host;
|
||||
|
||||
/// <summary>
|
||||
/// Validates <see cref="NodeOptions"/> at host startup (arch-review 08 round 2
|
||||
/// NF4). The headline invariant: <see cref="NodeOptions.NodeName"/> must be
|
||||
/// non-empty — an empty value normalises to a NULL <c>SourceNode</c> audit
|
||||
/// column, silently degrading the audit trail (the wonder-app-vd03 failure
|
||||
/// mode), so the host must fail fast at boot instead. Ports are bounded to the
|
||||
/// TCP range; <c>0</c> stays valid because it requests a dynamically-assigned
|
||||
/// port (CompositionRootTests rely on this).
|
||||
/// </summary>
|
||||
public sealed class NodeOptionsValidator : OptionsValidatorBase<NodeOptions>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Validate(ValidationBuilder builder, NodeOptions options)
|
||||
{
|
||||
builder.RequireThat(!string.IsNullOrWhiteSpace(options.NodeName),
|
||||
$"ScadaBridge:Node:{nameof(NodeOptions.NodeName)} must be a non-empty label " +
|
||||
"— an empty node name stamps the SourceNode audit column NULL.");
|
||||
|
||||
RequirePort(builder, options.RemotingPort, nameof(NodeOptions.RemotingPort));
|
||||
RequirePort(builder, options.GrpcPort, nameof(NodeOptions.GrpcPort));
|
||||
RequirePort(builder, options.MetricsPort, nameof(NodeOptions.MetricsPort));
|
||||
}
|
||||
|
||||
// 0 stays valid (dynamic-port request); reject only out-of-TCP-range values.
|
||||
private static void RequirePort(ValidationBuilder builder, int port, string field)
|
||||
{
|
||||
builder.RequireThat(port is >= 0 and <= 65_535,
|
||||
$"ScadaBridge:Node:{field} ({port}) must be in [0, 65535].");
|
||||
}
|
||||
}
|
||||
@@ -94,8 +94,12 @@ try
|
||||
builder.Services.AddTransport();
|
||||
// Script-artifact invalidation bus: in-process, per-node pub/sub
|
||||
// for ScriptArtifactsChanged. The Transport bundle importer publishes
|
||||
// post-commit; the Inbound API compiled-handler cache subscribes as the
|
||||
// consumer (wired in plan 06). Central-only — it lives beside the importer.
|
||||
// post-commit; the Inbound API's ScriptArtifactChangeSubscriber (a hosted
|
||||
// service registered by AddInboundAPI below) consumes ApiMethod notifications
|
||||
// and invalidates the compiled-handler cache (wired in plan R2-06 T1; the
|
||||
// per-request revision check remains the correctness fallback).
|
||||
// SharedScript/Template notifications currently have NO consumer — nothing at
|
||||
// central caches compiled artifacts of those kinds. Central-only.
|
||||
builder.Services.AddSingleton<
|
||||
ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Scripting.IScriptArtifactChangeBus,
|
||||
ZB.MOM.WW.ScadaBridge.Host.Services.InProcessScriptArtifactChangeBus>();
|
||||
@@ -337,6 +341,18 @@ try
|
||||
}
|
||||
|
||||
// Middleware pipeline
|
||||
|
||||
// Trusted-proxy forwarded headers (arch-review R2 N2): behind Traefik every client
|
||||
// shares the proxy's IP, collapsing the LoginThrottle's {username}|{ip} keys onto
|
||||
// one address (per-IP isolation gone + cheap username-lockout DoS). When enabled,
|
||||
// X-Forwarded-For from the CONFIGURED proxies only is honored. MUST run first —
|
||||
// everything downstream keys on Connection.RemoteIpAddress.
|
||||
var forwardedOptions = builder.Configuration
|
||||
.GetSection(ZB.MOM.WW.ScadaBridge.Security.ForwardedHeadersSecurityOptions.SectionName)
|
||||
.Get<ZB.MOM.WW.ScadaBridge.Security.ForwardedHeadersSecurityOptions>() ?? new();
|
||||
if (ZB.MOM.WW.ScadaBridge.Security.ForwardedHeadersSetup.Build(forwardedOptions) is { } fho)
|
||||
app.UseForwardedHeaders(fho);
|
||||
|
||||
app.UseWebSockets();
|
||||
app.UseRouting();
|
||||
app.UseAuthentication();
|
||||
|
||||
@@ -137,7 +137,21 @@ public static class SiteServiceRegistration
|
||||
services.TryAddEnumerable(
|
||||
ServiceDescriptor.Singleton<IValidateOptions<SiteRuntimeOptions>, SiteRuntimeOptionsValidator>());
|
||||
|
||||
services.Configure<DataConnectionOptions>(config.GetSection("ScadaBridge:DataConnection"));
|
||||
// OperationTrackingOptions for the site-local cached-call tracking store
|
||||
// (registered by AddSiteRuntime). Bind + eagerly validate here — the store
|
||||
// opens its SQLite connection string at construction, so a blank connection
|
||||
// string should fail the host at boot with a key-naming message rather than
|
||||
// throw opaquely on first resolve (arch-review 08r2 NF4/T8).
|
||||
services.AddOptions<SiteRuntime.Tracking.OperationTrackingOptions>()
|
||||
.Bind(config.GetSection("ScadaBridge:OperationTracking"))
|
||||
.ValidateOnStart();
|
||||
services.TryAddEnumerable(
|
||||
ServiceDescriptor.Singleton<IValidateOptions<SiteRuntime.Tracking.OperationTrackingOptions>,
|
||||
SiteRuntime.Tracking.OperationTrackingOptionsValidator>());
|
||||
|
||||
// NF8: DataConnectionOptions is now bound by AddDataConnectionLayer() itself
|
||||
// (canonical ScadaBridge:DataConnection section) — the duplicate Host binding
|
||||
// that used to mask the SCE's wrong section name is deleted.
|
||||
|
||||
services.AddOptions<StoreAndForwardOptions>()
|
||||
.Bind(config.GetSection("ScadaBridge:StoreAndForward"))
|
||||
@@ -157,19 +171,38 @@ public static class SiteServiceRegistration
|
||||
/// <param name="config">Application configuration supplying the option values.</param>
|
||||
public static void BindSharedOptions(IServiceCollection services, IConfiguration config)
|
||||
{
|
||||
services.Configure<NodeOptions>(config.GetSection("ScadaBridge:Node"));
|
||||
// Bind + eagerly validate: an empty NodeName would stamp the SourceNode audit
|
||||
// column NULL, so fail the host at boot with a key-naming message instead of
|
||||
// silently degrading the audit trail (arch-review 08r2 NF4).
|
||||
services.AddOptions<NodeOptions>().Bind(config.GetSection("ScadaBridge:Node")).ValidateOnStart();
|
||||
services.TryAddEnumerable(
|
||||
ServiceDescriptor.Singleton<IValidateOptions<NodeOptions>, NodeOptionsValidator>());
|
||||
// Bind + eagerly validate: ClusterOptionsValidator is registered (TryAddEnumerable)
|
||||
// by the ClusterInfrastructure module, so chaining ValidateOnStart() here makes a bad
|
||||
// ScadaBridge:Cluster section fail fast at host build instead of lazily on first resolve.
|
||||
services.AddOptions<ClusterOptions>().Bind(config.GetSection("ScadaBridge:Cluster")).ValidateOnStart();
|
||||
services.Configure<DatabaseOptions>(config.GetSection("ScadaBridge:Database"));
|
||||
services.Configure<CommunicationOptions>(config.GetSection("ScadaBridge:Communication"));
|
||||
// Bind + eagerly validate: a present-but-blank connection setting fails fast
|
||||
// here rather than opaquely at first DB use (arch-review 08r2 NF4).
|
||||
services.AddOptions<DatabaseOptions>().Bind(config.GetSection("ScadaBridge:Database")).ValidateOnStart();
|
||||
services.TryAddEnumerable(
|
||||
ServiceDescriptor.Singleton<IValidateOptions<DatabaseOptions>, DatabaseOptionsValidator>());
|
||||
// NF8: CommunicationOptions is now bound by AddCommunication() itself (canonical
|
||||
// ScadaBridge:Communication section) — the duplicate Host binding that used to mask
|
||||
// the SCE's wrong section name is deleted.
|
||||
// Bind + eagerly validate: HealthMonitoringOptionsValidator is registered (TryAddEnumerable)
|
||||
// by the HealthMonitoring module, so chaining ValidateOnStart() here makes a bad
|
||||
// ScadaBridge:HealthMonitoring section fail fast at host build instead of lazily on first resolve.
|
||||
services.AddOptions<HealthMonitoringOptions>().Bind(config.GetSection("ScadaBridge:HealthMonitoring")).ValidateOnStart();
|
||||
services.Configure<NotificationOptions>(config.GetSection("ScadaBridge:Notification"));
|
||||
services.Configure<LoggingOptions>(config.GetSection("ScadaBridge:Logging"));
|
||||
// NF8: NotificationOptions is bound by AddNotificationService() (central-only,
|
||||
// canonical ScadaBridge:Notification section). Sites don't deliver notifications
|
||||
// and have no IOptions<NotificationOptions> consumer, so the redundant Host
|
||||
// duplicate that bound it on both paths is deleted.
|
||||
// Bind + eagerly validate: an unrecognised MinimumLevel would silently fall back
|
||||
// to Information; fail fast at boot so the operator's intended floor is honoured
|
||||
// (arch-review 08r2 NF4).
|
||||
services.AddOptions<LoggingOptions>().Bind(config.GetSection("ScadaBridge:Logging")).ValidateOnStart();
|
||||
services.TryAddEnumerable(
|
||||
ServiceDescriptor.Singleton<IValidateOptions<LoggingOptions>, LoggingOptionsValidator>());
|
||||
|
||||
// Audit Log — exposes ScadaBridge:Node:NodeName to downstream audit
|
||||
// writers so they can stamp the SourceNode column. Registered here in
|
||||
|
||||
@@ -179,12 +179,22 @@ public static class EndpointExtensions
|
||||
httpContext.Items[AuditWriteMiddleware.AuditActorItemKey] =
|
||||
identity.DisplayName;
|
||||
|
||||
// N5: a body is present when Content-Length says so, OR when the request is
|
||||
// chunked — Transfer-Encoding present and Content-Length (necessarily) absent.
|
||||
// Computed once and used by BOTH the 415 guard and the JSON parse condition so
|
||||
// chunked and fixed-length bodies behave identically.
|
||||
var requestHeaders = httpContext.Request.Headers;
|
||||
var hasBody = httpContext.Request.ContentLength > 0
|
||||
|| (httpContext.Request.ContentLength is null && requestHeaders.TransferEncoding.Count > 0);
|
||||
|
||||
// S9: a body sent with an explicit non-JSON Content-Type is a media-type
|
||||
// mismatch, not malformed JSON — reject it with 415 rather than a misleading
|
||||
// 400 "invalid JSON". A body with NO Content-Type header is still parsed
|
||||
// leniently as JSON below (preserving existing callers). The sniff is
|
||||
// ordinal-ignore-case so "application/JSON" etc. are treated as JSON.
|
||||
if (httpContext.Request.ContentLength > 0
|
||||
// ordinal-ignore-case so "application/JSON" etc. are treated as JSON. N5: the
|
||||
// body signal is `hasBody`, so a chunked (no Content-Length) non-JSON body is
|
||||
// also caught here instead of slipping through to a misleading 400.
|
||||
if (hasBody
|
||||
&& !string.IsNullOrEmpty(httpContext.Request.ContentType)
|
||||
&& httpContext.Request.ContentType.Contains("json", StringComparison.OrdinalIgnoreCase) == false)
|
||||
{
|
||||
@@ -203,7 +213,7 @@ public static class EndpointExtensions
|
||||
// `Contains("json")` silently skipped JSON deserialization for any
|
||||
// capitalised value, leaving `body = null` and surfacing required
|
||||
// parameters as 400 "missing" even though the caller sent a valid body.
|
||||
if (httpContext.Request.ContentLength > 0
|
||||
if (hasBody
|
||||
|| httpContext.Request.ContentType?.Contains("json", StringComparison.OrdinalIgnoreCase) == true)
|
||||
{
|
||||
using var doc = await JsonDocument.ParseAsync(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using Akka.Actor;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Services;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.InboundApi;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
@@ -5,12 +6,16 @@ using ZB.MOM.WW.ScadaBridge.Commons.Types;
|
||||
namespace ZB.MOM.WW.ScadaBridge.InboundAPI;
|
||||
|
||||
/// <summary>
|
||||
/// C3: a routed call failed because the target site could not be reached (no contact /
|
||||
/// unreachable). Derives from <see cref="InvalidOperationException"/> so existing
|
||||
/// callers that catch the base type are unaffected, while
|
||||
/// <see cref="InboundScriptExecutor"/> catches this more specific type to surface the
|
||||
/// spec's <c>SITE_UNREACHABLE</c> error code instead of a generic <c>SCRIPT_ERROR</c>.
|
||||
/// A routing failure whose message does <em>not</em> indicate unreachability stays a
|
||||
/// C3/N3: a routed call failed because the target site never answered — the routed
|
||||
/// Ask <b>expired</b>. Unreachability is typed at the Ask boundary from
|
||||
/// <see cref="AskTimeoutException"/> (the communication layer's own signal: an
|
||||
/// unreachable/contactless site has its enveloped message dropped so the Ask times
|
||||
/// out caller-side) — it is <em>never</em> inferred from an error-message substring.
|
||||
/// Derives from <see cref="InvalidOperationException"/> so existing callers that
|
||||
/// catch the base type are unaffected, while <see cref="InboundScriptExecutor"/>
|
||||
/// catches this more specific type to surface the spec's <c>SITE_UNREACHABLE</c>
|
||||
/// error code instead of a generic <c>SCRIPT_ERROR</c>. A <c>Success=false</c>
|
||||
/// <em>response</em> means the site answered (reachable by definition) and stays a
|
||||
/// plain <see cref="InvalidOperationException"/>.
|
||||
/// </summary>
|
||||
public sealed class SiteUnreachableException : InvalidOperationException
|
||||
@@ -18,6 +23,11 @@ public sealed class SiteUnreachableException : InvalidOperationException
|
||||
/// <summary>Initializes a new <see cref="SiteUnreachableException"/> with the given message.</summary>
|
||||
/// <param name="message">The routing-level failure message that indicated the site was unreachable.</param>
|
||||
public SiteUnreachableException(string message) : base(message) { }
|
||||
|
||||
/// <summary>Initializes a new <see cref="SiteUnreachableException"/> wrapping the underlying Ask-timeout.</summary>
|
||||
/// <param name="message">The routing-level failure message.</param>
|
||||
/// <param name="innerException">The underlying <see cref="AskTimeoutException"/> from the routed Ask.</param>
|
||||
public SiteUnreachableException(string message, Exception innerException) : base(message, innerException) { }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -182,7 +192,7 @@ public class RouteTarget
|
||||
correlationId, _instanceCode, scriptName, ScriptArgs.Normalize(parameters),
|
||||
DateTimeOffset.UtcNow, _parentExecutionId);
|
||||
|
||||
var response = await _instanceRouter.RouteToCallAsync(siteId, request, token);
|
||||
var response = await RouteOrUnreachable(_instanceRouter.RouteToCallAsync(siteId, request, token));
|
||||
|
||||
if (!response.Success)
|
||||
{
|
||||
@@ -230,7 +240,7 @@ public class RouteTarget
|
||||
correlationId, _instanceCode, attributeNames.ToList(), DateTimeOffset.UtcNow,
|
||||
_parentExecutionId);
|
||||
|
||||
var response = await _instanceRouter.RouteToGetAttributesAsync(siteId, request, token);
|
||||
var response = await RouteOrUnreachable(_instanceRouter.RouteToGetAttributesAsync(siteId, request, token));
|
||||
|
||||
if (!response.Success)
|
||||
{
|
||||
@@ -294,7 +304,7 @@ public class RouteTarget
|
||||
AttributeValueCodec.Encode(targetValue), timeout, DateTimeOffset.UtcNow,
|
||||
_parentExecutionId);
|
||||
|
||||
var response = await _instanceRouter.RouteToWaitForAttributeAsync(siteId, request, token);
|
||||
var response = await RouteOrUnreachable(_instanceRouter.RouteToWaitForAttributeAsync(siteId, request, token));
|
||||
|
||||
if (!response.Success)
|
||||
{
|
||||
@@ -345,7 +355,7 @@ public class RouteTarget
|
||||
correlationId, _instanceCode, attributeValues, DateTimeOffset.UtcNow,
|
||||
_parentExecutionId);
|
||||
|
||||
var response = await _instanceRouter.RouteToSetAttributesAsync(siteId, request, token);
|
||||
var response = await RouteOrUnreachable(_instanceRouter.RouteToSetAttributesAsync(siteId, request, token));
|
||||
|
||||
if (!response.Success)
|
||||
{
|
||||
@@ -366,9 +376,9 @@ public class RouteTarget
|
||||
var siteId = await _instanceLocator.GetSiteIdForInstanceAsync(_instanceCode, cancellationToken);
|
||||
if (siteId == null)
|
||||
{
|
||||
// Route the site-resolution failure through the same classifier so a
|
||||
// message indicating the site is unreachable surfaces as SITE_UNREACHABLE;
|
||||
// a plain not-found / no-assigned-site stays a generic InvalidOperationException.
|
||||
// A null site id is a plain not-found / no-assigned-site — never
|
||||
// unreachability (nothing was routed), so it stays a generic
|
||||
// InvalidOperationException (SCRIPT_ERROR).
|
||||
throw RoutingFailure(
|
||||
$"Instance '{_instanceCode}' not found or has no assigned site");
|
||||
}
|
||||
@@ -377,19 +387,37 @@ public class RouteTarget
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// C3: classifies a routing-level failure message. When the message indicates the
|
||||
/// site could not be reached (unreachable / no contact) the failure is a
|
||||
/// <see cref="SiteUnreachableException"/> — so <see cref="InboundScriptExecutor"/>
|
||||
/// can surface the spec's <c>SITE_UNREACHABLE</c> code — otherwise a plain
|
||||
/// <see cref="InvalidOperationException"/>.
|
||||
/// N3: awaits a routed Ask and TYPES unreachability. AskTimeoutException means
|
||||
/// the site never answered (no ClusterClient contact, site down, partition) —
|
||||
/// the communication layer's own typed signal (CentralCommunicationActor drops
|
||||
/// envelopes for contactless sites so the Ask expires caller-side) — and
|
||||
/// surfaces as SiteUnreachableException so the executor maps it to the spec's
|
||||
/// SITE_UNREACHABLE code. A Success=false RESPONSE means the site answered and
|
||||
/// is therefore reachable; it flows through RoutingFailure as a plain
|
||||
/// InvalidOperationException (SCRIPT_ERROR).
|
||||
/// </summary>
|
||||
private static InvalidOperationException RoutingFailure(string message) =>
|
||||
IsUnreachable(message)
|
||||
? new SiteUnreachableException(message)
|
||||
: new InvalidOperationException(message);
|
||||
private static async Task<TResponse> RouteOrUnreachable<TResponse>(Task<TResponse> routedAsk)
|
||||
{
|
||||
try
|
||||
{
|
||||
return await routedAsk;
|
||||
}
|
||||
catch (AskTimeoutException ex)
|
||||
{
|
||||
throw new SiteUnreachableException(
|
||||
"Site did not respond to the routed request within the routing timeout — unreachable or not connected.",
|
||||
ex);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsUnreachable(string message) =>
|
||||
message.Contains("unreachable", StringComparison.OrdinalIgnoreCase)
|
||||
|| message.Contains("no contact", StringComparison.OrdinalIgnoreCase)
|
||||
|| message.Contains("no-contact", StringComparison.OrdinalIgnoreCase);
|
||||
/// <summary>
|
||||
/// C3/N3: a routing-level failure REPORTED BY the site or the locator. The
|
||||
/// remote end answered (or the instance simply has no site), so this is never
|
||||
/// unreachability — always a plain <see cref="InvalidOperationException"/>
|
||||
/// (SCRIPT_ERROR at the endpoint). Unreachability is typed at the Ask boundary
|
||||
/// by <c>RouteOrUnreachable</c> (AskTimeoutException → SiteUnreachableException),
|
||||
/// replacing the old message-substring sniff that a harmless rewording in the
|
||||
/// communication layer could silently defeat.
|
||||
/// </summary>
|
||||
private static InvalidOperationException RoutingFailure(string message) => new(message);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Scripting;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Scripting;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.InboundAPI;
|
||||
|
||||
/// <summary>
|
||||
/// N1 (arch-review 06 round 2): the Inbound API consumer of the script-artifact
|
||||
/// invalidation contract (docs/plans/2026-07-08-script-artifact-invalidation-contract.md,
|
||||
/// handoff item (a)). Subscribes to <see cref="IScriptArtifactChangeBus"/> on host
|
||||
/// start and, for every <c>ApiMethod</c> notification (e.g. a Transport bundle
|
||||
/// import overwriting method scripts, published post-commit), drops the compiled
|
||||
/// handler AND the known-bad record via
|
||||
/// <see cref="InboundScriptExecutor.InvalidateMethod"/>, so the next request
|
||||
/// recompiles from the fresh row without waiting for the per-request
|
||||
/// revision-check self-heal.
|
||||
/// <para>
|
||||
/// Fast-path only: the revision check in <c>ExecuteAsync</c> remains the
|
||||
/// correctness fallback — this bus is in-process/per-node and at-least-once
|
||||
/// (contract §4), so a missed or duplicate notification must always be harmless.
|
||||
/// The bus is optional (site roles and plain test hosts register none) — the
|
||||
/// subscriber then no-ops. SharedScript/Template notifications are ignored: this
|
||||
/// executor caches only ApiMethod handlers.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public sealed class ScriptArtifactChangeSubscriber : IHostedService
|
||||
{
|
||||
private readonly InboundScriptExecutor _executor;
|
||||
private readonly ILogger<ScriptArtifactChangeSubscriber> _logger;
|
||||
private readonly IScriptArtifactChangeBus? _bus;
|
||||
private IDisposable? _subscription;
|
||||
|
||||
/// <summary>Initializes the subscriber.</summary>
|
||||
/// <param name="executor">The compiled-handler cache to invalidate.</param>
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
/// <param name="bus">The change bus, or null when the host registers none (site roles, tests).</param>
|
||||
public ScriptArtifactChangeSubscriber(
|
||||
InboundScriptExecutor executor,
|
||||
ILogger<ScriptArtifactChangeSubscriber> logger,
|
||||
IScriptArtifactChangeBus? bus = null)
|
||||
{
|
||||
_executor = executor;
|
||||
_logger = logger;
|
||||
_bus = bus;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_bus is null)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
"No IScriptArtifactChangeBus registered; inbound handler-cache freshness relies on the per-request revision check alone.");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
_subscription = _bus.Subscribe(OnChanged);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_subscription?.Dispose();
|
||||
_subscription = null;
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private void OnChanged(ScriptArtifactsChanged notification)
|
||||
{
|
||||
if (!string.Equals(
|
||||
notification.ArtifactKind, ScriptArtifactKinds.ApiMethod, StringComparison.Ordinal))
|
||||
{
|
||||
return; // only ApiMethod handlers are cached by this executor
|
||||
}
|
||||
|
||||
foreach (var name in notification.Names)
|
||||
{
|
||||
_executor.InvalidateMethod(name);
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Invalidated {Count} inbound API method handler(s) after a {Source} script-artifact change.",
|
||||
notification.Names.Count, notification.Source);
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,12 @@ public static class ServiceCollectionExtensions
|
||||
// body size cap and active-node gating for POST /api/{methodName}.
|
||||
services.AddSingleton<InboundApiEndpointFilter>();
|
||||
|
||||
// N1: the ApiMethod consumer of the script-artifact invalidation bus.
|
||||
// The bus parameter is optional — hosts without a registered
|
||||
// IScriptArtifactChangeBus (site roles, plain test hosts) start it as a no-op
|
||||
// and the executor's per-request revision check alone keeps handlers fresh.
|
||||
services.AddHostedService<ScriptArtifactChangeSubscriber>();
|
||||
|
||||
return services;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +31,12 @@ namespace ZB.MOM.WW.ScadaBridge.KpiHistory;
|
||||
/// series. A one-shot backfill timer additionally folds the whole raw-retention
|
||||
/// window of already-recorded samples once shortly after start, so 30 d/90 d
|
||||
/// charts aren't blank until enough wall-clock passes after deploy; it reuses the
|
||||
/// same idempotent fold and runs at most once per actor lifetime.
|
||||
/// same idempotent fold and runs at most once per actor lifetime. The backfill is
|
||||
/// sliced into bounded 24 h fold windows (oldest-first) rather than one unbounded
|
||||
/// pass, and lowers its in-flight guard between slices so periodic folds interleave
|
||||
/// — periodic folds and backfill slices strictly alternate through the single
|
||||
/// <c>_rollupInFlight</c>/<c>_backfillInFlight</c> gate pair, so the two idempotent
|
||||
/// upserts never race on overlapping rows (arch-review 04 round 2, R1).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
@@ -118,11 +123,32 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
|
||||
private bool _rollupInFlight;
|
||||
|
||||
/// <summary>
|
||||
/// In-flight guard for the one-shot backfill fold. Set true when the backfill fold is
|
||||
/// launched and cleared when its <see cref="BackfillComplete"/> arrives. While true, periodic
|
||||
/// <see cref="RollupTick"/>s are skipped (like <see cref="_rollupInFlight"/>) so the (possibly
|
||||
/// long) full-retention-window backfill never runs concurrently with a periodic lookback fold
|
||||
/// on the same upsert rows.
|
||||
/// Width of one backfill fold slice (24 h). The repository fold materializes its whole
|
||||
/// window in memory, so the one-shot backfill must never hand it the full raw-retention
|
||||
/// window (90 d ≈ tens of millions of rows at fleet volume — arch-review 04 round 2, R1).
|
||||
/// One day per fold bounds each pass to roughly what a steady-state day writes, and the
|
||||
/// guard is lowered between slices so periodic folds interleave instead of stalling
|
||||
/// behind the historical pass.
|
||||
/// </summary>
|
||||
private static readonly TimeSpan BackfillSliceWidth = TimeSpan.FromHours(24);
|
||||
|
||||
/// <summary>
|
||||
/// Queue of remaining backfill fold slices (each a bounded <see cref="BackfillSliceWidth"/>
|
||||
/// window), oldest-first. Built once when the backfill plan is enumerated and drained one
|
||||
/// slice per fold; a periodic <see cref="RollupTick"/> may claim the fold path between slices
|
||||
/// (the two idempotent upserts strictly alternate through the
|
||||
/// <see cref="_rollupInFlight"/>/<see cref="_backfillInFlight"/> gate pair, never racing on
|
||||
/// overlapping rows — arch-review 04 round 2, R1).
|
||||
/// </summary>
|
||||
private readonly Queue<(DateTime FromHourUtc, DateTime ToHourUtc)> _backfillSlices = new();
|
||||
|
||||
/// <summary>
|
||||
/// In-flight guard for a single backfill fold slice. Set true when a slice fold is launched
|
||||
/// and cleared when its <see cref="BackfillSliceComplete"/> arrives. While true, periodic
|
||||
/// <see cref="RollupTick"/>s are skipped (like <see cref="_rollupInFlight"/>) so a backfill
|
||||
/// slice never runs concurrently with a periodic lookback fold on the same upsert rows; it is
|
||||
/// lowered BETWEEN slices so periodic folds interleave instead of stalling behind the whole
|
||||
/// historical pass.
|
||||
/// </summary>
|
||||
private bool _backfillInFlight;
|
||||
|
||||
@@ -160,10 +186,21 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
|
||||
Receive<RollupTick>(_ => HandleRollupTick());
|
||||
Receive<RollupComplete>(_ => _rollupInFlight = false); // lower the in-flight guard (success or fault)
|
||||
Receive<BackfillTick>(_ => HandleBackfillTick());
|
||||
Receive<BackfillComplete>(_ =>
|
||||
Receive<BackfillPlan>(HandleBackfillPlan);
|
||||
Receive<BackfillSliceComplete>(_ =>
|
||||
{
|
||||
_backfillInFlight = false; // lower the in-flight guard (success or fault)
|
||||
_backfillDone = true; // latch: at most once per actor lifetime
|
||||
_backfillInFlight = false; // release the fold path BETWEEN slices (R1)
|
||||
if (_backfillSlices.Count == 0)
|
||||
{
|
||||
_backfillDone = true; // latch: at most once per actor lifetime (unchanged semantics)
|
||||
_logger.LogInformation("KPI rollup backfill completed — all slices folded.");
|
||||
return;
|
||||
}
|
||||
|
||||
// Next slice via the timer, not inline: a RollupTick already queued in the mailbox
|
||||
// is processed first, so periodic folds interleave; the existing _rollupInFlight
|
||||
// deferral branch in HandleBackfillTick then re-arms the backfill behind it.
|
||||
Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.Zero);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -483,13 +520,15 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
|
||||
|
||||
/// <summary>
|
||||
/// Handles the one-shot backfill tick. Runs at most once per actor lifetime: if the backfill
|
||||
/// has already completed (<see cref="_backfillDone"/>) or is in flight
|
||||
/// has already completed (<see cref="_backfillDone"/>) or a slice is in flight
|
||||
/// (<see cref="_backfillInFlight"/>) the tick is ignored. If a periodic fold currently holds
|
||||
/// the fold path (<see cref="_rollupInFlight"/>) the backfill defers by re-arming its single
|
||||
/// timer, so the full-window backfill and the periodic lookback fold never race on overlapping
|
||||
/// upsert rows. Otherwise it raises the backfill guard and folds the whole raw-retention
|
||||
/// window <c>[TruncateToHour(now) − RetentionDays, TruncateToHour(now))</c> off the actor
|
||||
/// thread, piping a <see cref="BackfillComplete"/> back to lower the guard and set the latch.
|
||||
/// timer, so a backfill slice and the periodic lookback fold never race on overlapping upsert
|
||||
/// rows. When no slice plan exists yet it enumerates the raw-retention window
|
||||
/// <c>[TruncateToHour(now) − RetentionDays, TruncateToHour(now))</c> into bounded
|
||||
/// <see cref="BackfillSliceWidth"/> windows (oldest-first) and self-arms to start draining;
|
||||
/// otherwise it dequeues ONE slice, raises the guard, and folds it off the actor thread,
|
||||
/// piping a <see cref="BackfillSliceComplete"/> back to release the guard between slices.
|
||||
/// </summary>
|
||||
private void HandleBackfillTick()
|
||||
{
|
||||
@@ -500,38 +539,152 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
|
||||
|
||||
if (_rollupInFlight)
|
||||
{
|
||||
// A periodic fold is in flight; defer the one-shot backfill briefly so the two
|
||||
// A periodic fold is in flight; defer the next backfill slice briefly so the two
|
||||
// idempotent upserts don't contend on overlapping rows. Re-arm the single timer.
|
||||
Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.FromSeconds(5));
|
||||
return;
|
||||
}
|
||||
|
||||
if (_backfillSlices.Count == 0)
|
||||
{
|
||||
// No slice plan yet: resolve the rollup watermark off the actor thread, then plan on
|
||||
// the actor thread (Receive<BackfillPlan>). Raise the guard so periodic ticks defer
|
||||
// while planning is in flight; the plan handler lowers it. On a failover restart the
|
||||
// watermark lets the plan SKIP or SHRINK to the un-rolled tail instead of re-folding
|
||||
// the whole raw-retention window (arch-review 04 round 2, R1).
|
||||
_backfillInFlight = true;
|
||||
var planCancellationToken = _shutdownCts?.Token ?? CancellationToken.None;
|
||||
|
||||
RunBackfillPlanPass(planCancellationToken).PipeTo(
|
||||
Self,
|
||||
success: watermark => new BackfillPlan(watermark),
|
||||
failure: ex =>
|
||||
{
|
||||
_logger.LogError(ex, "KPI rollup backfill plan pass faulted unexpectedly.");
|
||||
return new BackfillPlan(null); // null watermark ⇒ plan the full retention window
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Dequeue and fold exactly ONE bounded slice.
|
||||
var slice = _backfillSlices.Dequeue();
|
||||
_backfillInFlight = true;
|
||||
var toHourUtc = TruncateToHour(DateTime.UtcNow);
|
||||
var fromHourUtc = toHourUtc - TimeSpan.FromDays(_options.RetentionDays);
|
||||
var cancellationToken = _shutdownCts?.Token ?? CancellationToken.None;
|
||||
|
||||
_logger.LogInformation(
|
||||
"KPI rollup backfill starting — folding existing samples in [{FromHour:o}, {ToHour:o}) ({RetentionDays}-day window).",
|
||||
fromHourUtc, toHourUtc, _options.RetentionDays);
|
||||
|
||||
RunBackfillPass(fromHourUtc, toHourUtc, cancellationToken).PipeTo(
|
||||
RunBackfillPass(slice.FromHourUtc, slice.ToHourUtc, cancellationToken).PipeTo(
|
||||
Self,
|
||||
success: () => BackfillComplete.Instance,
|
||||
success: () => BackfillSliceComplete.Instance,
|
||||
failure: ex =>
|
||||
{
|
||||
_logger.LogError(ex, "KPI rollup backfill faulted unexpectedly.");
|
||||
return BackfillComplete.Instance;
|
||||
_logger.LogError(ex, "KPI rollup backfill slice faulted unexpectedly.");
|
||||
return BackfillSliceComplete.Instance;
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs the one-shot backfill fold: opens a DI scope, resolves the repository, and folds the
|
||||
/// Enumerates <c>[<paramref name="fromHourUtc"/>, <paramref name="toHourUtc"/>)</c> into
|
||||
/// contiguous <see cref="BackfillSliceWidth"/> windows (the last possibly shorter),
|
||||
/// oldest-first, and enqueues them onto <see cref="_backfillSlices"/>.
|
||||
/// </summary>
|
||||
private void EnqueueBackfillSlices(DateTime fromHourUtc, DateTime toHourUtc)
|
||||
{
|
||||
var sliceStart = fromHourUtc;
|
||||
while (sliceStart < toHourUtc)
|
||||
{
|
||||
var sliceEnd = sliceStart + BackfillSliceWidth;
|
||||
if (sliceEnd > toHourUtc)
|
||||
{
|
||||
sliceEnd = toHourUtc;
|
||||
}
|
||||
|
||||
_backfillSlices.Enqueue((sliceStart, sliceEnd));
|
||||
sliceStart = sliceEnd;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Plans the backfill from the resolved rollup watermark (actor thread). The window floor is
|
||||
/// the newer of the watermark (re-folding the watermark hour itself is a cheap idempotent
|
||||
/// safety margin) and the raw-retention floor; a null watermark (fresh install) keeps the full
|
||||
/// retention window. If the floor is already within
|
||||
/// <see cref="KpiHistoryOptions.RollupLookbackHours"/> of now the periodic fold's lookback
|
||||
/// covers everything from there forward, so the whole historical pass is skipped (the failover
|
||||
/// fast-path); otherwise the tail <c>[floor, TruncateToHour(now))</c> is enumerated into
|
||||
/// bounded slices and draining is self-armed. Either way the plan-pass guard is lowered here
|
||||
/// (arch-review 04 round 2, R1).
|
||||
/// </summary>
|
||||
private void HandleBackfillPlan(BackfillPlan plan)
|
||||
{
|
||||
var toHourUtc = TruncateToHour(DateTime.UtcNow);
|
||||
var retentionFloor = toHourUtc - TimeSpan.FromDays(_options.RetentionDays);
|
||||
// Shrink to the un-rolled tail; re-fold the watermark hour itself (idempotent) as a safety
|
||||
// margin. A null watermark (fresh install) keeps the full retention window.
|
||||
var floor = plan.Watermark is { } w && w > retentionFloor ? w : retentionFloor;
|
||||
|
||||
if (floor >= toHourUtc - TimeSpan.FromHours(_options.RollupLookbackHours))
|
||||
{
|
||||
// Failover fast-path: rollups are already current — the periodic fold's lookback
|
||||
// window covers everything from the watermark forward, so the full historical pass is
|
||||
// skipped entirely (arch-review 04 round 2, R1).
|
||||
_backfillInFlight = false;
|
||||
_backfillDone = true;
|
||||
_logger.LogInformation(
|
||||
"KPI rollup backfill skipped — rollups current through {Watermark:o}.", plan.Watermark);
|
||||
return;
|
||||
}
|
||||
|
||||
EnqueueBackfillSlices(floor, toHourUtc);
|
||||
_backfillInFlight = false;
|
||||
|
||||
if (_backfillSlices.Count == 0)
|
||||
{
|
||||
// Degenerate window (no whole hours to fold) — nothing to backfill.
|
||||
_backfillDone = true;
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"KPI rollup backfill starting — folding [{FromHour:o}, {ToHour:o}) across {SliceCount} "
|
||||
+ "slice(s) of up to {SliceHours} h (watermark {Watermark:o}).",
|
||||
floor, toHourUtc, _backfillSlices.Count, BackfillSliceWidth.TotalHours, plan.Watermark);
|
||||
|
||||
// Start draining via the timer (uniform path with the between-slice re-arm).
|
||||
Timers.StartSingleTimer(BackfillTimerKey, BackfillTick.Instance, TimeSpan.Zero);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Resolves the newest folded rollup hour (the backfill watermark) off the actor thread in a
|
||||
/// fresh DI scope. Never faults — on cancellation or error it returns <c>null</c>, which the
|
||||
/// plan treats as "no rollups", keeping the full retention window (best-effort, mirrors the
|
||||
/// other passes).
|
||||
/// </summary>
|
||||
private async Task<DateTime?> RunBackfillPlanPass(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var scope = _serviceProvider.CreateAsyncScope();
|
||||
var repository = scope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();
|
||||
return await repository.GetLatestRollupHourAsync(cancellationToken);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Shutdown interrupted the lookup; a fresh active node re-plans. Not a failure.
|
||||
return null;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "KPI rollup backfill watermark lookup failed; planning the full retention window.");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs one backfill fold slice: opens a DI scope, resolves the repository, and folds the
|
||||
/// complete hours in <c>[<paramref name="fromHourUtc"/>, <paramref name="toHourUtc"/>)</c> into
|
||||
/// the hourly rollup table via the same idempotent upsert the periodic fold uses (Task 3). The
|
||||
/// whole body is wrapped so the returned task never faults — best-effort observability must
|
||||
/// never disrupt startup. The fold method returns no count, so completion is logged with the
|
||||
/// window and elapsed wall time only.
|
||||
/// the hourly rollup table via the same idempotent upsert the periodic fold uses. The whole
|
||||
/// body is wrapped so the returned task never faults — best-effort observability must never
|
||||
/// disrupt startup. The fold method returns no count, so each slice is logged at Debug with its
|
||||
/// window and elapsed wall time (the queue-drained completion is logged at Information).
|
||||
/// </summary>
|
||||
private async Task RunBackfillPass(
|
||||
DateTime fromHourUtc, DateTime toHourUtc, CancellationToken cancellationToken)
|
||||
@@ -543,8 +696,8 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
|
||||
var repository = scope.ServiceProvider.GetRequiredService<IKpiHistoryRepository>();
|
||||
await repository.FoldHourlyRollupsAsync(fromHourUtc, toHourUtc, cancellationToken);
|
||||
|
||||
_logger.LogInformation(
|
||||
"KPI rollup backfill completed — folded [{FromHour:o}, {ToHour:o}) in {ElapsedMs:F0} ms.",
|
||||
_logger.LogDebug(
|
||||
"KPI rollup backfill slice folded [{FromHour:o}, {ToHour:o}) in {ElapsedMs:F0} ms.",
|
||||
fromHourUtc, toHourUtc, (DateTime.UtcNow - startedAt).TotalMilliseconds);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
@@ -614,9 +767,11 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Self-tick triggering the one-shot backfill fold over the full raw-retention window. Armed
|
||||
/// once via a single timer in <see cref="PreStart"/> (and re-armed only to defer past an
|
||||
/// in-flight periodic fold); runs at most once per actor lifetime.
|
||||
/// Self-tick triggering the one-shot backfill: on the first tick it enumerates the full
|
||||
/// raw-retention window into bounded slices, and each subsequent tick folds one slice. Armed
|
||||
/// once via a single timer in <see cref="PreStart"/> (and re-armed to drain the next slice or
|
||||
/// to defer past an in-flight periodic fold); the whole pass runs at most once per actor
|
||||
/// lifetime.
|
||||
/// </summary>
|
||||
internal sealed class BackfillTick
|
||||
{
|
||||
@@ -625,13 +780,21 @@ public class KpiHistoryRecorderActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Piped-back completion of the one-shot backfill fold; lets the fold run off the actor thread
|
||||
/// and, on the actor thread, lowers the <c>_backfillInFlight</c> guard and latches
|
||||
/// Piped-back result of the backfill plan pass: the newest folded rollup hour (the watermark),
|
||||
/// or <c>null</c> when no rollups exist. Consumed on the actor thread to skip or shrink the
|
||||
/// backfill to the un-rolled tail (arch-review 04 round 2, R1).
|
||||
/// </summary>
|
||||
internal sealed record BackfillPlan(DateTime? Watermark);
|
||||
|
||||
/// <summary>
|
||||
/// Piped-back completion of ONE backfill fold slice; lets the slice run off the actor thread
|
||||
/// and, on the actor thread, lowers the <c>_backfillInFlight</c> guard BETWEEN slices — then
|
||||
/// either drains the next slice (self-armed timer) or, when the queue is empty, latches
|
||||
/// <c>_backfillDone</c> (fires on success and fault).
|
||||
/// </summary>
|
||||
internal sealed class BackfillComplete
|
||||
internal sealed class BackfillSliceComplete
|
||||
{
|
||||
public static readonly BackfillComplete Instance = new();
|
||||
private BackfillComplete() { }
|
||||
public static readonly BackfillSliceComplete Instance = new();
|
||||
private BackfillSliceComplete() { }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ internal static class ConfigSecretScrubber
|
||||
private static readonly string[] SecretNameFragments =
|
||||
["password", "secret", "token", "apikey", "credential", "passphrase"];
|
||||
|
||||
private static bool IsSecretName(string propertyName)
|
||||
internal static bool IsSecretName(string propertyName)
|
||||
{
|
||||
foreach (var fragment in SecretNameFragments)
|
||||
{
|
||||
@@ -127,6 +127,15 @@ internal static class ConfigSecretScrubber
|
||||
/// <param name="incoming">The client-submitted config JSON, possibly containing sentinel values.</param>
|
||||
/// <param name="stored">The previously stored config JSON to source real secret values from.</param>
|
||||
/// <returns>The incoming JSON with sentinel values replaced by the corresponding stored values.</returns>
|
||||
/// <remarks>
|
||||
/// <b>Array limitation (arch-review R2 N9):</b> sentinel restoration inside arrays
|
||||
/// pairs incoming[i] with stored[i] BY INDEX. A client that reorders an array of
|
||||
/// credentialed objects while round-tripping sentinels grafts the wrong stored
|
||||
/// secret into each slot — a silent misconfiguration (never a disclosure). Clients
|
||||
/// must not reorder arrays that carry sentinels (the CLI round-trip does not);
|
||||
/// key-based matching is a possible future refinement if a config type ever nests
|
||||
/// credentialed arrays.
|
||||
/// </remarks>
|
||||
public static string? MergeSentinels(string? incoming, string? stored)
|
||||
{
|
||||
if (string.IsNullOrEmpty(incoming))
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
using System.Text;
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -6,7 +5,6 @@ using ZB.MOM.WW.ScadaBridge.Commons.Interfaces.Repositories;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Streaming;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication;
|
||||
using ZB.MOM.WW.Auth.Abstractions.Ldap;
|
||||
using ZB.MOM.WW.ScadaBridge.Security;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.ManagementService;
|
||||
@@ -75,73 +73,35 @@ public class DebugStreamHub : Hub
|
||||
return;
|
||||
}
|
||||
|
||||
// Dev/test: when login is disabled, auto-authenticate the debug stream as the configured
|
||||
// dev user (ALL roles incl. Deployer, system-wide) — mirroring the cookie/management
|
||||
// bypass so the CLI debug stream is usable in a login-disabled (e.g. no-LDAP) deployment.
|
||||
// See ManagementAuthenticator.
|
||||
var bypassUser = ManagementAuthenticator.TryDisableLoginUser(httpContext);
|
||||
if (bypassUser is not null)
|
||||
// Authenticate via the SAME shared Basic→LDAP flow as POST /management and the
|
||||
// audit REST (arch-review R2 N1): ManagementAuthenticator applies the DisableLogin
|
||||
// bypass, the LoginThrottle lockout check BEFORE the bind, and RecordFailure/
|
||||
// RecordSuccess bookkeeping — so hub connection attempts can no longer be used as
|
||||
// an unthrottled LDAP-bind oracle, and hub failures feed the shared lockout.
|
||||
var outcome = await ManagementAuthenticator.AuthenticateAsync(httpContext);
|
||||
if (outcome.User is null)
|
||||
{
|
||||
Context.Items[RolesKey] = bypassUser.Roles;
|
||||
Context.Items[PermittedSiteIdsKey] = bypassUser.PermittedSiteIds;
|
||||
_logger.LogInformation(
|
||||
"DebugStreamHub connection established for {Username} (login disabled)", bypassUser.Username);
|
||||
await base.OnConnectedAsync();
|
||||
return;
|
||||
}
|
||||
|
||||
// Extract Basic Auth credentials
|
||||
var authHeader = httpContext.Request.Headers.Authorization.ToString();
|
||||
if (string.IsNullOrEmpty(authHeader) || !authHeader.StartsWith("Basic ", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
_logger.LogWarning("DebugStreamHub connection rejected: missing Basic Auth header");
|
||||
_logger.LogWarning("DebugStreamHub connection rejected: authentication failed or throttled");
|
||||
Context.Abort();
|
||||
return;
|
||||
}
|
||||
var user = outcome.User;
|
||||
|
||||
string username, password;
|
||||
try
|
||||
// Role check — Deployer role required (unchanged policy).
|
||||
if (!user.Roles.Contains(Roles.Deployer))
|
||||
{
|
||||
var decoded = Encoding.UTF8.GetString(Convert.FromBase64String(authHeader["Basic ".Length..]));
|
||||
var colon = decoded.IndexOf(':');
|
||||
if (colon < 0) throw new FormatException();
|
||||
username = decoded[..colon];
|
||||
password = decoded[(colon + 1)..];
|
||||
}
|
||||
catch
|
||||
{
|
||||
_logger.LogWarning("DebugStreamHub connection rejected: malformed Basic Auth");
|
||||
Context.Abort();
|
||||
return;
|
||||
}
|
||||
|
||||
// LDAP authentication
|
||||
var ldapAuth = httpContext.RequestServices.GetRequiredService<ILdapAuthService>();
|
||||
var authResult = await ldapAuth.AuthenticateAsync(username, password, Context.ConnectionAborted);
|
||||
if (!authResult.Succeeded)
|
||||
{
|
||||
_logger.LogWarning("DebugStreamHub connection rejected: LDAP auth failed for {Username}", username);
|
||||
Context.Abort();
|
||||
return;
|
||||
}
|
||||
|
||||
// Role check — Deployer role required
|
||||
var roleMapper = httpContext.RequestServices.GetRequiredService<RoleMapper>();
|
||||
var mappingResult = await roleMapper.MapGroupsToRolesAsync(authResult.Groups, Context.ConnectionAborted);
|
||||
|
||||
if (!mappingResult.Roles.Contains(Roles.Deployer))
|
||||
{
|
||||
_logger.LogWarning("DebugStreamHub connection rejected: {Username} lacks Deployer role", username);
|
||||
_logger.LogWarning("DebugStreamHub connection rejected: {Username} lacks Deployer role", user.Username);
|
||||
Context.Abort();
|
||||
return;
|
||||
}
|
||||
|
||||
// Persist the resolved identity on the connection so per-instance site-scope
|
||||
// enforcement can be applied to SubscribeInstance calls.
|
||||
Context.Items[RolesKey] = mappingResult.Roles.ToArray();
|
||||
Context.Items[PermittedSiteIdsKey] = mappingResult.PermittedSiteIds.ToArray();
|
||||
// enforcement can be applied to SubscribeInstance calls. PermittedSiteIds is the
|
||||
// authenticator's normalized form (empty == system-wide), same as every other surface.
|
||||
Context.Items[RolesKey] = user.Roles;
|
||||
Context.Items[PermittedSiteIdsKey] = user.PermittedSiteIds;
|
||||
|
||||
_logger.LogInformation("DebugStreamHub connection established for {Username}", username);
|
||||
_logger.LogInformation("DebugStreamHub connection established for {Username}", user.Username);
|
||||
await base.OnConnectedAsync();
|
||||
}
|
||||
|
||||
|
||||
@@ -467,7 +467,7 @@ public class ManagementActor : ReceiveActor
|
||||
SubmitSecuredWriteCommand cmd => await HandleSubmitSecuredWrite(sp, cmd, user),
|
||||
ApproveSecuredWriteCommand cmd => await HandleApproveSecuredWrite(sp, cmd, user),
|
||||
RejectSecuredWriteCommand cmd => await HandleRejectSecuredWrite(sp, cmd, user),
|
||||
ListSecuredWritesCommand cmd => await HandleListSecuredWrites(sp, cmd),
|
||||
ListSecuredWritesCommand cmd => await HandleListSecuredWrites(sp, cmd, user),
|
||||
|
||||
// Transport bundle operations
|
||||
ExportBundleCommand cmd => await HandleExportBundle(sp, cmd, user.Username),
|
||||
@@ -1247,6 +1247,11 @@ public class ManagementActor : ReceiveActor
|
||||
var site = await siteRepo.GetSiteByIdentifierAsync(cmd.SiteId)
|
||||
?? throw new ManagementCommandException($"Site '{cmd.SiteId}' not found.");
|
||||
|
||||
// Site scope (arch-review R2 N3): an LDAP-mapping-scoped Operator may only
|
||||
// submit device writes to their permitted sites — same rule C2 applies to
|
||||
// DeployArtifacts, on a higher-consequence path. Checked before the row exists.
|
||||
EnforceSiteScope(user, site.Id);
|
||||
|
||||
var connections = await siteRepo.GetDataConnectionsBySiteIdAsync(site.Id);
|
||||
var conn = connections.FirstOrDefault(c =>
|
||||
string.Equals(c.Name, cmd.ConnectionName, StringComparison.Ordinal))
|
||||
@@ -1293,6 +1298,11 @@ public class ManagementActor : ReceiveActor
|
||||
throw new ManagementCommandException(
|
||||
$"Secured write {cmd.Id} is '{row.Status}', not Pending; it cannot be approved.");
|
||||
|
||||
// Site scope (arch-review R2 N3): the approving Verifier must be permitted on
|
||||
// the row's target site — checked BEFORE the TTL/self-approval/CAS chain so an
|
||||
// out-of-scope verifier can neither consume the Pending transition nor relay.
|
||||
await EnforceSiteScopeForIdentifier(sp, user, row.SiteId);
|
||||
|
||||
// Server-side TTL: a Pending write older than the configured age is transitioned
|
||||
// to Expired and can never be approved — the stale setpoint is never relayed
|
||||
// (arch-review S2). Checked BEFORE the self-approval/CAS so an overdue row never
|
||||
@@ -1450,6 +1460,10 @@ public class ManagementActor : ReceiveActor
|
||||
throw new ManagementCommandException(
|
||||
$"Secured write {cmd.Id} is '{entity.Status}', not Pending; it cannot be rejected.");
|
||||
|
||||
// Site scope (arch-review R2 N3): the rejecting Verifier must be permitted on
|
||||
// the row's target site — checked BEFORE the TTL/self-approval chain.
|
||||
await EnforceSiteScopeForIdentifier(sp, user, entity.SiteId);
|
||||
|
||||
// Server-side TTL: an overdue Pending write expires here too, so a reject on a
|
||||
// stale row deterministically reports the expiry rather than recording a
|
||||
// human decision (arch-review S2).
|
||||
@@ -1479,16 +1493,36 @@ public class ManagementActor : ReceiveActor
|
||||
}
|
||||
|
||||
private static async Task<object?> HandleListSecuredWrites(
|
||||
IServiceProvider sp, ListSecuredWritesCommand cmd)
|
||||
IServiceProvider sp, ListSecuredWritesCommand cmd, AuthenticatedUser user)
|
||||
{
|
||||
// Site scoping (arch-review R2 N3): an explicit SiteId filter is scope-checked
|
||||
// like every other identifier-keyed command; an UNFILTERED list from a
|
||||
// site-scoped non-admin caller is constrained to their permitted sites at the
|
||||
// query level (rows AND totalCount both scoped — see ISecuredWriteRepository).
|
||||
IReadOnlyCollection<string>? permittedIdentifiers = null;
|
||||
if (cmd.SiteId is not null)
|
||||
{
|
||||
await EnforceSiteScopeForIdentifier(sp, user, cmd.SiteId);
|
||||
}
|
||||
else if (user.PermittedSiteIds.Length > 0
|
||||
&& !user.Roles.Contains(Roles.Administrator, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
var siteRepo = sp.GetRequiredService<ISiteRepository>();
|
||||
var permitted = new HashSet<string>(user.PermittedSiteIds);
|
||||
permittedIdentifiers = (await siteRepo.GetAllSitesAsync())
|
||||
.Where(s => permitted.Contains(s.Id.ToString()))
|
||||
.Select(s => s.SiteIdentifier)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
var repo = sp.GetRequiredService<ISecuredWriteRepository>();
|
||||
|
||||
// Offset paging (arch-review P3): clamp untrusted paging inputs so a crafted
|
||||
// command can neither request a negative offset nor an unbounded page.
|
||||
var take = Math.Clamp(cmd.Take, 1, 500);
|
||||
var skip = Math.Max(0, cmd.Skip);
|
||||
var rows = await repo.QueryAsync(cmd.Status, cmd.SiteId, skip, take);
|
||||
var totalCount = await repo.CountAsync(cmd.Status, cmd.SiteId);
|
||||
var rows = await repo.QueryAsync(cmd.Status, cmd.SiteId, skip, take, permittedIdentifiers);
|
||||
var totalCount = await repo.CountAsync(cmd.Status, cmd.SiteId, permittedIdentifiers);
|
||||
|
||||
// Opportunistic expiry sweep (arch-review S2): every time the list is read, walk
|
||||
// the page and expire any overdue Pending row. TryExpireIfStaleAsync self-filters
|
||||
@@ -1583,6 +1617,18 @@ public class ManagementActor : ReceiveActor
|
||||
await repo.SaveChangesAsync();
|
||||
var commService = sp.GetService<CommunicationService>();
|
||||
commService?.RefreshSiteAddresses();
|
||||
|
||||
// Dispose the deleted site's cached gRPC channels — RemoveSiteAsync is the
|
||||
// factory's designed site-deletion disposal path and previously had no
|
||||
// production caller (arch review 02 round 2, N8): a deleted site's channels
|
||||
// (both node endpoints) otherwise persist until process restart. Null-safe:
|
||||
// test/composition roots without the factory skip it. Any live-alarm
|
||||
// aggregator for the site reconciles to an empty snapshot and is reaped by
|
||||
// the viewer linger (documented acceptance — see Component-Communication.md).
|
||||
var grpcFactory = sp.GetService<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcClientFactory>();
|
||||
if (grpcFactory is not null && site is not null)
|
||||
await grpcFactory.RemoveSiteAsync(site.SiteIdentifier);
|
||||
|
||||
await AuditAsync(sp, user, "Delete", "Site", cmd.SiteId.ToString(), site?.Name ?? cmd.SiteId.ToString(), null);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.HttpOverrides;
|
||||
using System.Net;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Security;
|
||||
|
||||
/// <summary>
|
||||
/// Config shape for trusted-proxy forwarded-header handling
|
||||
/// (<c>ScadaBridge:Security:ForwardedHeaders</c>). Disabled by default: a deployment
|
||||
/// with no reverse proxy must NOT honor X-Forwarded-For (a client could spoof its
|
||||
/// throttle key). The documented Traefik topologies enable it with the docker
|
||||
/// network range so LoginThrottle keys on the REAL client IP (arch-review R2 N2).
|
||||
/// </summary>
|
||||
public sealed class ForwardedHeadersSecurityOptions
|
||||
{
|
||||
/// <summary>Configuration section binding this options class.</summary>
|
||||
public const string SectionName = "ScadaBridge:Security:ForwardedHeaders";
|
||||
|
||||
/// <summary>Whether X-Forwarded-For from trusted proxies is honored.</summary>
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
/// <summary>Exact proxy IPs allowed to supply X-Forwarded-For.</summary>
|
||||
public string[] KnownProxies { get; set; } = [];
|
||||
|
||||
/// <summary>CIDR networks allowed to supply X-Forwarded-For (e.g. "172.16.0.0/12").</summary>
|
||||
public string[] KnownNetworks { get; set; } = [];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the <see cref="ForwardedHeadersOptions"/> for <c>app.UseForwardedHeaders</c>.
|
||||
/// Pure and fail-fast: malformed entries throw at startup rather than silently
|
||||
/// trusting nothing/everything. Defaults (loopback) are CLEARED — only the
|
||||
/// explicitly configured proxies/networks are trusted, and ForwardLimit=1 means
|
||||
/// only the entry the trusted proxy itself appended is honored.
|
||||
/// </summary>
|
||||
public static class ForwardedHeadersSetup
|
||||
{
|
||||
/// <summary>
|
||||
/// Builds the forwarded-headers options, or <see langword="null"/> when disabled.
|
||||
/// </summary>
|
||||
/// <param name="options">The bound <see cref="ForwardedHeadersSecurityOptions"/>.</param>
|
||||
/// <returns>The configured <see cref="ForwardedHeadersOptions"/>, or <see langword="null"/> when disabled.</returns>
|
||||
public static ForwardedHeadersOptions? Build(ForwardedHeadersSecurityOptions options)
|
||||
{
|
||||
if (!options.Enabled) return null;
|
||||
var built = new ForwardedHeadersOptions
|
||||
{
|
||||
ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto,
|
||||
ForwardLimit = 1,
|
||||
};
|
||||
built.KnownProxies.Clear();
|
||||
// KnownIPNetworks (System.Net.IPNetwork) is the net10 successor to the now-deprecated
|
||||
// KnownNetworks; UseForwardedHeaders reads it identically.
|
||||
built.KnownIPNetworks.Clear();
|
||||
foreach (var proxy in options.KnownProxies)
|
||||
built.KnownProxies.Add(IPAddress.Parse(proxy));
|
||||
foreach (var network in options.KnownNetworks)
|
||||
{
|
||||
var parts = network.Split('/');
|
||||
built.KnownIPNetworks.Add(new(IPAddress.Parse(parts[0]), int.Parse(parts[1])));
|
||||
}
|
||||
return built;
|
||||
}
|
||||
}
|
||||
@@ -41,6 +41,18 @@ public sealed class LoginThrottle
|
||||
/// </summary>
|
||||
public const int MaxTrackedKeys = 10_000;
|
||||
|
||||
/// <summary>Amortization cadence: a full prune sweep runs every Nth failure write
|
||||
/// (plus immediately whenever the map exceeds <see cref="MaxTrackedKeys"/>), so a
|
||||
/// spray does not pay an O(n) scan — and a possible full sort — on every failed
|
||||
/// request (arch-review R2 N8). Lockout SEMANTICS are unaffected: IsLockedOut reads
|
||||
/// window/lockout expiry directly and never depends on pruning.</summary>
|
||||
internal const int PruneEveryNFailures = 64;
|
||||
|
||||
/// <summary>Diagnostics/test accessor: number of currently tracked keys.</summary>
|
||||
internal int TrackedKeyCount => _entries.Count;
|
||||
|
||||
private int _failureWrites;
|
||||
|
||||
private readonly TimeProvider _clock;
|
||||
private readonly IOptions<SecurityOptions> _options;
|
||||
|
||||
@@ -118,7 +130,13 @@ public sealed class LoginThrottle
|
||||
return Arm(existing with { Count = existing.Count + 1 }, opts, now, lockout);
|
||||
});
|
||||
|
||||
Prune(now);
|
||||
// Amortized prune (R2 N8): every Nth failure, or immediately when over the hard
|
||||
// cap — the cap is the memory-safety invariant and must never wait for the cadence.
|
||||
if (Interlocked.Increment(ref _failureWrites) % PruneEveryNFailures == 0
|
||||
|| _entries.Count > MaxTrackedKeys)
|
||||
{
|
||||
Prune(now);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -42,7 +42,7 @@ public sealed class SiteCallAuditKpiSampleSource : IKpiSampleSource
|
||||
private const string MetricBuffered = KpiMetrics.SiteCallAudit.Buffered;
|
||||
private const string MetricParked = KpiMetrics.SiteCallAudit.Parked;
|
||||
private const string MetricFailedLastInterval = KpiMetrics.SiteCallAudit.FailedLastInterval;
|
||||
private const string MetricDeliveredLastInterval = "deliveredLastInterval";
|
||||
private const string MetricDeliveredLastInterval = KpiMetrics.SiteCallAudit.DeliveredLastInterval;
|
||||
private const string MetricStuck = "stuck";
|
||||
private const string MetricOldestPendingAgeSeconds = "oldestPendingAgeSeconds";
|
||||
|
||||
|
||||
@@ -186,6 +186,10 @@ public class AlarmActor : ReceiveActor
|
||||
|
||||
// Handle the off-dispatcher trigger-expression evaluation result (P1).
|
||||
Receive<ExpressionEvalResult>(HandleExpressionEvalResult);
|
||||
|
||||
// Handle a faulted off-dispatcher evaluation task (N2) — clears in-flight
|
||||
// instead of stranding the alarm trigger with an unhandled Status.Failure.
|
||||
Receive<ExpressionEvalFailed>(HandleExpressionEvalFailed);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -551,7 +555,8 @@ public class AlarmActor : ReceiveActor
|
||||
}
|
||||
}, CancellationToken.None, TaskCreationOptions.DenyChildAttach,
|
||||
ScriptExecutionScheduler.Shared(_options)).Unwrap().PipeTo(self,
|
||||
success: r => new ExpressionEvalResult(r));
|
||||
success: r => new ExpressionEvalResult(r),
|
||||
failure: ex => new ExpressionEvalFailed(ex));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -576,6 +581,22 @@ public class AlarmActor : ReceiveActor
|
||||
if (_evalPending) { _evalPending = false; StartExpressionEvaluation(); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies a faulted off-dispatcher evaluation task (N2): logs the fault, then
|
||||
/// delegates to the false result (per the class contract — a throwing expression
|
||||
/// is treated as false so the state machine still runs, an Active alarm clears)
|
||||
/// which clears in-flight and drains any pending evaluation. Without this the
|
||||
/// alarm trigger would be permanently parked with _evalInFlight stuck true.
|
||||
/// </summary>
|
||||
private void HandleExpressionEvalFailed(ExpressionEvalFailed msg)
|
||||
{
|
||||
_healthCollector?.IncrementAlarmError();
|
||||
_logger.LogError(msg.Cause,
|
||||
"Alarm {Alarm} trigger-expression evaluation task faulted on {Instance}; treated as false",
|
||||
_alarmName, _instanceName);
|
||||
HandleExpressionEvalResult(new ExpressionEvalResult(false));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// HiLo level evaluator: returns the most-severe matching band for the
|
||||
/// given value. Severity order checked from highest to lowest so that a
|
||||
@@ -795,6 +816,15 @@ public class AlarmActor : ReceiveActor
|
||||
/// actor thread.
|
||||
/// </summary>
|
||||
private sealed record ExpressionEvalResult(bool Result);
|
||||
|
||||
/// <summary>
|
||||
/// Piped back to self when the off-dispatcher evaluation TASK itself faults
|
||||
/// (e.g. ObjectDisposedException from a disposed ScriptExecutionScheduler
|
||||
/// during shutdown) — the inner body's catch never sees that. Without this
|
||||
/// mapping the actor receives an unhandled Status.Failure and _evalInFlight
|
||||
/// stays true forever, permanently parking the expression trigger (N2).
|
||||
/// </summary>
|
||||
internal sealed record ExpressionEvalFailed(Exception Cause);
|
||||
}
|
||||
|
||||
internal enum RateOfChangeDirection { Either, Rising, Falling }
|
||||
|
||||
@@ -518,7 +518,11 @@ public class DeploymentManagerActor : ReceiveActor, IWithTimers
|
||||
// delete/disable commands, breaking that ordering. A deploy is an
|
||||
// infrequent admin command, so briefly holding the singleton for a pure
|
||||
// Roslyn compile is acceptable; the central deployer already Asks and
|
||||
// waits for the DeploymentStatusResponse.
|
||||
// waits for the DeploymentStatusResponse. Redeploys and multi-instance
|
||||
// deploys of unchanged scripts hit the process-wide compile cache
|
||||
// (SiteScriptCompileCache), so the synchronous gate recompiles only
|
||||
// genuinely new code and the Instance Actor's PreStart reuses the gate's
|
||||
// compile (N4).
|
||||
var errors = _deployCompileValidator.Validate(command.FlattenedConfigurationJson);
|
||||
if (errors.Count > 0)
|
||||
{
|
||||
|
||||
@@ -159,6 +159,10 @@ public class ScriptActor : ReceiveActor, IWithTimers
|
||||
|
||||
// Handle the off-dispatcher trigger-expression evaluation result (P1).
|
||||
Receive<ExpressionEvalResult>(HandleExpressionEvalResult);
|
||||
|
||||
// Handle a faulted off-dispatcher evaluation task (N2) — clears in-flight
|
||||
// instead of stranding the trigger with an unhandled Status.Failure.
|
||||
Receive<ExpressionEvalFailed>(HandleExpressionEvalFailed);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -310,7 +314,8 @@ public class ScriptActor : ReceiveActor, IWithTimers
|
||||
}
|
||||
}, CancellationToken.None, TaskCreationOptions.DenyChildAttach,
|
||||
ScriptExecutionScheduler.Shared(_options)).Unwrap().PipeTo(self,
|
||||
success: r => new ExpressionEvalResult(r));
|
||||
success: r => new ExpressionEvalResult(r),
|
||||
failure: ex => new ExpressionEvalFailed(ex));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -333,6 +338,18 @@ public class ScriptActor : ReceiveActor, IWithTimers
|
||||
if (_evalPending) { _evalPending = false; StartExpressionEvaluation(); }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies a faulted off-dispatcher evaluation task (N2): logs the fault via the
|
||||
/// existing error path (which also increments the health counter), then reuses the
|
||||
/// false-result path — clearing in-flight, applying the false edge, and draining any
|
||||
/// pending evaluation — so a transient scheduler fault cannot permanently park the trigger.
|
||||
/// </summary>
|
||||
private void HandleExpressionEvalFailed(ExpressionEvalFailed msg)
|
||||
{
|
||||
LogExpressionError(msg.Cause);
|
||||
HandleExpressionEvalResult(new ExpressionEvalResult(false));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Applies a WhileTrue trigger's condition-state transition: on the
|
||||
/// false→true edge, fire once and start the re-fire timer; on the
|
||||
@@ -621,6 +638,15 @@ public class ScriptActor : ReceiveActor, IWithTimers
|
||||
/// the actor thread.
|
||||
/// </summary>
|
||||
private sealed record ExpressionEvalResult(bool Result);
|
||||
|
||||
/// <summary>
|
||||
/// Piped back to self when the off-dispatcher evaluation TASK itself faults
|
||||
/// (e.g. ObjectDisposedException from a disposed ScriptExecutionScheduler
|
||||
/// during shutdown) — the inner body's catch never sees that. Without this
|
||||
/// mapping the actor receives an unhandled Status.Failure and _evalInFlight
|
||||
/// stays true forever, permanently parking the expression trigger (N2).
|
||||
/// </summary>
|
||||
internal sealed record ExpressionEvalFailed(Exception Cause);
|
||||
}
|
||||
|
||||
// ── Trigger config types ──
|
||||
|
||||
@@ -18,7 +18,7 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Actors;
|
||||
/// Inbound: receives replicated operations from peer and applies to local SQLite.
|
||||
/// Uses fire-and-forget (Tell) — no ack wait per design.
|
||||
/// </summary>
|
||||
public class SiteReplicationActor : ReceiveActor
|
||||
public class SiteReplicationActor : ReceiveActor, IWithTimers
|
||||
{
|
||||
private readonly SiteStorageService _storage;
|
||||
private readonly StoreAndForwardStorage _sfStorage;
|
||||
@@ -32,6 +32,30 @@ public class SiteReplicationActor : ReceiveActor
|
||||
private readonly TimeSpan _configFetchRetryDelay;
|
||||
private Address? _peerAddress;
|
||||
|
||||
/// <summary>Akka timer scheduler injected by the framework via <see cref="IWithTimers"/>.</summary>
|
||||
public ITimerScheduler Timers { get; set; } = null!;
|
||||
|
||||
// ── Chunked-resync assembly (standby side; actor-thread only) ──
|
||||
private string? _assemblingResyncId;
|
||||
private int _assemblingTotalChunks;
|
||||
private bool _assemblingTruncated;
|
||||
private readonly Dictionary<int, List<StoreAndForwardMessage>> _assemblingChunks = new();
|
||||
private const string ResyncAssemblyTimerKey = "sf-resync-assembly-timeout";
|
||||
|
||||
/// <summary>How long a partial chunk assembly may wait for its missing chunks before
|
||||
/// being discarded (a lost chunk = lost resync; the next peer-track retries). Ctor
|
||||
/// test seam; production default 30 s.</summary>
|
||||
private readonly TimeSpan _resyncAssemblyTimeout;
|
||||
|
||||
// ── Resync delivery confirmation (active side; actor-thread only) ──
|
||||
private string? _pendingAckResyncId;
|
||||
private const string ResyncAckTimerKey = "sf-resync-ack-timeout";
|
||||
|
||||
/// <summary>How long the active node waits for the standby's <see cref="SfBufferResyncAck"/>
|
||||
/// before warning + counting the resync as unacknowledged (lost chunks / dead peer). Ctor
|
||||
/// test seam; production default 60 s.</summary>
|
||||
private readonly TimeSpan _resyncAckTimeout;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum rows an active node returns in a single anti-entropy resync snapshot.
|
||||
/// A standby whose buffer exceeded this (Truncated snapshot) resyncs the oldest
|
||||
@@ -39,6 +63,48 @@ public class SiteReplicationActor : ReceiveActor
|
||||
/// </summary>
|
||||
private const int MaxResyncRows = 10_000;
|
||||
|
||||
/// <summary>
|
||||
/// Estimated per-chunk payload budget for a resync snapshot. Akka remoting's default
|
||||
/// <c>maximum-frame-size</c> is 128 000 bytes and <c>BuildHocon</c> sets no override,
|
||||
/// so the monolithic <see cref="SfBufferSnapshot"/> is silently undeliverable for any
|
||||
/// realistic backlog (review 02 round 2, N2). 64 000 bytes leaves ≈50% headroom for
|
||||
/// the JSON envelope, CLR type manifests, and the non-payload columns.
|
||||
/// </summary>
|
||||
internal const int MaxResyncChunkBytes = 64_000;
|
||||
|
||||
/// <summary>Row cap per resync chunk (bounds a chunk even when every row is tiny).</summary>
|
||||
internal const int MaxResyncChunkRows = 200;
|
||||
|
||||
/// <summary>
|
||||
/// Splits a resync snapshot into chunks that fit Akka remoting's default
|
||||
/// 128 000-byte frame (review 02 round 2, N2): rows accumulate until the estimated
|
||||
/// payload budget or the row cap is hit. Estimation is payload-dominated
|
||||
/// (payload_json length + 512 bytes fixed overhead per row); a single row whose
|
||||
/// payload exceeds the budget ships alone (Warning at the call site). Order is
|
||||
/// preserved (oldest-first, matching GetAllMessagesAsync).
|
||||
/// </summary>
|
||||
internal static List<List<StoreAndForwardMessage>> ChunkForRemoting(
|
||||
IReadOnlyList<StoreAndForwardMessage> rows, int maxChunkBytes, int maxChunkRows)
|
||||
{
|
||||
var chunks = new List<List<StoreAndForwardMessage>>();
|
||||
var current = new List<StoreAndForwardMessage>();
|
||||
var currentBytes = 0;
|
||||
foreach (var row in rows)
|
||||
{
|
||||
var estimate = (row.PayloadJson?.Length ?? 0) + 512;
|
||||
if (current.Count > 0 && (currentBytes + estimate > maxChunkBytes || current.Count >= maxChunkRows))
|
||||
{
|
||||
chunks.Add(current);
|
||||
current = new List<StoreAndForwardMessage>();
|
||||
currentBytes = 0;
|
||||
}
|
||||
current.Add(row);
|
||||
currentBytes += estimate;
|
||||
}
|
||||
if (current.Count > 0) chunks.Add(current);
|
||||
return chunks;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new <see cref="SiteReplicationActor"/> and registers Akka message handlers.
|
||||
/// </summary>
|
||||
@@ -54,10 +120,11 @@ public class SiteReplicationActor : ReceiveActor
|
||||
/// config never crosses the intra-site Akka hop. Null on nodes/tests without a fetcher.
|
||||
/// </param>
|
||||
/// <param name="isActiveOverride">
|
||||
/// Test seam for the active-node check that gates the buffer-resync roles (a
|
||||
/// standby requests a resync, the active node answers). Null (production) uses
|
||||
/// the repo-standard leader+Up check via <see cref="Cluster"/> — swap point for
|
||||
/// plan 01's shared active-node helper.
|
||||
/// Active-node check that gates the buffer-resync roles (a standby requests a
|
||||
/// resync, the active node answers). Production wiring passes the Host's
|
||||
/// <c>IClusterNodeProvider.SelfIsPrimary</c> delegate (the same instance gating the
|
||||
/// S&F delivery sweep); null falls back to the shared oldest-Up evaluator
|
||||
/// (<see cref="Communication.ClusterState.ActiveNodeEvaluator"/>).
|
||||
/// </param>
|
||||
/// <param name="options">Site runtime options, including the config-fetch retry count; production defaults apply when null.</param>
|
||||
/// <param name="configFetchRetryDelay">Delay between config-fetch retry attempts; defaults to 2 seconds when null.</param>
|
||||
@@ -70,7 +137,9 @@ public class SiteReplicationActor : ReceiveActor
|
||||
IDeploymentConfigFetcher? configFetcher = null,
|
||||
Func<bool>? isActiveOverride = null,
|
||||
SiteRuntimeOptions? options = null,
|
||||
TimeSpan? configFetchRetryDelay = null)
|
||||
TimeSpan? configFetchRetryDelay = null,
|
||||
TimeSpan? resyncAssemblyTimeout = null,
|
||||
TimeSpan? resyncAckTimeout = null)
|
||||
{
|
||||
_storage = storage;
|
||||
_sfStorage = sfStorage;
|
||||
@@ -80,6 +149,8 @@ public class SiteReplicationActor : ReceiveActor
|
||||
_logger = logger;
|
||||
_cluster = Cluster.Get(Context.System);
|
||||
_isActive = isActiveOverride ?? DefaultIsActive;
|
||||
_resyncAssemblyTimeout = resyncAssemblyTimeout ?? TimeSpan.FromSeconds(30);
|
||||
_resyncAckTimeout = resyncAckTimeout ?? TimeSpan.FromSeconds(60);
|
||||
// UA2: bound the standby's replicated-config fetch retries. At least one
|
||||
// attempt always runs; the fixed inter-attempt delay is a test seam
|
||||
// (production default 2 s).
|
||||
@@ -110,7 +181,30 @@ public class SiteReplicationActor : ReceiveActor
|
||||
|
||||
// Anti-entropy — full S&F buffer resync on peer (re)join
|
||||
Receive<RequestSfBufferResync>(HandleRequestSfBufferResync);
|
||||
Receive<SfBufferSnapshot>(HandleSfBufferSnapshot);
|
||||
Receive<SfResyncSnapshotLoaded>(HandleSfResyncSnapshotLoaded);
|
||||
Receive<SfBufferSnapshotChunk>(HandleSfBufferSnapshotChunk);
|
||||
Receive<ResyncAssemblyTimedOut>(HandleResyncAssemblyTimedOut);
|
||||
Receive<SfBufferResyncAck>(msg =>
|
||||
{
|
||||
if (msg.ResyncId == _pendingAckResyncId)
|
||||
{
|
||||
_pendingAckResyncId = null;
|
||||
Timers.Cancel(ResyncAckTimerKey);
|
||||
}
|
||||
ScadaBridgeTelemetry.RecordSfResyncCompleted();
|
||||
_logger.LogInformation("S&F resync {ResyncId} acknowledged by standby: {Rows} row(s) applied",
|
||||
msg.ResyncId, msg.RowCount);
|
||||
});
|
||||
Receive<ResyncAckTimedOut>(msg =>
|
||||
{
|
||||
if (msg.ResyncId != _pendingAckResyncId) return;
|
||||
_pendingAckResyncId = null;
|
||||
ScadaBridgeTelemetry.RecordSfResyncAckMissing();
|
||||
_logger.LogWarning(
|
||||
"S&F resync {ResyncId} was never acknowledged within {Window} — snapshot chunks may have been lost (frame drop / dead peer); the next peer-track retries",
|
||||
msg.ResyncId, _resyncAckTimeout);
|
||||
});
|
||||
Receive<SfBufferSnapshot>(HandleSfBufferSnapshot); // legacy monolithic handler — retained for rolling compat
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -181,21 +275,17 @@ public class SiteReplicationActor : ReceiveActor
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Repo-standard active-node check: this node is active when it is the current
|
||||
/// cluster leader AND its own <see cref="MemberStatus"/> is
|
||||
/// <see cref="MemberStatus.Up"/>. Mirrors <c>SiteCommunicationActor.DefaultIsActiveCheck</c>
|
||||
/// (swap point for plan 01's shared helper). Any other state reports standby —
|
||||
/// safe-by-default.
|
||||
/// Repo-standard active-node check: this node is active when it is the OLDEST Up
|
||||
/// member carrying the site role — the same oldest-Up semantics as the S&F delivery
|
||||
/// gate (IClusterNodeProvider.SelfIsPrimary → ClusterActivityEvaluator → shared
|
||||
/// ActiveNodeEvaluator). NEVER the cluster leader: leadership is lowest-address and
|
||||
/// diverges from singleton/delivery placement permanently after the lower-address
|
||||
/// node restarts — the divergence that made the delivering node wipe its own live
|
||||
/// buffer via a wrong-direction resync (review 02 round 2, N1 Critical). Any other
|
||||
/// state reports standby — safe-by-default.
|
||||
/// </summary>
|
||||
private bool DefaultIsActive()
|
||||
{
|
||||
var self = _cluster.SelfMember;
|
||||
if (self.Status != MemberStatus.Up)
|
||||
return false;
|
||||
|
||||
var leader = _cluster.State.Leader;
|
||||
return leader != null && leader == self.Address;
|
||||
}
|
||||
private bool DefaultIsActive() =>
|
||||
Communication.ClusterState.ActiveNodeEvaluator.SelfIsOldestUp(_cluster, _siteRole);
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates the active-node check, treating a throwing check as standby
|
||||
@@ -391,9 +481,10 @@ public class SiteReplicationActor : ReceiveActor
|
||||
|
||||
/// <summary>
|
||||
/// Active-node side of the anti-entropy resync: answers a standby's
|
||||
/// <see cref="RequestSfBufferResync"/> with a full-buffer <see cref="SfBufferSnapshot"/>
|
||||
/// (up to <see cref="MaxResyncRows"/> oldest rows). A non-active node ignores the
|
||||
/// request — only the authoritative node may answer.
|
||||
/// <see cref="RequestSfBufferResync"/> with a sequence of byte-budgeted
|
||||
/// <see cref="SfBufferSnapshotChunk"/>s (up to <see cref="MaxResyncRows"/> oldest rows).
|
||||
/// A non-active node ignores the request — only the authoritative node may answer.
|
||||
/// The snapshot is piped back to Self so chunking + ack bookkeeping stays actor-safe.
|
||||
/// </summary>
|
||||
private void HandleRequestSfBufferResync(RequestSfBufferResync msg)
|
||||
{
|
||||
@@ -405,9 +496,37 @@ public class SiteReplicationActor : ReceiveActor
|
||||
|
||||
var replyTo = Sender;
|
||||
_sfStorage.GetAllMessagesAsync(MaxResyncRows).PipeTo(
|
||||
replyTo,
|
||||
Self,
|
||||
success: result => new SfBufferSnapshot(result.Messages, result.Truncated));
|
||||
failure: ex => new Status.Failure(ex),
|
||||
success: result => new SfResyncSnapshotLoaded(replyTo, result.Messages, result.Truncated));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Active-node continuation: the resync snapshot finished loading; chunk it to fit the
|
||||
/// remoting frame and send the sequenced chunks to the requester (all sharing one
|
||||
/// resyncId). Task 7 arms the ack-timeout here.
|
||||
/// </summary>
|
||||
private void HandleSfResyncSnapshotLoaded(SfResyncSnapshotLoaded msg)
|
||||
{
|
||||
var chunks = ChunkForRemoting(msg.Messages, MaxResyncChunkBytes, MaxResyncChunkRows);
|
||||
if (chunks.Count == 0) chunks.Add(new List<StoreAndForwardMessage>()); // empty buffer still resyncs (clears the standby)
|
||||
var resyncId = Guid.NewGuid().ToString("N");
|
||||
for (var i = 0; i < chunks.Count; i++)
|
||||
{
|
||||
if (chunks[i].Count == 1 && (chunks[i][0].PayloadJson?.Length ?? 0) + 512 > MaxResyncChunkBytes)
|
||||
_logger.LogWarning(
|
||||
"Resync row {Id} alone exceeds the chunk budget ({Bytes}B payload); sending solo — it may exceed the remoting frame",
|
||||
chunks[i][0].Id, chunks[i][0].PayloadJson?.Length ?? 0);
|
||||
msg.ReplyTo.Tell(new SfBufferSnapshotChunk(resyncId, i + 1, chunks.Count, chunks[i], msg.Truncated), Self);
|
||||
}
|
||||
_logger.LogInformation(
|
||||
"Answered S&F resync request with {Rows} row(s) in {Chunks} chunk(s), resyncId={ResyncId}",
|
||||
msg.Messages.Count, chunks.Count, resyncId);
|
||||
// Arm the ack window: absence of an SfBufferResyncAck within it surfaces as a
|
||||
// Warning + counter (the silent-loss mode N2 flagged). Single-outstanding-resync
|
||||
// bookkeeping: a new request supersedes by overwriting the id and restarting the timer.
|
||||
_pendingAckResyncId = resyncId;
|
||||
Timers.StartSingleTimer(ResyncAckTimerKey, new ResyncAckTimedOut(resyncId), _resyncAckTimeout);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -435,13 +554,120 @@ public class SiteReplicationActor : ReceiveActor
|
||||
_logger.LogInformation(
|
||||
"Applying S&F buffer resync snapshot ({Count} rows), replacing local buffer", msg.Messages.Count);
|
||||
|
||||
_sfStorage.ReplaceAllAsync(msg.Messages)
|
||||
Task.Run(async () =>
|
||||
{
|
||||
// Belt-and-braces (N1): re-check at apply time. ReplaceAllAsync discards
|
||||
// every in-flight row (StoreAndForwardStorage.cs "Never call on an active
|
||||
// node"); a flip between message receipt and this point must abort.
|
||||
if (SafeIsActive())
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Discarding S&F buffer resync snapshot: this node became active before apply");
|
||||
return;
|
||||
}
|
||||
await _sfStorage.ReplaceAllAsync(msg.Messages);
|
||||
})
|
||||
.ContinueWith(t =>
|
||||
{
|
||||
if (t.IsFaulted)
|
||||
_logger.LogError(t.Exception, "Failed to apply S&F buffer resync snapshot");
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Standby-node side of the chunked anti-entropy resync: accumulates the chunks of one
|
||||
/// <c>ResyncId</c>, and once all have arrived, assembles them in sequence order and
|
||||
/// replaces the local buffer atomically, then acks. A new <c>ResyncId</c> discards any
|
||||
/// stale partial assembly (review 02 round 2, N2). An active node ignores chunks.
|
||||
/// </summary>
|
||||
private void HandleSfBufferSnapshotChunk(SfBufferSnapshotChunk msg)
|
||||
{
|
||||
if (SafeIsActive())
|
||||
{
|
||||
_logger.LogDebug("Ignoring S&F resync chunk — this node is active");
|
||||
return;
|
||||
}
|
||||
|
||||
if (_assemblingResyncId != msg.ResyncId)
|
||||
{
|
||||
if (_assemblingResyncId != null)
|
||||
_logger.LogWarning(
|
||||
"Discarding partial S&F resync assembly {Old} ({Have}/{Want} chunks): a new resync {New} superseded it",
|
||||
_assemblingResyncId, _assemblingChunks.Count, _assemblingTotalChunks, msg.ResyncId);
|
||||
_assemblingResyncId = msg.ResyncId;
|
||||
_assemblingTotalChunks = msg.TotalChunks;
|
||||
_assemblingTruncated = msg.Truncated;
|
||||
_assemblingChunks.Clear();
|
||||
}
|
||||
|
||||
_assemblingChunks[msg.Sequence] = msg.Messages;
|
||||
Timers.StartSingleTimer(ResyncAssemblyTimerKey, new ResyncAssemblyTimedOut(msg.ResyncId), _resyncAssemblyTimeout);
|
||||
|
||||
if (_assemblingChunks.Count < _assemblingTotalChunks)
|
||||
return;
|
||||
|
||||
// Complete: assemble in sequence order and apply atomically.
|
||||
var assembled = Enumerable.Range(1, _assemblingTotalChunks)
|
||||
.SelectMany(seq => _assemblingChunks[seq])
|
||||
.ToList();
|
||||
var resyncId = _assemblingResyncId!;
|
||||
var truncated = _assemblingTruncated;
|
||||
_assemblingResyncId = null;
|
||||
_assemblingChunks.Clear();
|
||||
Timers.Cancel(ResyncAssemblyTimerKey);
|
||||
|
||||
if (truncated)
|
||||
_logger.LogWarning(
|
||||
"S&F buffer resync snapshot truncated at {Cap} rows; divergence beyond the cap drains naturally",
|
||||
MaxResyncRows);
|
||||
_logger.LogInformation(
|
||||
"Applying chunked S&F resync {ResyncId} ({Count} rows), replacing local buffer", resyncId, assembled.Count);
|
||||
|
||||
// KNOWN, ACCEPTED race (review 02 round 2, N5 — do NOT "fix" this into something
|
||||
// worse): a replicated Remove sent after the active node read its snapshot but
|
||||
// before the snapshot's chunks is ordered BEFORE them on the wire (same
|
||||
// sender/receiver pair), so this apply can re-add the removed row → an orphan
|
||||
// Pending row that a later failover re-delivers ONCE. Bounded, self-correcting at
|
||||
// the next resync, and inherent to no-ack replication; a delivered-side dedup or
|
||||
// op-sequencing scheme would cost far more than one rare duplicate.
|
||||
var replyTo = Sender;
|
||||
Task.Run(async () =>
|
||||
{
|
||||
if (SafeIsActive()) // belt-and-braces, mirrors the monolithic path (T3)
|
||||
{
|
||||
_logger.LogWarning("Discarding chunked S&F resync {ResyncId}: node became active before apply", resyncId);
|
||||
return;
|
||||
}
|
||||
await _sfStorage.ReplaceAllAsync(assembled);
|
||||
replyTo.Tell(new SfBufferResyncAck(resyncId, assembled.Count));
|
||||
})
|
||||
.ContinueWith(t =>
|
||||
{
|
||||
if (t.IsFaulted)
|
||||
_logger.LogError(t.Exception, "Failed to apply chunked S&F resync {ResyncId}", resyncId);
|
||||
});
|
||||
}
|
||||
|
||||
private void HandleResyncAssemblyTimedOut(ResyncAssemblyTimedOut msg)
|
||||
{
|
||||
if (_assemblingResyncId != msg.ResyncId) return; // superseded already
|
||||
_logger.LogWarning(
|
||||
"S&F resync assembly {ResyncId} timed out with {Have}/{Want} chunks — discarding partial (a lost chunk; the next peer-track retries)",
|
||||
msg.ResyncId, _assemblingChunks.Count, _assemblingTotalChunks);
|
||||
ScadaBridgeTelemetry.RecordReplicationFailure();
|
||||
_assemblingResyncId = null;
|
||||
_assemblingChunks.Clear();
|
||||
}
|
||||
|
||||
/// <summary>Internal: the resync snapshot finished loading; chunk and send to the requester.</summary>
|
||||
internal sealed record SfResyncSnapshotLoaded(
|
||||
IActorRef ReplyTo, List<StoreAndForwardMessage> Messages, bool Truncated);
|
||||
|
||||
/// <summary>Internal: a partial chunk assembly for <paramref name="ResyncId"/> exceeded its window.</summary>
|
||||
internal sealed record ResyncAssemblyTimedOut(string ResyncId);
|
||||
|
||||
/// <summary>Internal: the active node's ack window for <paramref name="ResyncId"/> expired.</summary>
|
||||
internal sealed record ResyncAckTimedOut(string ResyncId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -458,3 +684,24 @@ public sealed record RequestSfBufferResync;
|
||||
/// Akka remoting; the message list rides the default serializer.
|
||||
/// </summary>
|
||||
public sealed record SfBufferSnapshot(List<StoreAndForwardMessage> Messages, bool Truncated);
|
||||
|
||||
/// <summary>
|
||||
/// Active→standby: one sequenced chunk of a full-buffer anti-entropy snapshot
|
||||
/// (review 02 round 2, N2 — the monolithic <see cref="SfBufferSnapshot"/> exceeds Akka
|
||||
/// remoting's default 128 000-byte frame for any realistic backlog). All chunks of one
|
||||
/// resync share <paramref name="ResyncId"/>; <paramref name="Sequence"/> is 1-based up to
|
||||
/// <paramref name="TotalChunks"/>. Additive message — the legacy monolithic snapshot
|
||||
/// handler is retained for rolling upgrades. Crosses intra-site Akka remoting (NOT
|
||||
/// ClusterClient — ClusterClientContractLockTests is intentionally not involved).
|
||||
/// </summary>
|
||||
public sealed record SfBufferSnapshotChunk(
|
||||
string ResyncId, int Sequence, int TotalChunks,
|
||||
List<StoreAndForwardMessage> Messages, bool Truncated);
|
||||
|
||||
/// <summary>
|
||||
/// Standby→active: delivery confirmation — the standby assembled all chunks of
|
||||
/// <paramref name="ResyncId"/> and applied them atomically (<paramref name="RowCount"/>
|
||||
/// rows installed). Absence within the ack window is surfaced by the active node
|
||||
/// (Warning + counter) — the silent-loss mode N2 flagged.
|
||||
/// </summary>
|
||||
public sealed record SfBufferResyncAck(string ResyncId, int RowCount);
|
||||
|
||||
@@ -14,8 +14,11 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Deployment;
|
||||
/// deployment; no partial state is applied).
|
||||
///
|
||||
/// Because it holds no actor state and only calls the (stateless)
|
||||
/// <see cref="ScriptCompilationService"/>, it is safe to run off the singleton
|
||||
/// mailbox on a <c>Task.Run</c> thread — the deploy path never blocks on Roslyn.
|
||||
/// <see cref="ScriptCompilationService"/>, it is pure and thread-safe; the Deployment
|
||||
/// Manager nevertheless invokes it <em>synchronously on the singleton's actor thread</em>
|
||||
/// as a pure prefix step, preserving mailbox-FIFO deploy ordering (see
|
||||
/// <see cref="Actors.DeploymentManagerActor.HandleDeploy"/>). Compiles of unchanged
|
||||
/// script bodies are deduped by the process-wide compile cache.
|
||||
/// </summary>
|
||||
public sealed class DeployCompileValidator
|
||||
{
|
||||
|
||||
@@ -13,6 +13,13 @@ namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
/// <see cref="ScriptTrustValidator"/>; this service keeps the real
|
||||
/// execution-path compile of the script against <see cref="ScriptGlobals"/> /
|
||||
/// <see cref="TriggerExpressionGlobals"/>.
|
||||
///
|
||||
/// Compile results — success AND failure — are memoised process-wide in
|
||||
/// <see cref="SiteScriptCompileCache"/> (keyed on code hash + globals type), so an
|
||||
/// unchanged script body compiles once per process lifetime and the deploy gate,
|
||||
/// the Instance Actor's PreStart recompile, and SharedScriptLibrary all reuse one
|
||||
/// compiled <c>Script<object?></c>. Error text is name-free by construction, so a
|
||||
/// cached failure renders correctly for every caller.
|
||||
/// </summary>
|
||||
public class ScriptCompilationService
|
||||
{
|
||||
@@ -92,10 +99,33 @@ public class ScriptCompilationService
|
||||
=> CompileCore(name, expression, typeof(TriggerExpressionGlobals));
|
||||
|
||||
/// <summary>
|
||||
/// Shared compilation path: validates the trust model, builds the script
|
||||
/// against the given globals type, and returns the compiled result.
|
||||
/// Shared compilation funnel for every site-side compile (the deploy gate, the
|
||||
/// Instance Actor's PreStart recompile, and SharedScriptLibrary). Results — success
|
||||
/// AND failure — are memoised process-wide in <see cref="SiteScriptCompileCache"/>,
|
||||
/// keyed on code hash + globals type: an unchanged script body compiles once per
|
||||
/// process lifetime and the compiled <c>Script<object?></c> is shared across all
|
||||
/// call sites (N4). Cached error text is name-free by construction (Roslyn diagnostics /
|
||||
/// the compile-exception message never embed the caller's script name), so a cached
|
||||
/// failure renders correctly for every caller.
|
||||
/// </summary>
|
||||
private ScriptCompilationResult CompileCore(string name, string code, Type globalsType)
|
||||
{
|
||||
var result = SiteScriptCompileCache.GetOrAdd(code, globalsType,
|
||||
() => CompileUncached(name, code, globalsType));
|
||||
if (!result.IsSuccess)
|
||||
_logger.LogDebug(
|
||||
"Script {Script}: returning cached compile failure ({ErrorCount} error(s))",
|
||||
name, result.Errors.Count); // miss-path logs name the FIRST caller; keep this caller visible
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The uncached compile: validates the trust model, builds the script against the
|
||||
/// given globals type, and returns the compiled result. The verdict is deterministic
|
||||
/// on the code (plus the compile-time-static trust policy and globals surface), so it
|
||||
/// is safe to cache alongside the compiled script.
|
||||
/// </summary>
|
||||
private ScriptCompilationResult CompileUncached(string name, string code, Type globalsType)
|
||||
{
|
||||
// Validate trust model
|
||||
var violations = ValidateTrustModel(code);
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Scripts;
|
||||
|
||||
/// <summary>
|
||||
/// Process-wide cache of site-side script <see cref="ScriptCompilationResult"/>s, keyed on
|
||||
/// the globals type plus a SHA-256 hash of the script code. Unlike the TemplateEngine's
|
||||
/// <c>ScriptCompileVerdictCache</c> (which caches central-validation <em>verdicts</em>), this
|
||||
/// caches the full compiled result — the immutable Roslyn <c>Script<object?></c> itself —
|
||||
/// so the synchronous deploy gate and the <c>InstanceActor.PreStart</c> recompile share one
|
||||
/// Roslyn compile per script body per process lifetime (N4).
|
||||
///
|
||||
/// <para>Soundness rests on three invariants:</para>
|
||||
/// <list type="number">
|
||||
/// <item>A compile result is a pure function of (code, globals type, compile-time-static
|
||||
/// trust policy) — none of which vary at runtime for a given key.</item>
|
||||
/// <item>The stored error text is name-free (Roslyn <c>Diagnostic.GetMessage()</c> /
|
||||
/// <c>"Compilation exception: …"</c>); callers prefix their own script-name context, so a
|
||||
/// shared cached failure renders correctly for every caller. <em>Failures are cached too</em> —
|
||||
/// a bad script body fails identically for every caller.</item>
|
||||
/// <item>Roslyn <c>Script<T></c> is immutable and thread-safe (<c>RunAsync</c> creates
|
||||
/// per-run state), so one compiled instance may be shared across actors and threads.</item>
|
||||
/// </list>
|
||||
///
|
||||
/// <para>
|
||||
/// Bounded at <see cref="MaxEntries"/> entries — deliberately smaller than the verdict cache's
|
||||
/// 4096 because entries pin compiled assemblies, not just verdict strings. On overflow the cache
|
||||
/// is cleared wholesale (results are recomputable, so a coarse reset avoids eviction bookkeeping).
|
||||
/// <see cref="Hits"/>/<see cref="Count"/>/<see cref="Clear"/> are exposed for tests and diagnostics.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
internal static class SiteScriptCompileCache
|
||||
{
|
||||
/// <summary>Upper bound on cached entries; the cache is cleared wholesale on overflow.</summary>
|
||||
internal const int MaxEntries = 1024;
|
||||
|
||||
private static readonly ConcurrentDictionary<string, ScriptCompilationResult> Cache = new();
|
||||
private static long _hits;
|
||||
|
||||
/// <summary>Number of cache hits observed since the last <see cref="Clear"/>.</summary>
|
||||
public static long Hits => Interlocked.Read(ref _hits);
|
||||
|
||||
/// <summary>Current number of cached compile results.</summary>
|
||||
public static int Count => Cache.Count;
|
||||
|
||||
/// <summary>
|
||||
/// Returns the cached compile result for <paramref name="code"/> against
|
||||
/// <paramref name="globalsType"/>, or computes it via <paramref name="factory"/> and caches
|
||||
/// it on a miss. A hit increments <see cref="Hits"/>. Both success and failure results are
|
||||
/// cached — the error text is name-free by construction.
|
||||
/// </summary>
|
||||
/// <param name="code">The script source code to look up (hashed to form the cache key).</param>
|
||||
/// <param name="globalsType">The Roslyn globals surface the script compiles against; part of the key so identical source under different globals stays distinct.</param>
|
||||
/// <param name="factory">Computes the compile result on a cache miss.</param>
|
||||
/// <returns>The cached or newly computed compile result.</returns>
|
||||
public static ScriptCompilationResult GetOrAdd(string code, Type globalsType, Func<ScriptCompilationResult> factory)
|
||||
{
|
||||
var key = globalsType.FullName + ":" + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(code)));
|
||||
|
||||
if (Cache.TryGetValue(key, out var result))
|
||||
{
|
||||
Interlocked.Increment(ref _hits);
|
||||
return result;
|
||||
}
|
||||
|
||||
result = factory();
|
||||
|
||||
// Coarse bound: on overflow drop everything rather than track evictions.
|
||||
if (Cache.Count >= MaxEntries)
|
||||
Cache.Clear();
|
||||
|
||||
Cache[key] = result;
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>Clears all cached compile results and resets the hit counter.</summary>
|
||||
public static void Clear()
|
||||
{
|
||||
Cache.Clear();
|
||||
Interlocked.Exchange(ref _hits, 0);
|
||||
}
|
||||
}
|
||||
@@ -60,6 +60,19 @@ public static class ServiceCollectionExtensions
|
||||
});
|
||||
services.AddSingleton<ISiteStreamSubscriber>(sp => sp.GetRequiredService<SiteStreamManager>());
|
||||
|
||||
// Site-local cached-operation tracking store — the SQLite source of truth
|
||||
// that Tracking.Status(TrackedOperationId) reads and the cached-call telemetry
|
||||
// forwarder writes. Registered HERE (site-only) so the two comments that claim
|
||||
// "the operational tracking store is registered by AddSiteRuntime" are true
|
||||
// again (AuditLog SCE + AkkaHostedService). Before this (arch-review 08 round 2
|
||||
// NF4/T8) it had no DI registration anywhere in src/, so every consumer resolved
|
||||
// it as null and silently ran degraded (no cached-drain, no PullSiteCalls
|
||||
// reconciliation, Tracking.Status audit-only). Central roots never call
|
||||
// AddSiteRuntime, so this stays absent on central — matching the SITE-ONLY
|
||||
// contract. OperationTrackingOptions is bound + validated by the Host's
|
||||
// SiteServiceRegistration site-options block.
|
||||
services.AddSingleton<Commons.Interfaces.IOperationTrackingStore, Tracking.OperationTrackingStore>();
|
||||
|
||||
// Site-local repository implementations backed by SQLite
|
||||
services.AddScoped<IExternalSystemRepository, SiteExternalSystemRepository>();
|
||||
|
||||
|
||||
@@ -56,5 +56,14 @@ public sealed class SiteRuntimeOptionsValidator : OptionsValidatorBase<SiteRunti
|
||||
builder.RequireThat(options.ConfigFetchRetryCount >= 0,
|
||||
$"ScadaBridge:SiteRuntime:ConfigFetchRetryCount must be >= 0 " +
|
||||
$"(was {options.ConfigFetchRetryCount}).");
|
||||
|
||||
builder.RequireThat(options.TagSubscribeRetryIntervalMs > 0,
|
||||
$"ScadaBridge:SiteRuntime:TagSubscribeRetryIntervalMs must be greater than 0 " +
|
||||
$"(was {options.TagSubscribeRetryIntervalMs}); a zero interval hot-loops the tag-subscribe " +
|
||||
"retry and a negative one throws inside the Instance Actor's retry scheduling.");
|
||||
|
||||
builder.RequireThat(options.StuckScriptGraceMs >= 0,
|
||||
$"ScadaBridge:SiteRuntime:StuckScriptGraceMs must be >= 0 " +
|
||||
$"(was {options.StuckScriptGraceMs}); a negative grace throws inside the stuck-script watchdog's delay.");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
using ZB.MOM.WW.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.SiteRuntime.Tracking;
|
||||
|
||||
/// <summary>
|
||||
/// Validates <see cref="OperationTrackingOptions"/> at host startup (arch-review
|
||||
/// 08 round 2 NF4). <see cref="OperationTrackingStore"/> opens the SQLite
|
||||
/// connection string in its constructor, so an empty/blank connection string
|
||||
/// must fail the host at boot rather than throw opaquely on first resolve. The
|
||||
/// retention window must be positive — a zero would make the host's terminal-row
|
||||
/// purge delete every tracking row.
|
||||
/// </summary>
|
||||
public sealed class OperationTrackingOptionsValidator : OptionsValidatorBase<OperationTrackingOptions>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override void Validate(ValidationBuilder builder, OperationTrackingOptions options)
|
||||
{
|
||||
builder.RequireThat(!string.IsNullOrWhiteSpace(options.ConnectionString),
|
||||
$"ScadaBridge:OperationTracking:{nameof(OperationTrackingOptions.ConnectionString)} " +
|
||||
"must be a non-empty SQLite connection string.");
|
||||
|
||||
builder.RequireThat(options.RetentionDays > 0,
|
||||
$"ScadaBridge:OperationTracking:{nameof(OperationTrackingOptions.RetentionDays)} " +
|
||||
$"({options.RetentionDays}) must be > 0.");
|
||||
}
|
||||
}
|
||||
@@ -31,5 +31,13 @@ public sealed class StoreAndForwardOptionsValidator : OptionsValidatorBase<Store
|
||||
builder.RequireThat(options.DefaultMaxRetries >= 0,
|
||||
$"ScadaBridge:StoreAndForward:DefaultMaxRetries must be >= 0 " +
|
||||
$"(was {options.DefaultMaxRetries}).");
|
||||
|
||||
builder.RequireThat(options.SweepBatchLimit >= 0,
|
||||
$"ScadaBridge:StoreAndForward:SweepBatchLimit must be >= 0 " +
|
||||
$"(was {options.SweepBatchLimit}); it bounds due rows per retry sweep — 0 means unlimited (legacy).");
|
||||
|
||||
builder.RequireThat(options.SweepTargetParallelism >= 1,
|
||||
$"ScadaBridge:StoreAndForward:SweepTargetParallelism must be >= 1 " +
|
||||
$"(was {options.SweepTargetParallelism}); it caps concurrent (category,target) sweep lanes — 1 means serial.");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -394,7 +394,7 @@ public class StoreAndForwardService
|
||||
});
|
||||
|
||||
_retryTimer = new Timer(
|
||||
_ => Volatile.Write(ref _sweepTask, RetryPendingMessagesAsync()),
|
||||
_ => KickSweep(),
|
||||
null,
|
||||
_options.RetryTimerInterval,
|
||||
_options.RetryTimerInterval);
|
||||
@@ -658,17 +658,37 @@ public class StoreAndForwardService
|
||||
/// <summary>
|
||||
/// Kicks a background retry sweep now (fire-and-forget). No-op before
|
||||
/// <see cref="StartAsync"/> (storage may be uninitialized and the timer, whose
|
||||
/// presence gates this, is not yet set). Overlap-safe: the sweep's
|
||||
/// <see cref="_retryInProgress"/> CAS makes a concurrent kick a cheap no-op.
|
||||
/// presence gates this, is not yet set). Overlap-safe: publishes into
|
||||
/// <see cref="_sweepTask"/> only when the CAS is won — see <see cref="KickSweep"/>.
|
||||
/// </summary>
|
||||
public void TriggerSweep()
|
||||
{
|
||||
if (_retryTimer == null) return;
|
||||
Volatile.Write(ref _sweepTask, RetryPendingMessagesAsync());
|
||||
KickSweep();
|
||||
}
|
||||
|
||||
/// <summary>The current drain handle — test seam for the N3 clobber regression.</summary>
|
||||
internal Task? CurrentSweepTaskForTest => Volatile.Read(ref _sweepTask);
|
||||
|
||||
/// <summary>
|
||||
/// Starts a sweep IF none is in flight, publishing the new task into
|
||||
/// <see cref="_sweepTask"/> only when this call wins the <see cref="_retryInProgress"/>
|
||||
/// CAS. A kick that loses the CAS returns without touching <see cref="_sweepTask"/> —
|
||||
/// pre-fix it overwrote the drain handle with an instantly-completed no-op, so
|
||||
/// <see cref="StopAsync"/> proceeded with disposal under a still-running sweep
|
||||
/// (review 02 round 2, N3) — defeating the drain exactly when a sweep outlives a tick.
|
||||
/// </summary>
|
||||
private void KickSweep()
|
||||
{
|
||||
if (Interlocked.CompareExchange(ref _retryInProgress, 1, 0) != 0)
|
||||
return;
|
||||
Volatile.Write(ref _sweepTask, RunSweepOwnedAsync());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Background retry sweep. Processes all pending messages that are due for retry.
|
||||
/// Self-CASes for direct callers (existing tests); the entry CAS is skipped when
|
||||
/// ownership is already held by <see cref="KickSweep"/>.
|
||||
/// </summary>
|
||||
/// <returns>A task representing the asynchronous retry sweep.</returns>
|
||||
internal async Task RetryPendingMessagesAsync()
|
||||
@@ -676,7 +696,16 @@ public class StoreAndForwardService
|
||||
// Prevent overlapping retry sweeps
|
||||
if (Interlocked.CompareExchange(ref _retryInProgress, 1, 0) != 0)
|
||||
return;
|
||||
await RunSweepOwnedAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The actual retry sweep body. The caller MUST already own the
|
||||
/// <see cref="_retryInProgress"/> flag (won the CAS); this method releases it in its
|
||||
/// finally. Never call directly without holding ownership.
|
||||
/// </summary>
|
||||
private async Task RunSweepOwnedAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
var gate = _deliveryGate;
|
||||
|
||||
@@ -5,14 +5,25 @@ using System.Text;
|
||||
namespace ZB.MOM.WW.ScadaBridge.TemplateEngine.Validation;
|
||||
|
||||
/// <summary>
|
||||
/// Process-wide cache of script compile verdicts, keyed on a SHA-256 hash of the
|
||||
/// script code. The verdict (clean/compiles, or the name-free error text) is a pure
|
||||
/// function of the code plus the compile-time-static <c>ScriptTrustPolicy</c> and
|
||||
/// globals surface, so it is sound to memoise it across the whole process: an
|
||||
/// Process-wide cache of script compile verdicts, keyed on the pair
|
||||
/// (globals surface, SHA-256 hash of the script code). The verdict (clean/compiles,
|
||||
/// or the name-free error text) is a pure function of the code plus the
|
||||
/// compile-time-static <c>ScriptTrustPolicy</c> AND the globals surface it was
|
||||
/// compiled against, so it is sound to memoise it across the whole process: an
|
||||
/// unchanged script runs the forbidden-API analysis and the Roslyn compile exactly
|
||||
/// once, removing both the repeat CPU cost and the repeat compiled-assembly load.
|
||||
///
|
||||
/// <para>
|
||||
/// The globals surface is part of the key because a verdict is NOT interchangeable
|
||||
/// across surfaces: a trigger expression that compiles clean against
|
||||
/// <c>TriggerCompileSurface</c> is not vetted for a <c>ScriptCompileSurface</c>
|
||||
/// script body (different globals resolve different identifiers), so a code-only key
|
||||
/// could return a stale "clean" for code never compiled against the caller's surface.
|
||||
/// Keeping the surface in the key makes that cross-surface reuse structurally
|
||||
/// impossible even as more compile surfaces gain cache entries.
|
||||
/// </para>
|
||||
///
|
||||
/// <para>
|
||||
/// The stored error is deliberately <em>name-free</em> — the caller's script name is
|
||||
/// formatted into the returned message at read time, so two callers submitting the
|
||||
/// same bad code each see their own name in the failure text while sharing one verdict.
|
||||
@@ -40,17 +51,22 @@ public static class ScriptCompileVerdictCache
|
||||
public static int Count => Cache.Count;
|
||||
|
||||
/// <summary>
|
||||
/// Returns the cached verdict for <paramref name="code"/>, or computes it via
|
||||
/// <paramref name="factory"/> and caches it on a miss. A hit increments
|
||||
/// <see cref="Hits"/>. The verdict's error text must be name-free — the caller
|
||||
/// formats the script name in on return.
|
||||
/// Returns the cached verdict for the pair (<paramref name="surface"/>,
|
||||
/// <paramref name="code"/>), or computes it via <paramref name="factory"/> and
|
||||
/// caches it on a miss. A hit increments <see cref="Hits"/>. The verdict's error
|
||||
/// text must be name-free — the caller formats the script name in on return.
|
||||
/// </summary>
|
||||
/// <param name="surface">
|
||||
/// The globals-surface discriminator (e.g. <c>nameof(ScriptCompileSurface)</c> or
|
||||
/// <c>nameof(TriggerCompileSurface)</c>). Part of the key so a verdict computed
|
||||
/// against one surface is never reused for another.
|
||||
/// </param>
|
||||
/// <param name="code">The script source code to look up (hashed to form the cache key).</param>
|
||||
/// <param name="factory">Computes the verdict on a cache miss.</param>
|
||||
/// <returns>The cached or newly computed compile verdict.</returns>
|
||||
public static (bool Ok, string? Error) GetOrAdd(string code, Func<(bool Ok, string? Error)> factory)
|
||||
public static (bool Ok, string? Error) GetOrAdd(string surface, string code, Func<(bool Ok, string? Error)> factory)
|
||||
{
|
||||
var key = Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(code)));
|
||||
var key = surface + ":" + Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(code)));
|
||||
|
||||
if (Cache.TryGetValue(key, out var verdict))
|
||||
{
|
||||
|
||||
@@ -39,7 +39,7 @@ public class ScriptCompiler
|
||||
// globals surface are compile-time static), so memoise it process-wide: an
|
||||
// unchanged script compiles exactly once per process. The cached error is
|
||||
// name-free — the script name is formatted in below at read time.
|
||||
var (ok, error) = ScriptCompileVerdictCache.GetOrAdd(code, () =>
|
||||
var (ok, error) = ScriptCompileVerdictCache.GetOrAdd(nameof(ScriptCompileSurface), code, () =>
|
||||
{
|
||||
// Authoritative forbidden-API verdict first — a security violation must
|
||||
// gate the script regardless of whether it otherwise compiles. Report
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user