Compare commits
103 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| bb7aa6209f | |||
| 3ef56be2dd | |||
| e913dab5db | |||
| aac79579ab | |||
| 9871d4772d | |||
| 53881c6220 | |||
| 756296886b | |||
| b0f5941e46 | |||
| 38dd7678f2 | |||
| 10406a3541 | |||
| e245237c2b | |||
| e23f816bfb | |||
| d44fe1d6b5 | |||
| 59420d8568 | |||
| 94dbe9f1af | |||
| 935f002dbf | |||
| 25f07f89dd | |||
| a756e47682 | |||
| 15f188e4f9 | |||
| 2faf243189 | |||
| f4b065b9f6 | |||
| dc2df628e3 | |||
| 7755745f2f | |||
| b2d8dd70ed | |||
| 58d97ad4e8 | |||
| b5ea6bb461 | |||
| 7c9add3d73 | |||
| 896d81e286 | |||
| 25cbe5cd3e | |||
| 13583322b5 | |||
| f3e1de5f37 | |||
| 94fdc18c3c | |||
| f4a6cb1db2 | |||
| dc9424d3bd | |||
| afec56d03b | |||
| f56798aeb9 | |||
| 13df92fbd8 | |||
| 95f8ba918d | |||
| 0bc13b5292 | |||
| a1a38b5538 | |||
| 1742e38c10 | |||
| 7b2d04605e | |||
| 07b83561d1 | |||
| f920b4cbf5 | |||
| 44ca7c8623 | |||
| 7c1ea12331 | |||
| 7171892984 | |||
| 88d38bb900 | |||
| 3ff073d1ea | |||
| 8e2066b4bd | |||
| 9735ac3b7c | |||
| 77c5731b7b | |||
| e04b1c9199 | |||
| 75e3dc2794 | |||
| f1e26fed4f | |||
| ca34a2d65d | |||
| e2ac5d117a | |||
| 6c5218913b | |||
| da8463534b | |||
| 9958f80026 | |||
| 5744aad028 | |||
| 87d575dce4 | |||
| 1c30611b1e | |||
| 1cb14d22bf | |||
| 55bca95ad2 | |||
| 5fe96b6677 | |||
| 2c0daee481 | |||
| 62394f5b85 | |||
| 55f2889c24 | |||
| 5fdd8a570a | |||
| 9bc70d1af3 | |||
| 6ba52a68f0 | |||
| 882c7ca3cd | |||
| c69a1c441b | |||
| 22a34f7f31 | |||
| f6b6184e70 | |||
| e5dbcee17c | |||
| fd0e88e74c | |||
| 0a9715d819 | |||
| 01033d7aaf | |||
| dc53f04b81 | |||
| 917694c33d | |||
| 3e504ca9c4 | |||
| 02baf0c27b | |||
| bfcf82975c | |||
| f78781d9ef | |||
| e2352d1666 | |||
| 347d59fc62 | |||
| 404c06b2fa | |||
| a8f86b5336 | |||
| 50322bacb3 | |||
| 9e6f66dd8f | |||
| 30c92e8e59 | |||
| d4302c6ac4 | |||
| c46e5bbd15 | |||
| 5fe74db971 | |||
| 2c03e0a684 | |||
| 8624e21372 | |||
| 84dbf20a43 | |||
| 8769ee9765 | |||
| 0152180929 | |||
| 75c71adf45 | |||
| 53f69cde37 |
+15
-1
@@ -83,6 +83,12 @@ jobs:
|
||||
- name: .NET client
|
||||
run: dotnet build clients/dotnet/ZB.MOM.WW.MxGateway.Client.slnx -c Release
|
||||
|
||||
# The .NET client was previously only compiled here, so its test project — including
|
||||
# MxGatewayClientWireTests, which drives the client against a real loopback gRPC
|
||||
# server (TST-24) — never ran in CI. Every other client job already runs its tests.
|
||||
- name: .NET client tests
|
||||
run: dotnet test clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/ZB.MOM.WW.MxGateway.Client.Tests.csproj -c Release --no-build
|
||||
|
||||
- name: Go client
|
||||
working-directory: clients/go
|
||||
run: |
|
||||
@@ -165,6 +171,14 @@ jobs:
|
||||
# visible even though nobody watches the Actions page.
|
||||
if: github.event_name == 'schedule'
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
# `github.server_url` is the URL Gitea hands the runner — the docker-network-internal
|
||||
# `http://gitea:3000`. That is the right base for the issue-creation API call below (the job
|
||||
# container resolves `gitea` only on that network, and has no LAN egress to the public origin),
|
||||
# but it is useless as a link a human clicks out of the issue. So browser-facing URLs in the
|
||||
# issue body use the public origin instead. TST-25 acceptance Check 6 caught this: every
|
||||
# nightly issue since #126 carried an unreachable `http://gitea:3000/...` run link.
|
||||
PUBLIC_SERVER_URL: https://gitea.dohertylan.com
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Full Worker.Tests + live-MXAccess smoke on windev
|
||||
@@ -184,7 +198,7 @@ jobs:
|
||||
-H "Authorization: token ${{ github.token }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/issues" \
|
||||
-d "{\"title\":\"nightly-windev failed on ${{ github.sha }}\",\"body\":\"The scheduled windev Worker + live-MXAccess run failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} . A red nightly may mean the Windows tier is down rather than the change — see docs/GatewayTesting.md (Continuous Integration).\"}"
|
||||
-d "{\"title\":\"nightly-windev failed on ${{ github.sha }}\",\"body\":\"The scheduled windev Worker + live-MXAccess run failed: ${PUBLIC_SERVER_URL}/${{ github.repository }}/actions/runs/${{ github.run_id }} . A red nightly may mean the Windows tier is down rather than the change — see docs/GatewayTesting.md (Continuous Integration).\"}"
|
||||
|
||||
# NOTE: there is intentionally no native `windows` runner job. act_runner v0.6.1 host-mode on
|
||||
# Windows is broken and Windows containers are impractical for the net48/x86/MXAccess Worker, so the
|
||||
|
||||
@@ -10,7 +10,7 @@ The architecture is a two-process design — read `gateway.md` before making str
|
||||
|
||||
- **Gateway** (`src/ZB.MOM.WW.MxGateway.Server`, .NET 10, x64): ASP.NET Core gRPC server. Owns the public API, sessions, auth, the Blazor dashboard, and the Galaxy Repository SQL browse RPCs. The Galaxy-browse implementation comes from the shared **`ZB.MOM.WW.GalaxyRepository`** package (`AddZbGalaxyRepository`/`MapZbGalaxyRepository`), not inline code; mxaccessgw adds `GatewayBrowseScopeProvider` (per-key browse-subtree scoping) and a host-side dashboard summary projector. See `A2-galaxyrepository-adoption-handoff.md`. **Never instantiates MXAccess COM directly.**
|
||||
- **Worker** (`src/ZB.MOM.WW.MxGateway.Worker`, .NET Framework 4.8, **x86**): one process per session. Owns one MXAccess COM instance on a dedicated STA, pumps Windows messages, and converts COM events to protobuf.
|
||||
- **IPC**: gateway↔worker uses one bidirectional named pipe per worker (`mxaccess-gateway-{gatewayPid}-{sessionId}`) with length-prefixed `WorkerEnvelope` protobuf frames. Gateway hosts the pipe server and launches the worker. **gRPC is not used inside the worker** — .NET Framework 4.8 doesn't have a first-class gRPC stack.
|
||||
- **IPC**: gateway↔worker uses one bidirectional named pipe per worker (`mxgw-{gatewayPid}-{sessionUid}` — kept short so the macOS/Linux test matrix's Unix-domain-socket path fits the 104-byte macOS `sun_path` limit) with length-prefixed `WorkerEnvelope` protobuf frames. Gateway hosts the pipe server and launches the worker. **gRPC is not used inside the worker** — .NET Framework 4.8 doesn't have a first-class gRPC stack.
|
||||
- **Contracts** (`src/ZB.MOM.WW.MxGateway.Contracts`): multi-targets `net10.0;net48` and owns the `.proto` files (`mxaccess_gateway.proto`, `mxaccess_worker.proto`, `galaxy_repository.proto`). All other projects consume the generated types from here. Do not hand-edit anything under `Generated/`. Note `galaxy_repository.proto` is intentionally kept here as the generation source for the language clients even though the gateway server consumes the wire-identical Galaxy types from the `ZB.MOM.WW.GalaxyRepository` package — it is not dead code; deleting it breaks all five clients.
|
||||
|
||||
The worker must do all MXAccess COM calls on its dedicated STA thread, and the STA loop must pump Windows messages (`MsgWaitForMultipleObjectsEx` + `PeekMessage`/`DispatchMessage`) so MXAccess events deliver. A plain blocking queue on an STA is not enough.
|
||||
@@ -124,7 +124,7 @@ powershell -ExecutionPolicy Bypass -File scripts/run-client-e2e-tests.ps1
|
||||
|
||||
When source code changes, build and test the affected component before reporting work done. If the change crosses component boundaries, build each affected component — don't rely on a single top-level build:
|
||||
|
||||
**Run targeted tests per task, never the full suite each time.** When executing a plan task-by-task, run only the tests that exercise the code that task touched (`dotnet test --filter "FullyQualifiedName~<TestClass>"`, or the per-task test named in the plan). The full gateway suite is slow — run it at most once per phase (after a related batch of tasks lands), not after every task. This is a speed guideline, not a correctness one: the suite exits cleanly (verified on macOS and the Windows dev box — 0 surviving `testhost`/worker processes after a full run), so filtered runs are about turnaround, not about avoiding a process leak.
|
||||
**Run targeted tests per task, never the full suite each time.** When executing a plan task-by-task, run only the tests that exercise the code that task touched (`dotnet test --filter "FullyQualifiedName~<TestClass>"`, or the per-task test named in the plan). The full gateway suite is slow — run it at most once per phase (after a related batch of tasks lands), not after every task. This is a speed guideline, not a correctness one: the suite exits cleanly on both macOS and windev (0 surviving `testhost`/worker processes after a full run), so filtered runs are about turnaround. The long-standing windev full-suite wedge — every test reported, then `testhost` never exiting — was a zero-buffer named pipe blocking one test's write forever; it is fixed and the `--blame-hang` workaround is no longer needed. See "Running the Gateway Suite on windev" in `docs/GatewayTesting.md`, which also documents the load-sensitivity caveat that still applies there.
|
||||
|
||||
| Changed area | Required verification |
|
||||
|---|---|
|
||||
|
||||
@@ -150,7 +150,7 @@ Sequence these together rather than piecemeal — several are one change set spa
|
||||
|
||||
- Close **CLI-24** and **CLI-34** as `Done` (incidentally fixed; evidence in [../50-clients.md](../50-clients.md)).
|
||||
- ~~When CLI-38 lands, close old **CLI-08** with a pointer here.~~ Done 2026-08-07: CLI-38 landed and old CLI-08 is now `Done` in the first-cycle tracker, pointing at [CLI-38](50-clients.md#cli-38--align-netgojava-on-hresult--0-lands-prior-cli-08-cures-the-doc-drift---medium--p1).
|
||||
- ~~When WRK-26 lands, its doc section also discharges the WorkerFrameProtocol gap~~ Done 2026-08-07: WRK-26 landed and **IPC-29** is discharged, pointing at [WRK-26](20-worker.md#wrk-26--write-priority-and-overflow-doc-drift-from-the-wrk-07-change---low--p1). When TST-25 lands, revisit old **TST-05** (scheduled live smoke) and **TST-24** (client wire tests), which it unlocks.
|
||||
- ~~When WRK-26 lands, its doc section also discharges the WorkerFrameProtocol gap~~ Done 2026-08-07: WRK-26 landed and **IPC-29** is discharged, pointing at [WRK-26](20-worker.md#wrk-26--write-priority-and-overflow-doc-drift-from-the-wrk-07-change---low--p1). ~~When TST-25 lands, revisit old **TST-05** (scheduled live smoke) and **TST-24** (client wire tests), which it unlocks.~~ TST-05 revisited 2026-08-10: `Partially done` in the first-cycle tracker — the `nightly-windev` job closes the scheduled-cadence half, but the finding's coverage-audit half stays open (the live suite reaches all six late-added COM commands and none of the five control commands). TST-24 revisited in the same change and closed `Done`: Go/Rust/Java already had real-server wire tests, and the two genuine gaps (.NET, Python) now have them plus a CI step.
|
||||
|
||||
## Change log
|
||||
|
||||
@@ -159,7 +159,7 @@ Sequence these together rather than piecemeal — several are one change set spa
|
||||
| 2026-07-13 | Initial tracking doc generated from the six domain remediation designs. All 47 findings `Not started` (IPC-31, SEC-35 `N/A`). |
|
||||
| 2026-07-13 | TST-25/TST-26 → `In progress` (branch `fix/tst-25-windev-ci`). Added `scripts/ci/{windev-worker-ci.ps1,run-windev-ci.sh,windev.known_hosts}`, `windows-x86` (per-push) + `nightly-windev` (scheduled) jobs in `ci.yml`, and the TST-26 doc/comment fixes (GatewayTesting.md, Contracts.md, check-codegen.ps1). Mechanism hand-verified on windev: `build`→0, bogus-SHA→nonzero (lock released), `test`→356 passed/0 failed in ~50s (per-push stays `test`, no demotion), and run-windev-ci.sh SSH+EncodedCommand exit-code propagation confirmed. |
|
||||
| 2026-07-13 | Operator bring-up complete: dedicated CI ed25519 key installed in windev `administrators_authorized_keys` (authorized into `dohertj2`, which owns the working MXAccess/toolchain env — a fresh OS account would break the build; the key is independently revocable), Gitea secrets `WINDEV_SSH_KEY`/`WINDEV_SSH_KNOWN_HOSTS` + variable `WINDEV_SSH_USER=dohertj2` stored, runner→`10.100.0.48:22` egress verified on the `traefik` net, issue-write confirmed. **TST-25/TST-26 → `Done`:** credentialed `windows-x86` ran GREEN on `d769244` (Gitea run #37) — Linux runner SSHed windev, checked out the SHA in `C:\build\mxaccessgw-ci` under lock, ran the x86 Worker build + `Worker.Tests`, exit 0; `nightly-windev` correctly skipped on the push event. Branch merged to `main`. Follow-ups (old tracker): revisit **TST-05** (scheduled live smoke — now covered by `nightly-windev`) and **TST-24** (client wire tests) which this unlocks. |
|
||||
| 2026-07-13 | Ran the TST-25 acceptance checks (scripts/ci/README.md) — they caught **two real CI defects, both fixed** on `fix/tst-25-ci-key-log-leak`: (1) **CI SSH key leaked in cleartext** in the `windows-x86` step env echo (Gitea's line-oriented masker missed the multiline PEM) — rotated the CI key on windev (old pubkey revoked), stored the key **base64-encoded** so the masker redacts it to `***` (confirmed on run #38), taught `run-windev-ci.sh` to decode, dropped the redundant public known-hosts secret from the job env; (2) **bootstrap lock race** — `run-windev-ci.sh`'s pre-hand-off `git fetch`/`checkout` ran outside the worktree lock, so concurrent runs collided on `.git/index.lock`; the bootstrap now holds the lock (ps1 re-uses it via `MXGW_CI_LOCK_HELD`), retest confirmed clean serialization. Also **deflaked** `SessionManagerTests` fail-fast timing assertions (absolute `<100ms` wall-clock bound flaked under CI load; now anchored to the configured timeout / dropped for the zero-timeout case). Checks passed: unreachable-host fast-fail (exit 255/15s), deliberate-red propagation (Worker.Tests failure → exit 1), lock concurrency (2nd run waits), no-key-in-logs (masked). Merge target `df7e20d` verified GREEN via the local windev path (Worker build + 356 tests); merged to `main` `19cbf7b`. Check 6 (forced-failure nightly issue): issue endpoint+token proven live at bring-up (#124); in-CI forced-failure probe abandoned to shared-runner congestion (residual `if: failure()` gating is standard Actions). |
|
||||
| 2026-07-13 | Ran the TST-25 acceptance checks (scripts/ci/README.md) — they caught **two real CI defects, both fixed** on `fix/tst-25-ci-key-log-leak`: (1) **CI SSH key leaked in cleartext** in the `windows-x86` step env echo (Gitea's line-oriented masker missed the multiline PEM) — rotated the CI key on windev (old pubkey revoked), stored the key **base64-encoded** so the masker redacts it to `***` (confirmed on run #38), taught `run-windev-ci.sh` to decode, dropped the redundant public known-hosts secret from the job env; (2) **bootstrap lock race** — `run-windev-ci.sh`'s pre-hand-off `git fetch`/`checkout` ran outside the worktree lock, so concurrent runs collided on `.git/index.lock`; the bootstrap now holds the lock (ps1 re-uses it via `MXGW_CI_LOCK_HELD`), retest confirmed clean serialization. Also **deflaked** `SessionManagerTests` fail-fast timing assertions (absolute `<100ms` wall-clock bound flaked under CI load; now anchored to the configured timeout / dropped for the zero-timeout case). Checks passed: unreachable-host fast-fail (exit 255/15s), deliberate-red propagation (Worker.Tests failure → exit 1), lock concurrency (2nd run waits), no-key-in-logs (masked). Merge target `df7e20d` verified GREEN via the local windev path (Worker build + 356 tests); merged to `main` `19cbf7b`. Check 6 (forced-failure nightly issue): issue endpoint+token proven live at bring-up (#124); in-CI forced-failure probe abandoned to shared-runner congestion (residual `if: failure()` gating is standard Actions). **Superseded 2026-08-10 — Check 6 is now Done; see the 2026-08-10 change-log entry in `archreview/remediation/00-tracking.md`.** |
|
||||
| 2026-07-13 | New finding **TST-30** (`Low`/`P2`) added — surfaced during TST-25 acceptance: CI runs on a single shared `gitea-runner` (`maxParallel=1`, co-located `10.100.0.35`) interleaved with `dohertj2/lmxopcua`, and Gitea 1.26 exposes no run cancel/delete, so queue latency is unbounded under cross-repo contention and the runner is a single point of failure. Design: add a second/labelled runner + document the no-cancel reality and the `run-windev-ci.sh` bypass. Roll-ups updated (Testing Low 2→3, total 47→48; P2 9→10). |
|
||||
| 2026-08-07 | **TST-29 → `Done`:** migrated the Phase-5 (orphan-worker reattach) deferred-not-planned governance record and the settled Phase-4 Viewer-default decision from `oldtasks.md` into a new "Session-Resilience Epic Scope" entry in `docs/DesignDecisions.md`; repointed CLAUDE.md and `stillpending.md:7,165` from `oldtasks.md` to `docs/DesignDecisions.md` / `docs/plans/2026-06-15-session-resilience.md.tasks.json`; `git rm oldtasks.md`. The five untracked root docs-review artifacts (`MxAccessGateway-docs-{issues,fixed,final}.md`, `MxGatewayClient-docs-{issues,fixed}.md`) were absent from this worktree — delete from the main working tree separately. |
|
||||
| 2026-08-07 | **GWC-24 → `Done`** (branch `fix/gwc-24-staging-bound`). `WorkerClient._eventStaging` is now `Channel.CreateBounded` at `2 × EventChannelCapacity` (`Wait`, single reader/writer, no sync continuations); a rejected staging `TryWrite` faults the client `ProtocolViolation` with `QueueOverflow("worker-event-staging")` unless `IsTerminalState()` (shutdown stays a silent drop), so a consumer draining slower than its worker produces dies at a fixed ceiling instead of growing gateway memory. Queue-depth accounting moved from `EnqueueWorkerEventAsync` to `StageWorkerEvent`, so the single gauge reports staged + queued; the timed-write fault (`EventChannelFullModeTimeout` / `QueueOverflow("worker-events")`) is unchanged and still catches the full-stall case first. No new config key — total gateway-side buffering is `3 × MxGateway:Events:QueueCapacity`, derived; coordination with still-open old **GWC-21** (`EventChannelFullModeTimeout` configurability) remains open and was not blocked on. Docs same commit: `GatewayProcessDesign.md` (two overflow faults), `MxAccessWorkerInstanceDesign.md`, `GatewayConfiguration.md`, `Metrics.md`. Tests: `WorkerClientTests.StagingChannelOverflowFaultsWorkerWithoutWaitingForFullModeTimeout` and `.WorkerEventQueueDepthGaugeCountsStagedEvents`; `WorkerClientTests` 22/22 green, `NonWindows.slnx` builds with 0 warnings. |
|
||||
|
||||
@@ -175,7 +175,7 @@ Independent of the runner count, document the **no-cancel** reality (Gitea 1.26
|
||||
|
||||
## Cross-domain dependencies
|
||||
|
||||
- **TST-25 → old TST-05 / old TST-24:** the SSH-driven nightly is where the scheduled live-MXAccess smoke lands (closes old TST-05), and a working CI Windows tier unblocks wiring client wire-behavior tests into CI (old TST-24).
|
||||
- **TST-25 → old TST-05 / old TST-24:** the SSH-driven nightly is where the scheduled live-MXAccess smoke lands (closes old TST-05), and a working CI Windows tier unblocks wiring client wire-behavior tests into CI (old TST-24). *Followed up 2026-08-10:* old TST-05 is `Partially done` — the nightly closes the scheduling half, but the live suite still covers none of the five worker **control** commands; old TST-24 is `Done`, and its client wire tests turned out to need no Windows tier at all (they run in the `portable` job). See the first-cycle tracker.
|
||||
- **TST-25 ↔ IPC-24/IPC-25:** the nightly windev job is also the natural home for any Windows-side codegen verification the contracts/IPC remediation adds; coordinate job naming so both plans extend the same `windows-x86`/nightly jobs rather than adding parallel ones.
|
||||
- **TST-26 ⊂ TST-25:** same commit, by rule.
|
||||
- **TST-27:** ships in the cycle's P1 doc-drift batch alongside WRK-26 and CLI-42 (roadmap item 8); its `/browse` residual stays with TST-16 (prior cycle).
|
||||
|
||||
@@ -4,15 +4,15 @@ These were discovered while remediating the 2026-07-12 backlog but were **out of
|
||||
|
||||
| ID (proposed) | Area | Severity (est.) | Summary |
|
||||
|---|---|---|---|
|
||||
| NEXT-01 | Testing / macOS | Low | Fake-worker/e2e gateway tests fail on macOS under the default `TMPDIR` because the `CoreFxPipe_mxaccess-gateway-{pid}-{sessionId}` path exceeds the 104-char Unix-domain-socket `sun_path` limit under `/var/folders/…/T/`. Workaround today is `TMPDIR=/tmp`. Fix options: shorten the pipe name, or document the `TMPDIR=/tmp` requirement in `docs/GatewayTesting.md`. Surfaced independently by multiple remediation agents. |
|
||||
| NEXT-02 | Clients (.NET, Java) | Low | The .NET and Java CLIs render the raw `ReplayGap` sentinel `MxEvent` on `stream-events` instead of a typed gap row — Java text mode prints `0 MX_EVENT_FAMILY_UNSPECIFIED`. Same defect class as CLI-36 (Go) / CLI-35 (Python), which were fixed this cycle; the .NET/Java halves were out of scope. The cross-language smoke matrix now records this divergence honestly. |
|
||||
| NEXT-03 | Gateway alarms | Low | `GatewayAlarmMonitor.ApplyReconcile` feed-repair broadcasts (the new acked-delta from GWC-26 **and** the pre-existing Raise/Clear repair) are **at-least-once, not exactly-once**: a periodic reconcile can synthesize a transition whose matching live transition is still buffered in the alarm lease, so both broadcast as indistinguishable duplicates on the alarm feed (StreamAlarms + dashboard hub). Pre-existing (the Raise/Clear repair always had it); GWC-26 documented the at-least-once contract rather than closing the race. Closing it needs reconcile/live serialization or a monotonic dedup marker. |
|
||||
| NEXT-04 | Worker frame writer | Low | WRK-22/WRK-25 cancellation path: a frame `Claimed` by a concurrent lock-holder just before its caller's cancellation races in is never awaited by that caller; if the write then faults, `TrySetException` lands on a `Task` nobody observes (unobserved-task-exception). By-design residual, non-crash (no `UnobservedTaskException` handler registered), pre-existing to single-frame WRK-22 and amplified per-batch by WRK-25. Hygiene fix: attach a fault-observing continuation to abandoned/tombstoned frame completions. |
|
||||
| NEXT-05 | Worker frame writer | Info | A batch whose remaining frames are tombstoned by cancellation leaves dead `PendingFrame` entries in `_eventFrames`/`_controlFrames` until a future `DequeueNext` pops and skips them. Same pre-existing behavior as single-frame WRK-22, amplified per-batch; in practice heartbeats purge them promptly, so not a real leak. |
|
||||
| ~~NEXT-01~~ | Testing / macOS | Low | **Resolved 2026-08-10** — the pipe name is now `mxgw-{pid}-{sessionUid}` (session guid hex, worst-case 43 chars), which fits the 104-byte `sun_path` budget under the default macOS `TMPDIR`; the three test-fixture pipe names were shortened the same way, and a `SessionManagerTests` regression pins the format and length budget. All previously failing suites (SessionWorkerClientFactory, e2e fake-worker smoke, WorkerClient, reconnect-replay) pass 33/33 under the default `TMPDIR` — this also retired the separately-remembered "macOS pipe-timeout test failures", which were this throw misread. Docs updated (gateway.md, GatewayProcessDesign, GatewayConfiguration, Sessions, CLAUDE.md). Original finding: Fake-worker/e2e gateway tests fail on macOS under the default `TMPDIR` because the `CoreFxPipe_mxaccess-gateway-{pid}-{sessionId}` path exceeds the 104-char Unix-domain-socket `sun_path` limit under `/var/folders/…/T/`. Workaround today is `TMPDIR=/tmp`. Fix options: shorten the pipe name, or document the `TMPDIR=/tmp` requirement in `docs/GatewayTesting.md`. Surfaced independently by multiple remediation agents. |
|
||||
| ~~NEXT-02~~ | Clients (.NET, Java) | Low | **Resolved 2026-08-10** — both CLIs now branch on the sentinel and emit the typed cross-CLI row with numeric cursors (Java text mode prints `REPLAY_GAP requested_after=<n> oldest_available=<n>`; .NET emits the `{"replayGap":{…}}` row in jsonl/text and inside the `--json` events array). CrossLanguageSmokeMatrix.md's divergence table collapsed to one converged contract. New CLI regressions in both languages (.NET 35/35, Java 52/52). Original finding: The .NET and Java CLIs render the raw `ReplayGap` sentinel `MxEvent` on `stream-events` instead of a typed gap row — Java text mode prints `0 MX_EVENT_FAMILY_UNSPECIFIED`. Same defect class as CLI-36 (Go) / CLI-35 (Python), which were fixed this cycle; the .NET/Java halves were out of scope. The cross-language smoke matrix now records this divergence honestly. |
|
||||
| ~~NEXT-03~~ | Gateway alarms | Low | **Resolved 2026-08-10** — best-effort dedup in `GatewayAlarmMonitor`: a buffered live transition whose worker timestamp + resulting state the cache already carries from a repair is suppressed, and reconcile Clear repairs tombstone the instance by `original_raise_timestamp` for one reconcile generation so the buffered live Clear dedups too. Positive-match only (unset timestamps never suppress), so the documented at-least-once consumer contract stands; serialization was rejected as the larger change that still needs a worker-side high-water mark to be correct. Two new race-driving regressions; alarm suites 18/18. Original finding: `GatewayAlarmMonitor.ApplyReconcile` feed-repair broadcasts (the new acked-delta from GWC-26 **and** the pre-existing Raise/Clear repair) are **at-least-once, not exactly-once**: a periodic reconcile can synthesize a transition whose matching live transition is still buffered in the alarm lease, so both broadcast as indistinguishable duplicates on the alarm feed (StreamAlarms + dashboard hub). Pre-existing (the Raise/Clear repair always had it); GWC-26 documented the at-least-once contract rather than closing the race. Closing it needs reconcile/live serialization or a monotonic dedup marker. |
|
||||
| ~~NEXT-04~~ | Worker frame writer | Low | **Resolved 2026-08-10** — the tombstone helpers now attach a fault-observing continuation to every frame of a cancelled call (a cancelled task never fires `OnlyOnFaulted`, so unconditional attach is safe; covers both the claimed-mid-write frame and the already-faulted-by-`FailAllQueued` frame where `TrySetCanceled` loses). New regression drives the exact abandonment and asserts a marker exception never reaches `TaskScheduler.UnobservedTaskException`. Original finding: WRK-22/WRK-25 cancellation path: a frame `Claimed` by a concurrent lock-holder just before its caller's cancellation races in is never awaited by that caller; if the write then faults, `TrySetException` lands on a `Task` nobody observes (unobserved-task-exception). By-design residual, non-crash (no `UnobservedTaskException` handler registered), pre-existing to single-frame WRK-22 and amplified per-batch by WRK-25. Hygiene fix: attach a fault-observing continuation to abandoned/tombstoned frame completions. |
|
||||
| ~~NEXT-05~~ | Worker frame writer | Info | **Resolved 2026-08-10 as a documented decision** — the lazy `DequeueNext` purge stays: any subsequent write drains both queues to empty and the heartbeat loop bounds tombstone residency to one interval, while eager `Queue<T>` rebuilds under `_gate` would add ordering-invariant surface next to the WRK-22 interlock for no real gain. Rationale recorded in `docs/WorkerFrameProtocol.md`. Original finding: A batch whose remaining frames are tombstoned by cancellation leaves dead `PendingFrame` entries in `_eventFrames`/`_controlFrames` until a future `DequeueNext` pops and skips them. Same pre-existing behavior as single-frame WRK-22, amplified per-batch; in practice heartbeats purge them promptly, so not a real leak. |
|
||||
| ~~NEXT-06~~ | Testing / live LDAP | Medium | **Resolved 2026-08-07** — fixtures realigned to the shared directory (`admin`/`password` for the GwAdmin success path, `gw-viewer`/`password` for the bind-succeeds-but-no-role path); verified `Failed: 0, Passed: 5` live against the shared GLAuth at `10.100.0.35:3893`, so the success-path assertion (GwAdmin group claim + Admin role claim) now fails if the service-account credential is wrong. Original finding: `DashboardLdapLiveTests` fixtures have drifted from the shared GLAuth directory, leaving the suite with **no positive-proof coverage of the service-account bind**. Its only success-path test, `AuthenticateAsync_AdminInGwAdminGroup_Succeeds`, binds `admin`/`admin123`, but the directory's `admin` user carries the standard dev password (`scadaproj/infra/glauth/config.toml`), so that assertion cannot pass. `AuthenticateAsync_ReadOnlyUserMissingGwAdminGroup_Fails` binds fixture user `readonly`, which **does not exist** in the GLAuth config at all — it passes for the wrong reason (user-not-found rather than the group-missing branch it names; the `readonly` name is in fact barred by the README's user/group case-collision rule). The three remaining tests are negative assertions that pass whether or not the service account can bind. Net effect: a green `DashboardLdapLiveTests` run proves nothing about the bind credential — surfaced during SEC-36, where the suite was considered as a substitute for the deferred dashboard-login check and rejected. Fix: realign the fixtures to real directory users (e.g. `multi-role`/`gw-viewer`) or add the missing users to the GLAuth config, and add one test that fails when the service-account credential is wrong. |
|
||||
| ~~NEXT-07~~ | Deployment / windev | High | **Resolved 2026-08-07** — a fresh portable framework-dependent publish of `origin/main` (`a346d51`) was built in a clean clone at `C:\build\mxgw-redeploy`, deployed to `C:\publish\mxaccessgw\Server-20260807`, and the `MxAccessGw` NSSM service repointed at it; the service now holds a stable PID with both `5120`/`5130` listening, a worker spawned, the Galaxy snapshot restored (129 objects / 56,731 attributes) and a clean event log. Root cause confirmed as the version skew this row predicted: the deployed 2026-06-25 build carried `ZB.MOM.WW.Auth.ApiKeys` 0.1.2.0, which supports auth-DB schema 2, against a `gateway-auth.db` stamped at schema 3 on 2026-07-15 by an ephemeral run of newer code — schema 3 is the current shared-lib version (`SqliteAuthSchema.CurrentVersion=3` in Auth 0.1.5), so the redeploy is the forward fix and the DB was left alone. Rollback artifacts kept: `C:\ProgramData\MxGateway\gateway-auth.db.bak-next07` (with `-wal`/`-shm`) and the previous `C:\publish\mxaccessgw\Server` directory. Two side effects worth recording: the old deploy's `appsettings.json` held the LDAP bind password in **plaintext on disk**, while the new one keeps the repo's `${secret:ldap/mxgateway/bind}` token with the NSSM environment supplying the value, so no plaintext LDAP secret remains on that host; and the redeploy tripped the SEC-06 `Ldap:Transport=None` production hard-stop (`GatewayOptionsValidator.cs:178`), resolved by relabelling the host — windev runs `Dashboard:DisableLogin=true`, which this repo's own docs mark dev/test-only, so its `Production` label contradicted its configuration and `DOTNET_ENVIRONMENT` was changed to `Staging` (that one NSSM environment entry only; the other nine preserved byte-identical). SEC-06 is untouched for genuinely production hosts — see NEXT-08 for the posture problem that relabelling defers. Original finding: The `10.100.0.48` (windev) gateway deployment is **stale and crash-looping**, and has been since at least 2026-08-06 (~10k Hosting-failed events/day). The deployed Server binary dates to 2026-06-25 and predates the auth-DB migration of 2026-07-15: it opens a schema-version-3 `gateway-auth.db` that it supports only at version 2 and aborts at startup, so the `MxAccessGw` service never reaches a listening state. Not a code defect in the current tree — a deploy-drift/operations gap — but it means the repo's only deployed host has been dark for over a day and any host-level verification (including SEC-36's dashboard-login check) is blocked until it is repaired. Fix: deploy a current Server build to windev, or restore/downgrade the auth DB to schema 2 if the old binary must stand. Worth asking separately why a service in a permanent restart loop raised no alert. Discovered during SEC-36. |
|
||||
| NEXT-08 | Security / LDAP posture | Medium | **The shared GLAuth offers no TLS, so SEC-06 makes it undeployable from a `Production`-labelled host.** `GatewayOptionsValidator` (`src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs:178`) refuses to start when `Ldap:Transport=None` in the `Production` environment, and `docs/GatewayConfiguration.md`'s `Transport` row states "Deployed hosts must set `Ldaps` or `StartTls`" — but the shared instance at `10.100.0.35:3893` has `[ldaps] enabled=false`, port `3894` closed, and answers StartTLS with `protocolError`, so neither value can work against it. That instruction is currently unsatisfiable for every host that authenticates there. windev sidestepped it on 2026-08-07 by moving to the `Staging` environment name (NEXT-07), which is honest for a dev/test rig but is not available to a real production host. Resolution needs either LDAPS/StartTLS on the shared GLAuth (certificate plus a trust story on each gateway host) or an explicit written posture decision that production gateways bind a different, TLS-capable directory. Surfaced during the NEXT-07 redeploy. |
|
||||
| NEXT-09 | Build / versioning | Low | **Windows builds stamp git's error text into `InformationalVersion`.** `src/Directory.Build.props:29` runs `git -C "$(MSBuildThisFileDirectory)" …`; MSBuild's directory property ends in a backslash, which escapes the closing quote, so the command is malformed on Windows. The target carries `ContinueOnError`, so the failure is silent and git's stderr is captured as the revision — an observed stamp reads `0.1.2+fatal: cannot change to …`. Any Windows build without a preset `SourceRevisionId` therefore ships a binary that cannot be correlated back to a commit, defeating the point of TST-11. Not reproducible on macOS/Linux, where the separator is `/`. Fix sketch: append `.` to the path or trim the trailing separator before quoting. Surfaced while identifying the deployed binary during NEXT-07. |
|
||||
| ~~NEXT-09~~ | Build / versioning | Low | **Resolved 2026-08-10** — the Exec path now appends `.` so the trailing backslash can no longer escape the closing quote, and `SourceRevisionId` is additionally gated on a short-SHA regex so no future git failure text can be stamped either. macOS stamp verified unchanged; Windows stamp verified on windev with this batch. Original finding: **Windows builds stamp git's error text into `InformationalVersion`.** `src/Directory.Build.props:29` runs `git -C "$(MSBuildThisFileDirectory)" …`; MSBuild's directory property ends in a backslash, which escapes the closing quote, so the command is malformed on Windows. The target carries `ContinueOnError`, so the failure is silent and git's stderr is captured as the revision — an observed stamp reads `0.1.2+fatal: cannot change to …`. Any Windows build without a preset `SourceRevisionId` therefore ships a binary that cannot be correlated back to a commit, defeating the point of TST-11. Not reproducible on macOS/Linux, where the separator is `/`. Fix sketch: append `.` to the path or trim the trailing separator before quoting. Surfaced while identifying the deployed binary during NEXT-07. |
|
||||
| NEXT-10 | Docs / glauth | Medium | **`glauth.md`'s "Pre-provisioned users" table contradicts both the directory and the rest of its own file.** It documents `readonly`/`readonly123` and `admin`/`admin123`, neither of which matches `scadaproj/infra/glauth/config.toml` (`readonly` does not exist there; `admin` carries the standard dev password), and lists the `ReadOnly` gid as `5501` against an actual `5601`. Its dashboard section, by contrast, is correct — so the file is internally inconsistent and a reader cannot tell which half to trust. This table was the **root cause of the NEXT-06 fixture drift**, and it has propagated further: `docs/GatewayTesting.md`'s `MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD` default and the matching literal in `WorkerLiveMxAccessSmokeTests` both take `admin123` from it. Deliberately **not** fixed in the 2026-08-07 pass: the table is entangled with the OPC-UA group taxonomy (gids, role mapping, and the sister-repo consumers of the same directory), so reconciling it means sweeping that taxonomy as one unit rather than patching two rows. |
|
||||
|
||||
## Operator actions still pending (from this cycle's runbooks)
|
||||
|
||||
@@ -217,7 +217,7 @@ Full design + implementation for each row lives in the linked domain doc under i
|
||||
| TST-02 | High | P0 | M | TST-04 | Done | Reconnect owner re-validation not implemented |
|
||||
| TST-03 | High | P1 | M | — | Done | No CI exists |
|
||||
| TST-04 | High | P2 | L | — | Done | Session-resilience epic 16/28 tasks unfinished |
|
||||
| TST-05 | Medium | P1 | S | TST-03 | Not started | Real-worker control/COM paths verified opt-in only |
|
||||
| TST-05 | Medium | P1 | S | TST-03 | Partially done | Real-worker control/COM paths verified opt-in only. **Scheduling half closed 2026-08-10** by the TST-25 `nightly-windev` job (`.gitea/workflows/ci.yml`, cron `0 6 * * *` → `scripts/ci/run-windev-ci.sh live`, which sets `MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`, runs `WorkerLiveMxAccessSmokeTests`, and opens a Gitea issue on failure) — "opt-in, run by memory" is now "runs nightly, reports failures". **Residual: the coverage-audit half.** The live suite's 8 facts cover all six late-added COM commands but none of the five control commands (`Ping`, `GetSessionState`, `GetWorkerInfo`, `DrainEvents`, `ShutdownWorker`), which real workers answer in `Worker/Ipc/WorkerPipeSession.cs` yet are still only exercised through `FakeWorkerHarness` canned replies — precisely the masking the finding named |
|
||||
| TST-06 | Medium | — | M | — | Not started | Dashboard live-data path untested |
|
||||
| TST-07 | Medium | — | S | — | Not started | Real-clock sleeps with negative assertions are latent flakes |
|
||||
| TST-08 | Medium | P1 | M | — | Done | Full-suite orphaned testhost processes (does not reproduce; doc de-stale) |
|
||||
@@ -236,7 +236,7 @@ Full design + implementation for each row lives in the linked domain doc under i
|
||||
| TST-21 | Low | — | S | — | Not started | Log rotation configured but minimal |
|
||||
| TST-22 | Low | — | S | — | Not started | Config-shape JSON block omits documented keys |
|
||||
| TST-23 | Low | P2 | S | — | Done | Bidirectional `Session` RPC never built |
|
||||
| TST-24 | Low | P2 | M | TST-03 | Not started | Client wire behaviour has no automated verification. **Gate cleared:** TST-03 CI is Done (live and green 2026-07-10; Windows/x86 tier green 2026-07-13 via the TST-25/TST-26 SSH-driven windev job), so TST-24 is unblocked — deferred by choice now, not CI-gated |
|
||||
| TST-24 | Low | P2 | M | TST-03 | Done | Client wire behaviour has no automated verification — closed 2026-08-10. Go/Rust/Java already had real-server wire tests (the finding's premise was stale); the genuine gaps were .NET (transport-interface fake everywhere, no server package) and Python (stub monkeypatch everywhere but one opt-in TLS test). Added `MxGatewayClientWireTests` + `WireFakeGatewayServer` (Kestrel h2c) and `tests/test_wire_fake_gateway.py` (`grpc.aio` loopback), plus a `dotnet test` step for the .NET client in the `portable` CI job. Caught a real bug: Python `connect()` built the `grpc.aio` channel inside `asyncio.to_thread` and failed for every non-stub connection |
|
||||
|
||||
## Cross-cutting clusters
|
||||
|
||||
@@ -253,6 +253,9 @@ Findings the review flagged as one coordinated design pass — sequence them tog
|
||||
|
||||
| Date | Change |
|
||||
|---|---|
|
||||
| 2026-08-10 | **TST-25 acceptance Check 6 (forced-failure nightly issue) → Done.** The 2026-07-13 record wrote this check off as "abandoned to shared-runner congestion"; that was wrong on both counts. The 2026-07-13 probe *did* land (issue #125, `[CHECK6 PROBE]`, run 375), and since 2026-07-17 the `nightly-windev` `if: failure()` step has filed an issue on **every** red nightly — #126–#139, all authored by the `gitea-actions` bot. Traced run 672 (schedule, main, red) line by line: main step fails → `exitcode '1': failure` → the `if: failure()` step runs → `POST /api/v1/repos/dohertj2/mxaccessgw/issues` with the built-in token masked to `***` → issue #139 created at the matching timestamp. Re-confirmed by a fresh forced-failure probe on the throwaway branch `test/tst25-check6-nightly-issue` (temporary `tst25-check6-probe.yml` reproducing the job shape with `exit 1` for the live step; run 677 → issue #140). Branch deleted, issues #125 and #140 closed with explanatory comments. **One real defect found and fixed** (`fix/tst25-nightly-issue-path`, not merged): `${{ github.server_url }}` is the runner-internal `http://gitea:3000`, so every filed issue's run link was unreachable from a browser. The API call must keep using it (the job container resolves `gitea` only on the docker network and has no LAN egress to the public origin), so the fix adds a `PUBLIC_SERVER_URL: https://gitea.dohertylan.com` job env used **only** for the browser-facing link in the issue body; the probe validated the fixed template (#140 carries a `https://gitea.dohertylan.com/...` link that returns 200). **Separately observed, not fixed:** the nightly has been red continuously since at least 2026-07-17 (run 672: `x86 Worker.Tests failed with exit code 1`, 1 failed / 398 passed / 11 skipped — the known `EventBurst_DrainLoopCoalescesFlushes` class of flake), and the step de-duplicates nothing, so 14 issues are open, seven of them (#132–#138) for the identical SHA `47c0b64`. Worth a follow-up: fix the red nightly, and consider having the step reuse an open issue with the same title instead of filing a new one. |
|
||||
| 2026-08-10 | **TST-24 → `Done`: per-client wire tests land for the two clients that lacked them** (branch `feat/tst-24-client-wire-tests`). Audit first corrected the finding's premise: **Go, Rust, and Java already had real-server wire tests** — `newBufconnClient`/`fakeGatewayServer` over `grpc/test/bufconn`, `spawn_fake_gateway` over a loopback `TcpListener` with tonic's `Server`, and `InProcessGateway`/`TestGatewayService` over `InProcessServerBuilder` — each already asserting the round trip, the server-observed `authorization` bearer header, and the `ReplayGap` sentinel. The real gaps were **.NET** (every test substituted `FakeGatewayTransport` for `IMxGatewayClientTransport`, and the test project had no server package) and **Python** (stub monkeypatching everywhere except one opt-in TLS test serving only `OpenSession`). Added `WireFakeGatewayServer` + `MxGatewayClientWireTests` (Kestrel h2c on `127.0.0.1:0` serving `MxAccessGatewayBase`; new `Grpc.AspNetCore.Server` 2.76.0 + `Microsoft.AspNetCore.App` refs on the test project) and `clients/python/tests/test_wire_fake_gateway.py` (`grpc.aio` server on `127.0.0.1:0`, no new deps). Four shapes each: full round trip with every reply field asserted, the bearer header **as received by the server** on the streaming RPC too, the `ReplayGap` sentinel surfaced as the client's typed signal, and a genuine `PERMISSION_DENIED` mapping to the typed authorization error. CI: the `portable` job only *built* the .NET client, so a `dotnet test` step was added. **The new tests immediately caught a shipped bug** — Python `GatewayClient.connect()`/`GalaxyRepositoryClient.connect()` constructed the `grpc.aio` channel inside `asyncio.to_thread`, which raises `RuntimeError: There is no current event loop in thread 'asyncio_0'` because a `grpc.aio` channel binds to the loop current on the constructing thread; every non-stub connection failed, and the one test guarding the off-loop behaviour (Client.Python-028) monkeypatched `create_channel` and so asserted the bug. Fixed by splitting `resolve_channel_security` (blocking TOFU probe, off-loop) from `create_channel` (on-loop), with the `-028` tests retargeted to assert both halves. Verified: .NET 133 passed/1 skipped (pre-existing live-gateway skip), Python 168 passed/1 skipped plus 6/6 opt-in TLS. Docs: `docs/GatewayTesting.md` § Client Wire Tests, `clients/dotnet/README.md`, `clients/python/README.md`. |
|
||||
| 2026-08-10 | **TST-05 revisited under the restored Windows tier → `Partially done`** (branch `feat/tst-24-client-wire-tests`, doc/tracker-only). The finding's **scheduling** half is closed: cycle-2 TST-25's `nightly-windev` job (cron `0 6 * * *`) runs `scripts/ci/run-windev-ci.sh live` → `windev-worker-ci.ps1 -Mode live`, which sets `MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`, runs `WorkerLiveMxAccessSmokeTests` on windev after the x86 build/Worker.Tests/full-slnx steps, and files a Gitea issue when red. The **coverage-audit** half is *not* closed, and the audit the design asked for now has a negative answer: the suite's eight `[LiveMxAccessFact]`s cover all six late-added COM commands (`Suspend`, `Activate`, `AuthenticateUser`, `ArchestrAUserToId`, `AddBufferedItem`, `SetBufferedUpdateInterval`) but zero of the five control commands — `MxCommandKind.{Ping,GetSessionState,GetWorkerInfo,DrainEvents,ShutdownWorker}` appear nowhere in `WorkerLiveMxAccessSmokeTests.cs`, so the exact paths the Finding calls masked are still only proven against `FakeWorkerHarness` canned replies while the real implementations live in `Worker/Ipc/WorkerPipeSession.cs`. Residual work (two `[LiveMxAccessFact]`s, windev-only to author and verify) is specified in [60-testing-docs-gaps.md](60-testing-docs-gaps.md#tst-05--real-worker-controlcom-paths-verified-opt-in-only---medium--p1). |
|
||||
| 2026-07-10 | **TST-15 design fleshed out** (still `Not started` — design only, not implementation): `docs/plans/2026-07-10-dashboard-session-acl-tst15.md`. Resolves the crux the deferral left open — the dashboard is LDAP-identity (Admin/Viewer) while sessions are API-key-owned (`OwnerKeyId`), two disjoint identity domains — via a **session tag** sourced from the owning API key (rides in the existing `ApiKeyConstraints` JSON blob, no SQLite migration). Admin-sees-all; Viewer may `SubscribeSession` iff `session.Tags ∩ viewer.GrantedTags ≠ ∅` (new `Dashboard:GroupToTag` map → hub-token tag claims); untagged sessions Admin-only by default (`Dashboard:UntaggedSessionVisibility`). Includes the enforcement path (`HubTokenPayload.Tags` + `IDashboardSessionAcl` gate at `SubscribeSession`), task breakdown (epic Tasks 16–19), test plan incl. live-LDAP, and rejected alternatives (client-supplied tag; group→key-id map). Tracker + `60-testing-docs-gaps.md` TST-15 section point at the doc. **TST-03 investigated:** the CI never ran because the repo had **zero registered Gitea Actions runners** (Actions is enabled; runs are created on push/PR/nightly but fail instantly with nothing to execute them). A Mac runner proved the pipeline executes but cannot clone — this Gitea hands runners the internal `http://gitea:3000` URL, reachable only by a runner co-located on the gitea Docker network. Fix = run a co-located runner on the Gitea host (recipe prepared, `scratchpad/gitea-runner/setup-gitea-host-runner.sh`); pending host access. TST-03 stays `In review`. |
|
||||
| 2026-07-09 | **P2 Epic wrap — user decision: DEFER TST-15 + TST-24, close the epic.** Epic bucket result: 5 of 7 findings `Done` (CLI-15, CLI-04, CLI-30, TST-01, TST-04); **TST-15** and **TST-24** intentionally deferred to a follow-up (kept `Not started`, not `Won't fix` — they are gated, not rejected). **TST-15** (dashboard EventsHub per-session ACL) is epic Phase 4 — a real feature needing a new session-"tag" mechanism + dashboard group→tag config, not a mechanical fix; the `EventsHub` `TODO(per-session-acl)` stays, and the already-shipped **SEC-25** mitigation (tag *values* redacted from the dashboard mirror by default) means no sensitive payload leaks through the hub today regardless of the missing ACL — so deferring carries no value-leak risk. **TST-24** (per-client wire tests) depends on **TST-03** (CI), which is `In review` (YAML authored, never run on a Gitea runner) — no point wiring client tests into a pipeline that isn't live yet. Net P2: 35/38 `Done`; remaining = TST-15 (deferred feature), TST-24 (deferred, CI-gated), TST-14 (user deletes their own untracked gitignored `*-docs-*.md` files). |
|
||||
| 2026-07-09 | P2 Epic — **Java client completes CLI-15 + CLI-04 locally** (commit `1cc0fa4`); **CLI-15, CLI-04, CLI-30, TST-01 all → `Done` (5/5 clients + server e2e)**. Java CLI-15: `MxEventStreamItem` record + `MxEventStream.nextItem()` (`isReplayGap()`/`replayGap()`/`event()`); existing `Iterator<MxEvent>` path unchanged, sentinel never swallowed. Java CLI-04: Phase 1 `adviseSupervisory`/`writeSecured`/`writeSecured2`/`authenticateUser`/`archestrAUserToId` + Phase 2 `addBufferedItem`/`setBufferedUpdateInterval`/`suspend`/`activate` (unregister already present) on `MxGatewaySession`, each through `invokeCommand` → `ensureProtocolSuccess`+`ensureMxAccessSuccess`; credentials scrubbed via `MxGatewaySecrets.redactCredentials` (tests assert absent from message/toString/CLI). `gradle test` 106/0 (58 client + 48 cli), no generated churn. Built locally with `JAVA_HOME=/opt/homebrew/opt/openjdk@17` — Java toolchain now works on the Mac (see prior note). Shared docs `ClientLibrariesDesign.md` + CLAUDE.md updated to "all five clients". **TST-01 → Done** (server e2e `fed0685` + all 5 client `ReplayGap` consumers). This closes session-resilience epic Phase 3 fully. |
|
||||
|
||||
@@ -159,6 +159,14 @@ This finding is the umbrella; TST-01/02/15 are its actionable slices. The remedi
|
||||
|
||||
## TST-05 — Real-worker control/COM paths verified opt-in only `Medium` · `P1`
|
||||
|
||||
> **Status revisit 2026-08-10 (unlocked by TST-25): `Partially done` — one half closed, one half open.**
|
||||
>
|
||||
> **Closed — the scheduled cadence.** The `nightly-windev` job in `.gitea/workflows/ci.yml` (cron `0 6 * * *`, gated `if: github.event_name == 'schedule'`) runs `scripts/ci/run-windev-ci.sh live`, which drives `scripts/ci/windev-worker-ci.ps1 -Mode live` on windev: x86 Worker build → full `Worker.Tests` → full-slnx build → `MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1 dotnet test … --filter FullyQualifiedName~WorkerLiveMxAccessSmokeTests`. A red nightly opens a Gitea issue, so nobody has to watch the Actions page. That is exactly this finding's **Design** paragraph, and it is the `live-mxaccess` job the design referred to (renamed; the removed job it originally pointed at is gone — see cycle-2 TST-25/TST-26).
|
||||
>
|
||||
> **Open — the coverage audit.** The design also required auditing `WorkerLiveMxAccessSmokeTests.cs` for coverage of *each* of the eleven late-added command kinds and adding missing `[LiveMxAccessFact]` cases. That audit now has an answer, and it is negative for five of the eleven. The suite's eight facts reach all six late-added **COM** commands (`Suspend`, `Activate`, `AuthenticateUser`, `ArchestrAUserToId`, `AddBufferedItem`, `SetBufferedUpdateInterval` — the `NewComCommands_RoundTripWithRealReplies` and `BufferedItem_*` facts). None of them sends any of the five **control** commands: `MxCommandKind.{Ping,GetSessionState,GetWorkerInfo,DrainEvents,ShutdownWorker}` do not appear anywhere in the file. Those are the very kinds the Finding below names as masked. The real worker answers them off-STA in `src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs` (dispatch switch at `:574`+), so the nightly exercises that code path only incidentally, never by assertion — a regression in `CreatePingReply`/`CreateSessionStateReply`/`CreateWorkerInfoReply`/the drain snapshot/the shutdown-after-reply ordering still ships green through both CI and the nightly.
|
||||
>
|
||||
> **Residual work to close TST-05 fully** (small, Windows-only): add one `[LiveMxAccessFact]` to `WorkerLiveMxAccessSmokeTests` that, against a live worker, invokes `Ping` → `GetSessionState` → `GetWorkerInfo` → `DrainEvents` and asserts each returns a non-`INVALID_REQUEST` reply carrying real worker state (e.g. `worker_process_id` matching the launched process), plus a separate fact for `ShutdownWorker` asserting the OK reply arrives *before* the worker exits and the session is then faulted/closed. `ShutdownWorker` needs `admin` scope and terminates the worker, so it must be the last fact in its own fixture. Not done here because it can only be authored and verified on windev with MXAccess installed; this revisit is doc/tracker-only.
|
||||
|
||||
**Finding.** All eleven late-added command kinds are unit-tested against fakes and live-verified once on the dev rig (`stillpending.md` §1.1), but the default suite exercises `Ping`/`GetWorkerInfo`/`DrainEvents`/`ShutdownWorker` only through `FakeWorkerHarness.RespondToControlCommandAsync` (verify current line range in `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/Fakes/FakeWorkerHarness.cs`), which returns canned replies.
|
||||
|
||||
**Impact.** A worker-side regression in these paths is invisible until someone sets `MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`.
|
||||
@@ -449,6 +457,24 @@ If TST-02's interim mitigation (flip retention off) is chosen instead of impleme
|
||||
|
||||
## TST-24 — Client wire behaviour has no automated verification `Low` · `—`
|
||||
|
||||
> **Resolution 2026-08-10 (branch `feat/tst-24-client-wire-tests`): `Done`.** All five clients now drive their public API against a fake gateway served over a real gRPC transport, in the client's own default suite, and all five run in CI.
|
||||
>
|
||||
> **Corrected premise.** The Finding's "no in-process gateway integration tests" was already stale when it was re-verified: **Go, Rust, and Java had real-server wire tests**, not mocks — Go's `newBufconnClient`/`fakeGatewayServer` (`clients/go/mxgateway/client_session_test.go`) over `grpc/test/bufconn`, Rust's `spawn_fake_gateway` (`clients/rust/tests/client_behavior.rs`) over a loopback `TcpListener` with tonic's `Server`, and Java's `InProcessGateway`/`TestGatewayService` (`MxGatewayClientSessionTests.java`) over `InProcessServerBuilder`. Each already asserted the round trip, the `authorization` bearer header as *observed by the server*, and the `ReplayGap` sentinel. The cycle-2 re-verification cited `clients/python/tests/test_replay_gap.py` as evidence for "the other four clients still unit-test against mocks"; that generalized from Python to Go/Rust/Java incorrectly. `InProcessGatewayHarness` (the "template" the Impact paragraph names) is in fact the *thinner* of the Java harnesses — it serves only `streamEvents`/`closeSession` for the CLI tests.
|
||||
>
|
||||
> **Real gap, and what was built.** Two clients genuinely had none. **.NET** substituted `FakeGatewayTransport` for `IMxGatewayClientTransport` in every test, so not even the generated stub ran, and its test project had no server package at all. **Python** monkeypatched `MxAccessGatewayStub` everywhere except one opt-in TLS test that served only `OpenSession`. Both now have the pattern:
|
||||
> - `clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/WireFakeGatewayServer.cs` + `MxGatewayClientWireTests.cs` — Kestrel h2c on `127.0.0.1:0` serving `MxAccessGateway.MxAccessGatewayBase`; needed new `Grpc.AspNetCore.Server` + `Microsoft.AspNetCore.App` references on the test project.
|
||||
> - `clients/python/tests/test_wire_fake_gateway.py` — a `grpc.aio` server on `127.0.0.1:0` serving `MxAccessGatewayServicer`; no new dependencies (`grpcio` is a runtime dep).
|
||||
>
|
||||
> Each covers the four shapes the Design asked for: round trip (`OpenSession` → `Invoke`/`Register` → `StreamEvents` → `CloseSession` with every reply field asserted), the bearer header as received by the server on the streaming RPC as well as the unary ones, the `ReplayGap` sentinel surfaced as the client's typed signal (TST-01), and a real `PERMISSION_DENIED` mapping to the typed authorization error.
|
||||
>
|
||||
> **CI.** The `portable` job previously only *built* the .NET client; a `dotnet test` step was added, so its wire tests actually run. Go/Rust/Python already ran their suites there and Java in the `java` job.
|
||||
>
|
||||
> **Bug this immediately caught** — the justification for the whole finding. `GatewayClient.connect()` / `GalaxyRepositoryClient.connect()` in the Python client were **broken for every real (non-stub) connection**: they built the `grpc.aio` channel inside `asyncio.to_thread`, and a `grpc.aio` channel binds to the event loop current on the constructing thread, so the worker thread raised `RuntimeError: There is no current event loop in thread 'asyncio_0'`. No mock-based test could see it — the one test asserting the off-loop behaviour (`Client.Python-028`) monkeypatched `create_channel` and therefore asserted the bug. Fixed by splitting `resolve_channel_security` (blocking TOFU probe, runs off-loop) from `create_channel` (must run on the loop thread), keeping the Client.Python-028 guarantee; the two `-028` tests were retargeted to assert both halves.
|
||||
>
|
||||
> **Deliberately out of scope.** Only the four session RPCs are served — the alarm feed (`StreamAlarms`, `QueryActiveAlarms`, `AcknowledgeAlarm`) and Galaxy browse are not, matching the Design's "full parity is out of scope". Java's `InProcessGatewayHarness` still lacks `openSession`/`invoke`; the client-module tests cover those shapes, so it was left alone.
|
||||
>
|
||||
> **Docs.** `docs/GatewayTesting.md` § Client Wire Tests (the cross-client pattern + per-client harness table), `clients/dotnet/README.md`, `clients/python/README.md`.
|
||||
|
||||
**Finding.** All five clients have unit tests (13/8/3/13/7 files for dotnet/go/rust/python/java) but no in-process or containerized gateway integration tests; the only cross-language verification is the operator-run `scripts/run-client-e2e-tests.ps1`. `CrossLanguageSmokeMatrixTests` checks shapes only.
|
||||
|
||||
**Impact.** Low-to-moderate: a gateway contract change can pass every default suite and break all five clients (partly mitigated by shared-proto codegen). The Java CLI already proves the cheap pattern — `InProcessGatewayHarness` (`stillpending.md` §8).
|
||||
|
||||
@@ -23,6 +23,20 @@ dotnet build clients/dotnet/ZB.MOM.WW.MxGateway.Client.slnx
|
||||
dotnet test clients/dotnet/ZB.MOM.WW.MxGateway.Client.slnx --no-build
|
||||
```
|
||||
|
||||
Most tests substitute `FakeGatewayTransport` for `IMxGatewayClientTransport`, so
|
||||
they never touch the wire. `MxGatewayClientWireTests` is the exception: it drives
|
||||
the ordinary public API against `WireFakeGatewayServer`, a real gRPC server
|
||||
(Kestrel h2c on an ephemeral loopback port) serving
|
||||
`MxAccessGateway.MxAccessGatewayBase`. Only the gateway's behaviour is canned —
|
||||
the HTTP/2 framing, protobuf serialization, `authorization` metadata, and gRPC
|
||||
status codes are genuine, so it catches decode and metadata breaks a transport
|
||||
fake cannot see. No MXAccess or worker is involved; it runs in the default suite.
|
||||
See `docs/GatewayTesting.md` (Client Wire Tests) for the cross-client pattern.
|
||||
|
||||
```powershell
|
||||
dotnet test clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/ZB.MOM.WW.MxGateway.Client.Tests.csproj --filter FullyQualifiedName~MxGatewayClientWireTests
|
||||
```
|
||||
|
||||
## Packaging
|
||||
|
||||
Create local library and CLI artifacts from the repository root:
|
||||
|
||||
@@ -1418,14 +1418,16 @@ public static class MxGatewayClientCli
|
||||
.WithCancellation(cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
if (jsonLines)
|
||||
{
|
||||
output.WriteLine(ProtobufJsonFormatter.Format(gatewayEvent));
|
||||
}
|
||||
else if (json)
|
||||
if (json && !jsonLines)
|
||||
{
|
||||
events.Add(gatewayEvent);
|
||||
}
|
||||
else if (gatewayEvent.ReplayGap is { } replayGap)
|
||||
{
|
||||
// Render the ReplayGap sentinel as the typed cross-CLI row instead of the raw
|
||||
// sentinel MxEvent (NEXT-02, mirroring the Go/Python/Rust CLIs).
|
||||
output.WriteLine(FormatReplayGapRow(replayGap));
|
||||
}
|
||||
else
|
||||
{
|
||||
output.WriteLine(ProtobufJsonFormatter.Format(gatewayEvent));
|
||||
@@ -1835,7 +1837,31 @@ public static class MxGatewayClientCli
|
||||
|
||||
private static JsonElement EventToJsonElement(MxEvent gatewayEvent)
|
||||
{
|
||||
return JsonDocument.Parse(ProtobufJsonFormatter.Format(gatewayEvent)).RootElement.Clone();
|
||||
string row = gatewayEvent.ReplayGap is { } replayGap
|
||||
? FormatReplayGapRow(replayGap)
|
||||
: ProtobufJsonFormatter.Format(gatewayEvent);
|
||||
return JsonDocument.Parse(row).RootElement.Clone();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Formats the typed ReplayGap row shared by the CLIs (NEXT-02). Hand-built so the
|
||||
/// cursors are JSON numbers like the Go/Python/Rust rows, not the protobuf JSON
|
||||
/// formatter's quoted uint64 strings.
|
||||
/// </summary>
|
||||
/// <param name="replayGap">Replay gap sentinel payload.</param>
|
||||
/// <returns>A single-line JSON row describing the gap.</returns>
|
||||
private static string FormatReplayGapRow(ReplayGap replayGap)
|
||||
{
|
||||
return JsonSerializer.Serialize(
|
||||
new
|
||||
{
|
||||
replayGap = new
|
||||
{
|
||||
requestedAfterSequence = replayGap.RequestedAfterSequence,
|
||||
oldestAvailableSequence = replayGap.OldestAvailableSequence,
|
||||
},
|
||||
},
|
||||
JsonOptions);
|
||||
}
|
||||
|
||||
private static MxValue ParseValue(CliArguments arguments)
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Text.Json;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using ZB.MOM.WW.MxGateway.Client.Cli;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
@@ -585,6 +586,84 @@ public sealed class MxGatewayClientCliTests
|
||||
Assert.DoesNotContain("ON_WRITE_COMPLETE", output.ToString());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies stream-events renders the ReplayGap sentinel as the typed cross-CLI row —
|
||||
/// numeric cursors under a replayGap key — instead of the raw sentinel MxEvent (NEXT-02).
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_StreamEvents_RendersReplayGapAsTypedRow()
|
||||
{
|
||||
using var output = new StringWriter();
|
||||
using var error = new StringWriter();
|
||||
FakeCliClient fakeClient = new();
|
||||
fakeClient.Events.Add(new MxEvent
|
||||
{
|
||||
ReplayGap = new ReplayGap
|
||||
{
|
||||
RequestedAfterSequence = 7,
|
||||
OldestAvailableSequence = 42,
|
||||
},
|
||||
});
|
||||
fakeClient.Events.Add(new MxEvent
|
||||
{
|
||||
SessionId = "session-fixture",
|
||||
Family = MxEventFamily.OnDataChange,
|
||||
WorkerSequence = 43,
|
||||
});
|
||||
|
||||
int exitCode = await MxGatewayClientCli.RunAsync(
|
||||
[
|
||||
"stream-events",
|
||||
"--endpoint",
|
||||
"http://localhost:5000",
|
||||
"--api-key",
|
||||
"test-api-key",
|
||||
"--session-id",
|
||||
"session-fixture",
|
||||
"--max-events",
|
||||
"2",
|
||||
],
|
||||
output,
|
||||
error,
|
||||
_ => fakeClient);
|
||||
|
||||
Assert.Equal(0, exitCode);
|
||||
string[] rows = output.ToString().Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries);
|
||||
Assert.Equal(2, rows.Length);
|
||||
using JsonDocument gapRow = JsonDocument.Parse(rows[0]);
|
||||
JsonElement gap = gapRow.RootElement.GetProperty("replayGap");
|
||||
Assert.Equal(7UL, gap.GetProperty("requestedAfterSequence").GetUInt64());
|
||||
Assert.Equal(42UL, gap.GetProperty("oldestAvailableSequence").GetUInt64());
|
||||
Assert.Equal(JsonValueKind.Number, gap.GetProperty("requestedAfterSequence").ValueKind);
|
||||
Assert.DoesNotContain("MX_EVENT_FAMILY_UNSPECIFIED", rows[0], StringComparison.Ordinal);
|
||||
Assert.Contains("workerSequence", rows[1], StringComparison.Ordinal);
|
||||
|
||||
// The aggregate --json shape carries the same typed row inside the events array.
|
||||
using var aggregateOutput = new StringWriter();
|
||||
int aggregateExit = await MxGatewayClientCli.RunAsync(
|
||||
[
|
||||
"stream-events",
|
||||
"--endpoint",
|
||||
"http://localhost:5000",
|
||||
"--api-key",
|
||||
"test-api-key",
|
||||
"--session-id",
|
||||
"session-fixture",
|
||||
"--max-events",
|
||||
"2",
|
||||
"--json",
|
||||
],
|
||||
aggregateOutput,
|
||||
error,
|
||||
_ => fakeClient);
|
||||
|
||||
Assert.Equal(0, aggregateExit);
|
||||
using JsonDocument aggregate = JsonDocument.Parse(aggregateOutput.ToString());
|
||||
JsonElement firstRow = aggregate.RootElement.GetProperty("events")[0];
|
||||
Assert.Equal(42UL, firstRow.GetProperty("replayGap").GetProperty("oldestAvailableSequence").GetUInt64());
|
||||
}
|
||||
|
||||
|
||||
/// <summary>Verifies that stream-alarms with --max-events stops output and distinguishes payload cases.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Client.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Drives the public client API against <see cref="WireFakeGatewayServer"/> — a real
|
||||
/// gRPC server on loopback — so the transport, protobuf serialization, call metadata,
|
||||
/// and gRPC status mapping are all exercised. Every other test in this project
|
||||
/// substitutes <see cref="FakeGatewayTransport"/> and therefore proves nothing about
|
||||
/// what actually crosses the wire.
|
||||
/// </summary>
|
||||
public sealed class MxGatewayClientWireTests
|
||||
{
|
||||
private const string ApiKey = "mxgw_wiretest_secret";
|
||||
|
||||
/// <summary>
|
||||
/// Verifies the full session happy path decodes real wire bytes end to end.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task SessionRoundTrip_OverRealTransport_DecodesEveryReplyField()
|
||||
{
|
||||
await using WireFakeGatewayServer server = await WireFakeGatewayServer.StartAsync();
|
||||
await using MxGatewayClient client = server.CreateClient(ApiKey);
|
||||
|
||||
MxGatewaySession session = await client.OpenSessionAsync(
|
||||
new OpenSessionRequest { ClientSessionName = "wire-test" });
|
||||
|
||||
Assert.Equal(WireFakeGatewayServer.FakeGatewayService.SessionId, session.SessionId);
|
||||
Assert.Equal("fake-backend", session.OpenSessionReply.BackendName);
|
||||
Assert.Equal(1234, session.OpenSessionReply.WorkerProcessId);
|
||||
Assert.Equal(3u, session.OpenSessionReply.GatewayProtocolVersion);
|
||||
Assert.Equal(["events", "invoke"], session.OpenSessionReply.Capabilities);
|
||||
|
||||
int serverHandle = await session.RegisterAsync("wire-test-client");
|
||||
Assert.Equal(WireFakeGatewayServer.FakeGatewayService.ServerHandle, serverHandle);
|
||||
|
||||
MxCommandRequest? invoke = server.Service.InvokeRequest;
|
||||
Assert.NotNull(invoke);
|
||||
Assert.Equal(MxCommandKind.Register, invoke.Command.Kind);
|
||||
Assert.Equal("wire-test-client", invoke.Command.Register.ClientName);
|
||||
|
||||
List<MxEvent> events = await CollectAsync(client.StreamEventsAsync(
|
||||
new StreamEventsRequest { SessionId = session.SessionId }));
|
||||
|
||||
MxEvent single = Assert.Single(events);
|
||||
Assert.Equal(MxEventFamily.OnDataChange, single.Family);
|
||||
Assert.Equal(WireFakeGatewayServer.FakeGatewayService.ServerHandle, single.ServerHandle);
|
||||
Assert.Equal(WireFakeGatewayServer.FakeGatewayService.ItemHandle, single.ItemHandle);
|
||||
Assert.Equal(17, single.Value.Int32Value);
|
||||
Assert.Equal(192, single.Quality);
|
||||
Assert.Equal(9ul, single.WorkerSequence);
|
||||
Assert.Equal(MxEvent.BodyOneofCase.OnDataChange, single.BodyCase);
|
||||
|
||||
CloseSessionReply closeReply = await session.CloseAsync();
|
||||
Assert.Equal(SessionState.Closed, closeReply.FinalState);
|
||||
Assert.Equal(
|
||||
WireFakeGatewayServer.FakeGatewayService.SessionId,
|
||||
server.Service.CloseSessionRequest?.SessionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies the API key reaches the server as a bearer header on unary and
|
||||
/// streaming calls alike. A transport fake can only assert what the client passes;
|
||||
/// this asserts what the server receives.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ApiKey_ReachesTheServerAsBearerMetadata_OnEveryRpc()
|
||||
{
|
||||
await using WireFakeGatewayServer server = await WireFakeGatewayServer.StartAsync();
|
||||
await using MxGatewayClient client = server.CreateClient(ApiKey);
|
||||
|
||||
MxGatewaySession session = await client.OpenSessionAsync(
|
||||
new OpenSessionRequest { ClientSessionName = "wire-test" });
|
||||
await session.RegisterAsync("wire-test-client");
|
||||
await CollectAsync(client.StreamEventsAsync(
|
||||
new StreamEventsRequest { SessionId = session.SessionId }));
|
||||
await session.CloseAsync();
|
||||
|
||||
string expected = $"Bearer {ApiKey}";
|
||||
Assert.Equal(
|
||||
new Dictionary<string, string>
|
||||
{
|
||||
["OpenSession"] = expected,
|
||||
["Invoke"] = expected,
|
||||
["StreamEvents"] = expected,
|
||||
["CloseSession"] = expected,
|
||||
},
|
||||
server.Service.AuthorizationByMethod);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies the gateway's replay-gap sentinel survives serialization and is
|
||||
/// surfaced as a typed, non-terminal stream item.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task ReplayGapSentinel_SurvivesTheWire_AsTypedStreamItem()
|
||||
{
|
||||
await using WireFakeGatewayServer server = await WireFakeGatewayServer.StartAsync(service =>
|
||||
service.ReplayGap = new ReplayGap
|
||||
{
|
||||
RequestedAfterSequence = 3,
|
||||
OldestAvailableSequence = 8,
|
||||
});
|
||||
await using MxGatewayClient client = server.CreateClient(ApiKey);
|
||||
|
||||
MxGatewaySession session = await client.OpenSessionAsync(
|
||||
new OpenSessionRequest { ClientSessionName = "wire-test" });
|
||||
|
||||
List<MxEventStreamItem> items = [];
|
||||
IAsyncEnumerable<MxEvent> stream = client.StreamEventsAsync(new StreamEventsRequest
|
||||
{
|
||||
SessionId = session.SessionId,
|
||||
AfterWorkerSequence = 3,
|
||||
});
|
||||
|
||||
await foreach (MxEventStreamItem item in stream.AsStreamItemsAsync())
|
||||
{
|
||||
items.Add(item);
|
||||
}
|
||||
|
||||
Assert.Equal(2, items.Count);
|
||||
Assert.True(items[0].IsReplayGap);
|
||||
Assert.Equal(3ul, items[0].ReplayGap!.RequestedAfterSequence);
|
||||
Assert.Equal(8ul, items[0].ReplayGap!.OldestAvailableSequence);
|
||||
Assert.False(items[1].IsReplayGap);
|
||||
Assert.Equal(MxEventFamily.OnDataChange, items[1].Event.Family);
|
||||
Assert.Equal(3ul, server.Service.StreamEventsRequest?.AfterWorkerSequence);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies a genuine <c>PERMISSION_DENIED</c> status maps to the typed client
|
||||
/// exception rather than a bare RpcException.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task PermissionDeniedStatus_MapsToAuthorizationException()
|
||||
{
|
||||
await using WireFakeGatewayServer server = await WireFakeGatewayServer.StartAsync(
|
||||
service => service.DenyInvoke = true);
|
||||
await using MxGatewayClient client = server.CreateClient(ApiKey);
|
||||
|
||||
MxGatewaySession session = await client.OpenSessionAsync(
|
||||
new OpenSessionRequest { ClientSessionName = "wire-test" });
|
||||
|
||||
await Assert.ThrowsAsync<MxGatewayAuthorizationException>(
|
||||
() => session.RegisterAsync("wire-test-client"));
|
||||
}
|
||||
|
||||
private static async Task<List<MxEvent>> CollectAsync(IAsyncEnumerable<MxEvent> stream)
|
||||
{
|
||||
List<MxEvent> events = [];
|
||||
await foreach (MxEvent gatewayEvent in stream)
|
||||
{
|
||||
events.Add(gatewayEvent);
|
||||
}
|
||||
|
||||
return events;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,264 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Net;
|
||||
using Grpc.Core;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.AspNetCore.Hosting;
|
||||
using Microsoft.AspNetCore.Hosting.Server;
|
||||
using Microsoft.AspNetCore.Hosting.Server.Features;
|
||||
using Microsoft.AspNetCore.Server.Kestrel.Core;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Client.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// Hosts the real <c>mxaccess_gateway.v1.MxAccessGateway</c> service on a loopback
|
||||
/// Kestrel endpoint so client tests exercise genuine HTTP/2 framing, protobuf
|
||||
/// serialization, call metadata, and gRPC status propagation.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// This is the counterpart of <see cref="FakeGatewayTransport"/>: that fake replaces
|
||||
/// <c>IMxGatewayClientTransport</c>, so nothing below the client wrapper runs. This one
|
||||
/// replaces only the gateway's <em>behaviour</em> — every byte between the client and
|
||||
/// the service is the real wire format. Contract breaks that a transport fake cannot
|
||||
/// see (a field the client never decodes, metadata it does not actually send, a status
|
||||
/// code it maps differently once it arrives as a real <see cref="RpcException"/>) fail
|
||||
/// here.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Plaintext h2c is used deliberately: TLS is covered by
|
||||
/// <c>MxGatewayClientTlsHandlerTests</c>, and h2c keeps the harness certificate-free so
|
||||
/// it runs identically on every CI host. See <c>docs/GatewayTesting.md</c>
|
||||
/// (Client Wire Tests) for the shared pattern and its Python counterpart.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
internal sealed class WireFakeGatewayServer : IAsyncDisposable
|
||||
{
|
||||
private readonly WebApplication _app;
|
||||
|
||||
private WireFakeGatewayServer(WebApplication app, FakeGatewayService service, int port)
|
||||
{
|
||||
_app = app;
|
||||
Service = service;
|
||||
Endpoint = new Uri($"http://127.0.0.1:{port}");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the canned service backing the endpoint; tests read its recorded requests.
|
||||
/// </summary>
|
||||
public FakeGatewayService Service { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the h2c endpoint to point <see cref="MxGatewayClientOptions.Endpoint"/> at.
|
||||
/// </summary>
|
||||
public Uri Endpoint { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Starts a server on an ephemeral loopback port.
|
||||
/// </summary>
|
||||
/// <param name="configure">Optional configuration of the canned service.</param>
|
||||
/// <returns>The started server.</returns>
|
||||
public static async Task<WireFakeGatewayServer> StartAsync(Action<FakeGatewayService>? configure = null)
|
||||
{
|
||||
FakeGatewayService service = new();
|
||||
configure?.Invoke(service);
|
||||
|
||||
WebApplicationBuilder builder = WebApplication.CreateBuilder();
|
||||
builder.Logging.ClearProviders();
|
||||
builder.WebHost.ConfigureKestrel(options =>
|
||||
// Port 0 lets the OS pick; HTTP/2 without TLS (h2c) is what the client's
|
||||
// plain http:// endpoint negotiates via RequestVersionExact.
|
||||
options.Listen(IPAddress.Loopback, 0, listen => listen.Protocols = HttpProtocols.Http2));
|
||||
builder.Services.AddGrpc();
|
||||
builder.Services.AddSingleton(service);
|
||||
|
||||
WebApplication app = builder.Build();
|
||||
app.MapGrpcService<FakeGatewayService>();
|
||||
await app.StartAsync().ConfigureAwait(false);
|
||||
|
||||
return new WireFakeGatewayServer(app, service, ResolvePort(app));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Creates a client bound to this server's endpoint.
|
||||
/// </summary>
|
||||
/// <param name="apiKey">API key the client should present.</param>
|
||||
/// <returns>A client that talks to this server over h2c.</returns>
|
||||
public MxGatewayClient CreateClient(string apiKey) =>
|
||||
MxGatewayClient.Create(new MxGatewayClientOptions
|
||||
{
|
||||
Endpoint = Endpoint,
|
||||
ApiKey = apiKey,
|
||||
UseTls = false,
|
||||
DefaultCallTimeout = TimeSpan.FromSeconds(30),
|
||||
});
|
||||
|
||||
/// <inheritdoc />
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await _app.StopAsync().ConfigureAwait(false);
|
||||
await _app.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static int ResolvePort(WebApplication app)
|
||||
{
|
||||
IServerAddressesFeature? addresses = app.Services
|
||||
.GetRequiredService<IServer>()
|
||||
.Features
|
||||
.Get<IServerAddressesFeature>();
|
||||
string address = addresses?.Addresses.FirstOrDefault()
|
||||
?? throw new InvalidOperationException("Kestrel did not report a bound address.");
|
||||
return new Uri(address).Port;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Canned gateway answering the four session RPCs with gateway-shaped replies.
|
||||
/// </summary>
|
||||
internal sealed class FakeGatewayService : MxAccessGateway.MxAccessGatewayBase
|
||||
{
|
||||
/// <summary>The session id every reply carries.</summary>
|
||||
public const string SessionId = "wire-session-1";
|
||||
|
||||
/// <summary>The server handle the canned Register reply returns.</summary>
|
||||
public const int ServerHandle = 4242;
|
||||
|
||||
/// <summary>The item handle the canned data-change event carries.</summary>
|
||||
public const int ItemHandle = 77;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the <c>authorization</c> header value observed per RPC name.
|
||||
/// </summary>
|
||||
public ConcurrentDictionary<string, string> AuthorizationByMethod { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets a value indicating whether <c>Invoke</c> fails with
|
||||
/// <see cref="StatusCode.PermissionDenied"/> instead of replying.
|
||||
/// </summary>
|
||||
public bool DenyInvoke { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets the replay-gap sentinel emitted at the head of the event stream.
|
||||
/// </summary>
|
||||
public ReplayGap? ReplayGap { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the last <c>Invoke</c> request the client sent, as decoded from the wire.
|
||||
/// </summary>
|
||||
public MxCommandRequest? InvokeRequest { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the last <c>StreamEvents</c> request the client sent.
|
||||
/// </summary>
|
||||
public StreamEventsRequest? StreamEventsRequest { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the last <c>CloseSession</c> request the client sent.
|
||||
/// </summary>
|
||||
public CloseSessionRequest? CloseSessionRequest { get; private set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<OpenSessionReply> OpenSession(
|
||||
OpenSessionRequest request,
|
||||
ServerCallContext context)
|
||||
{
|
||||
Record(context);
|
||||
return Task.FromResult(new OpenSessionReply
|
||||
{
|
||||
SessionId = SessionId,
|
||||
BackendName = "fake-backend",
|
||||
WorkerProcessId = 1234,
|
||||
WorkerProtocolVersion = 1,
|
||||
GatewayProtocolVersion = 3,
|
||||
Capabilities = { "events", "invoke" },
|
||||
ProtocolStatus = Ok(),
|
||||
});
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<MxCommandReply> Invoke(MxCommandRequest request, ServerCallContext context)
|
||||
{
|
||||
Record(context);
|
||||
InvokeRequest = request;
|
||||
|
||||
if (DenyInvoke)
|
||||
{
|
||||
throw new RpcException(new Status(StatusCode.PermissionDenied, "invoke scope required"));
|
||||
}
|
||||
|
||||
return Task.FromResult(new MxCommandReply
|
||||
{
|
||||
SessionId = request.SessionId,
|
||||
CorrelationId = request.ClientCorrelationId,
|
||||
Kind = request.Command.Kind,
|
||||
ProtocolStatus = Ok(),
|
||||
Hresult = 0,
|
||||
Register = new RegisterReply { ServerHandle = ServerHandle },
|
||||
});
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task StreamEvents(
|
||||
StreamEventsRequest request,
|
||||
IServerStreamWriter<MxEvent> responseStream,
|
||||
ServerCallContext context)
|
||||
{
|
||||
Record(context);
|
||||
StreamEventsRequest = request;
|
||||
|
||||
if (ReplayGap is not null)
|
||||
{
|
||||
// The sentinel shape the gateway emits: family unspecified, body unset,
|
||||
// only replay_gap populated.
|
||||
await responseStream.WriteAsync(new MxEvent
|
||||
{
|
||||
SessionId = request.SessionId,
|
||||
ReplayGap = ReplayGap,
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await responseStream.WriteAsync(new MxEvent
|
||||
{
|
||||
SessionId = request.SessionId,
|
||||
Family = MxEventFamily.OnDataChange,
|
||||
ServerHandle = ServerHandle,
|
||||
ItemHandle = ItemHandle,
|
||||
Value = new MxValue { Int32Value = 17 },
|
||||
Quality = 192,
|
||||
WorkerSequence = 9,
|
||||
OnDataChange = new OnDataChangeEvent(),
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<CloseSessionReply> CloseSession(
|
||||
CloseSessionRequest request,
|
||||
ServerCallContext context)
|
||||
{
|
||||
Record(context);
|
||||
CloseSessionRequest = request;
|
||||
return Task.FromResult(new CloseSessionReply
|
||||
{
|
||||
SessionId = request.SessionId,
|
||||
FinalState = SessionState.Closed,
|
||||
ProtocolStatus = Ok(),
|
||||
});
|
||||
}
|
||||
|
||||
private static ProtocolStatus Ok() => new() { Code = ProtocolStatusCode.Ok };
|
||||
|
||||
private void Record(ServerCallContext context)
|
||||
{
|
||||
string? authorization = context.RequestHeaders.GetValue("authorization");
|
||||
if (authorization is not null)
|
||||
{
|
||||
// context.Method is the fully-qualified "/package.Service/Method";
|
||||
// key on the bare method name so assertions stay readable.
|
||||
AuthorizationByMethod[context.Method[(context.Method.LastIndexOf('/') + 1)..]] =
|
||||
authorization;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+9
@@ -12,6 +12,15 @@
|
||||
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
|
||||
<PackageReference Include="xunit" Version="2.9.3" />
|
||||
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.4" />
|
||||
<!-- Wire tests only (WireFakeGatewayServer): hosts the real MxAccessGateway service
|
||||
on loopback Kestrel so the client is driven over genuine HTTP/2 + protobuf rather
|
||||
than a substituted transport. Version tracks the gateway server's Grpc.AspNetCore
|
||||
(src/ZB.MOM.WW.MxGateway.Server) and the client's Grpc.Net.Client, both 2.76.0. -->
|
||||
<PackageReference Include="Grpc.AspNetCore.Server" Version="2.76.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<FrameworkReference Include="Microsoft.AspNetCore.App" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
+25
-4
@@ -5,6 +5,7 @@ import com.zb.mom.ww.mxgateway.client.DeployEventStream;
|
||||
import com.zb.mom.ww.mxgateway.client.GalaxyRepositoryClient;
|
||||
import com.zb.mom.ww.mxgateway.client.LazyBrowseNode;
|
||||
import com.zb.mom.ww.mxgateway.client.MxEventStream;
|
||||
import com.zb.mom.ww.mxgateway.client.MxEventStreamItem;
|
||||
import com.zb.mom.ww.mxgateway.client.MxGatewayAlarmFeedSubscription;
|
||||
import com.zb.mom.ww.mxgateway.client.MxGatewayClient;
|
||||
import com.zb.mom.ww.mxgateway.client.MxGatewayClientOptions;
|
||||
@@ -59,6 +60,7 @@ import mxaccess_gateway.v1.MxaccessGateway.MxValue;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.OnAlarmTransitionEvent;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.OpenSessionRequest;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.PingCommand;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.ReplayGap;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.StreamAlarmsRequest;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.SubscribeResult;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.Write2BulkEntry;
|
||||
@@ -1654,11 +1656,30 @@ public final class MxGatewayCli implements Callable<Integer> {
|
||||
MxEventStream events = client.session(sessionId).streamEventsAfter(afterWorkerSequence)) {
|
||||
int count = 0;
|
||||
while (events.hasNext()) {
|
||||
MxEvent event = events.next();
|
||||
if (json) {
|
||||
client.out().println(protoJson(event));
|
||||
MxEventStreamItem item = events.nextItem();
|
||||
if (item.isReplayGap()) {
|
||||
// Render the ReplayGap sentinel as the typed cross-CLI row (NEXT-02,
|
||||
// mirroring the Go/Python/Rust/.NET CLIs) instead of the raw sentinel
|
||||
// event, whose text form printed "0 MX_EVENT_FAMILY_UNSPECIFIED".
|
||||
ReplayGap gap = item.replayGap();
|
||||
if (json) {
|
||||
client.out().printf(
|
||||
"{\"replayGap\":{\"requestedAfterSequence\":%s,\"oldestAvailableSequence\":%s}}%n",
|
||||
Long.toUnsignedString(gap.getRequestedAfterSequence()),
|
||||
Long.toUnsignedString(gap.getOldestAvailableSequence()));
|
||||
} else {
|
||||
client.out().printf(
|
||||
"REPLAY_GAP requested_after=%s oldest_available=%s%n",
|
||||
Long.toUnsignedString(gap.getRequestedAfterSequence()),
|
||||
Long.toUnsignedString(gap.getOldestAvailableSequence()));
|
||||
}
|
||||
} else {
|
||||
client.out().printf("%d %s%n", event.getWorkerSequence(), event.getFamily());
|
||||
MxEvent event = item.event();
|
||||
if (json) {
|
||||
client.out().println(protoJson(event));
|
||||
} else {
|
||||
client.out().printf("%d %s%n", event.getWorkerSequence(), event.getFamily());
|
||||
}
|
||||
}
|
||||
count++;
|
||||
if (limit > 0 && count >= limit) {
|
||||
|
||||
+54
@@ -43,6 +43,7 @@ import mxaccess_gateway.v1.MxaccessGateway.OpenSessionRequest;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.ProtocolStatus;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.ProtocolStatusCode;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.RegisterReply;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.ReplayGap;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.SessionState;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.StreamAlarmsRequest;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.SubscribeResult;
|
||||
@@ -902,6 +903,59 @@ final class MxGatewayCliTests {
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void streamEventsRendersReplayGapAsTypedRow() {
|
||||
// NEXT-02: the ReplayGap sentinel must render as the typed cross-CLI
|
||||
// row (numeric cursors under a replayGap key in --json, a REPLAY_GAP
|
||||
// line in text mode), never as the raw sentinel event — text mode
|
||||
// used to print "0 MX_EVENT_FAMILY_UNSPECIFIED".
|
||||
MxEvent gap = MxEvent.newBuilder()
|
||||
.setReplayGap(ReplayGap.newBuilder()
|
||||
.setRequestedAfterSequence(7L)
|
||||
.setOldestAvailableSequence(42L)
|
||||
.build())
|
||||
.build();
|
||||
MxEvent dataChange = MxEvent.newBuilder()
|
||||
.setFamily(MxEventFamily.MX_EVENT_FAMILY_ON_DATA_CHANGE)
|
||||
.setSessionId("session-cli")
|
||||
.setWorkerSequence(43L)
|
||||
.build();
|
||||
|
||||
try (InProcessGatewayHarness harness = new InProcessGatewayHarness()) {
|
||||
harness.setScriptedEvents(List.of(gap, dataChange));
|
||||
CliRun jsonRun = execute(
|
||||
new HarnessClientFactory(harness),
|
||||
"stream-events",
|
||||
"--session-id",
|
||||
"session-cli",
|
||||
"--json");
|
||||
|
||||
assertEquals(0, jsonRun.exitCode(), "errors:\n" + jsonRun.errors());
|
||||
String jsonOut = jsonRun.output();
|
||||
assertTrue(
|
||||
jsonOut.contains(
|
||||
"{\"replayGap\":{\"requestedAfterSequence\":7,\"oldestAvailableSequence\":42}}"),
|
||||
jsonOut);
|
||||
assertTrue(jsonOut.contains("\"family\":\"MX_EVENT_FAMILY_ON_DATA_CHANGE\""), jsonOut);
|
||||
assertFalse(jsonOut.contains("MX_EVENT_FAMILY_UNSPECIFIED"), jsonOut);
|
||||
}
|
||||
|
||||
try (InProcessGatewayHarness harness = new InProcessGatewayHarness()) {
|
||||
harness.setScriptedEvents(List.of(gap, dataChange));
|
||||
CliRun textRun = execute(
|
||||
new HarnessClientFactory(harness),
|
||||
"stream-events",
|
||||
"--session-id",
|
||||
"session-cli");
|
||||
|
||||
assertEquals(0, textRun.exitCode(), "errors:\n" + textRun.errors());
|
||||
String textOut = textRun.output();
|
||||
assertTrue(textOut.contains("REPLAY_GAP requested_after=7 oldest_available=42"), textOut);
|
||||
assertFalse(textOut.contains("MX_EVENT_FAMILY_UNSPECIFIED"), textOut);
|
||||
assertTrue(textOut.contains("43 MX_EVENT_FAMILY_ON_DATA_CHANGE"), textOut);
|
||||
}
|
||||
}
|
||||
|
||||
// ---- galaxy-discover / galaxy-watch over the in-process harness (Task 6) ----
|
||||
|
||||
@Test
|
||||
|
||||
@@ -47,6 +47,19 @@ The tests import the generated gateway and worker stubs, run fake async gateway
|
||||
stubs, verify API key metadata, exercise stream cancellation, load shared value
|
||||
and command fixtures, and check deterministic CLI output.
|
||||
|
||||
`tests/test_wire_fake_gateway.py` is the one suite that does **not** substitute a
|
||||
stub: it serves a canned `MxAccessGatewayServicer` from a real `grpc.aio` server
|
||||
on an ephemeral loopback port and drives the ordinary `GatewayClient` API against
|
||||
it. Only the gateway's behaviour is canned — the HTTP/2 framing, protobuf
|
||||
serialization, `authorization` metadata, and gRPC status codes are genuine, so it
|
||||
catches decode and metadata breaks a stub fake cannot see. No MXAccess, no worker,
|
||||
no TLS, so it runs in the default suite. See `docs/GatewayTesting.md`
|
||||
(Client Wire Tests) for the cross-client pattern.
|
||||
|
||||
```powershell
|
||||
python -m pytest tests/test_wire_fake_gateway.py
|
||||
```
|
||||
|
||||
## Packaging
|
||||
|
||||
Install the package in editable mode for local development:
|
||||
@@ -398,6 +411,15 @@ point: the `require_certificate_validation=True` keyword on
|
||||
`--require-certificate-validation` CLI flag. See
|
||||
[Gateway Configuration](../../docs/GatewayConfiguration.md#automatic-self-signed-certificate).
|
||||
|
||||
Channel construction is split in two: `resolve_channel_security(options)` performs
|
||||
the blocking part (the trust-on-first-use certificate probe) and
|
||||
`create_channel(options, security=...)` builds the channel. The async `connect`
|
||||
classmethods run the first off the event loop and the second on it, because a
|
||||
`grpc.aio` channel binds to the event loop current on the constructing thread —
|
||||
building it inside `asyncio.to_thread` raises
|
||||
`RuntimeError: There is no current event loop in thread 'asyncio_N'`. Callers that
|
||||
build their own channel should keep `create_channel` on the loop thread.
|
||||
|
||||
## CLI
|
||||
|
||||
The CLI emits deterministic JSON for automation:
|
||||
|
||||
@@ -12,7 +12,7 @@ from .auth import merge_metadata
|
||||
from .errors import ensure_protocol_success, map_rpc_error
|
||||
from .generated import mxaccess_gateway_pb2 as pb
|
||||
from .generated import mxaccess_gateway_pb2_grpc as pb_grpc
|
||||
from .options import ClientOptions, create_channel
|
||||
from .options import ClientOptions, create_channel, resolve_channel_security
|
||||
|
||||
|
||||
class GatewayClient:
|
||||
@@ -58,9 +58,13 @@ class GatewayClient:
|
||||
if stub is not None:
|
||||
return cls(options=resolved, stub=stub)
|
||||
|
||||
# create_channel may perform a blocking TLS certificate probe (TOFU
|
||||
# default); run it off the event loop so connect never freezes the loop.
|
||||
channel = await asyncio.to_thread(create_channel, resolved)
|
||||
# Resolving security may perform a blocking TLS certificate probe (TOFU
|
||||
# default); run that off the event loop so connect never freezes it. The
|
||||
# channel itself must be built on the loop thread — a grpc.aio channel
|
||||
# binds to the loop current on the constructing thread, and a worker
|
||||
# thread has none.
|
||||
security = await asyncio.to_thread(resolve_channel_security, resolved)
|
||||
channel = create_channel(resolved, security=security)
|
||||
return cls(
|
||||
options=resolved,
|
||||
stub=pb_grpc.MxAccessGatewayStub(channel),
|
||||
|
||||
@@ -21,7 +21,12 @@ from .auth import merge_metadata
|
||||
from .errors import MxGatewayError, map_rpc_error
|
||||
from .generated import galaxy_repository_pb2 as galaxy_pb
|
||||
from .generated import galaxy_repository_pb2_grpc as galaxy_pb_grpc
|
||||
from .options import BrowseChildrenOptions, ClientOptions, create_channel
|
||||
from .options import (
|
||||
BrowseChildrenOptions,
|
||||
ClientOptions,
|
||||
create_channel,
|
||||
resolve_channel_security,
|
||||
)
|
||||
|
||||
_DISCOVER_HIERARCHY_PAGE_SIZE = 5000
|
||||
_BROWSE_CHILDREN_PAGE_SIZE = 500
|
||||
@@ -70,9 +75,13 @@ class GalaxyRepositoryClient:
|
||||
if stub is not None:
|
||||
return cls(options=resolved, stub=stub)
|
||||
|
||||
# create_channel may perform a blocking TLS certificate probe (TOFU
|
||||
# default); run it off the event loop so connect never freezes the loop.
|
||||
channel = await asyncio.to_thread(create_channel, resolved)
|
||||
# Resolving security may perform a blocking TLS certificate probe (TOFU
|
||||
# default); run that off the event loop so connect never freezes it. The
|
||||
# channel itself must be built on the loop thread — a grpc.aio channel
|
||||
# binds to the loop current on the constructing thread, and a worker
|
||||
# thread has none.
|
||||
security = await asyncio.to_thread(resolve_channel_security, resolved)
|
||||
channel = create_channel(resolved, security=security)
|
||||
return cls(
|
||||
options=resolved,
|
||||
stub=galaxy_pb_grpc.GalaxyRepositoryStub(channel),
|
||||
|
||||
@@ -105,7 +105,72 @@ def _split_authority(endpoint: str) -> tuple[str, int]:
|
||||
return (host or "localhost", int(port))
|
||||
|
||||
|
||||
def create_channel(options: ClientOptions) -> grpc.aio.Channel:
|
||||
@dataclass(frozen=True)
|
||||
class ChannelSecurity:
|
||||
"""Transport security resolved for one channel.
|
||||
|
||||
`credentials` is `None` for a plaintext channel. `target_name_override` is
|
||||
the SNI/authority override the TOFU path needs, kept separate from the
|
||||
caller's explicit `server_name_override` so the caller always wins.
|
||||
"""
|
||||
|
||||
credentials: grpc.ChannelCredentials | None = None
|
||||
target_name_override: str | None = None
|
||||
|
||||
|
||||
def resolve_channel_security(options: ClientOptions) -> ChannelSecurity:
|
||||
"""Resolve transport security for `options`, running any blocking probe.
|
||||
|
||||
This is the only blocking part of channel construction: the TOFU path opens
|
||||
a real TCP+TLS socket to fetch the server's certificate. It is split out of
|
||||
`create_channel` because a `grpc.aio` channel binds to the event loop
|
||||
*current on the constructing thread*, so the channel itself must be built on
|
||||
the loop thread — building it inside `asyncio.to_thread` raises
|
||||
``RuntimeError: There is no current event loop in thread 'asyncio_N'``. The
|
||||
async `connect` classmethods therefore run this function off the loop and
|
||||
then call `create_channel` on it.
|
||||
"""
|
||||
|
||||
if options.plaintext:
|
||||
return ChannelSecurity()
|
||||
|
||||
if options.ca_file:
|
||||
root_certificates = Path(options.ca_file).read_bytes()
|
||||
return ChannelSecurity(
|
||||
credentials=grpc.ssl_channel_credentials(root_certificates=root_certificates)
|
||||
)
|
||||
|
||||
if options.require_certificate_validation:
|
||||
return ChannelSecurity(credentials=grpc.ssl_channel_credentials())
|
||||
|
||||
# Lenient default: grpc-python has no per-channel skip-verify, so fetch the
|
||||
# server's certificate (unverified) and pin it for this channel (TOFU).
|
||||
# The probe opens a real blocking TCP+TLS socket, so it MUST be bounded —
|
||||
# a black-holed / firewall-drop host would otherwise hang on the OS default
|
||||
# connect timeout (minutes). Bound it by call_timeout (or a short fixed
|
||||
# fallback) so the dial fails fast as a transport error.
|
||||
host, port = _split_authority(options.endpoint)
|
||||
probe_timeout = options.call_timeout if options.call_timeout else _TOFU_PROBE_TIMEOUT_SECONDS
|
||||
try:
|
||||
presented = ssl.get_server_certificate((host, port), timeout=probe_timeout)
|
||||
except OSError as error:
|
||||
raise MxGatewayTransportError(
|
||||
f"failed to fetch TLS certificate from {options.endpoint}: {error}"
|
||||
) from error
|
||||
# The gateway self-signed cert always carries a "localhost" SAN, so default
|
||||
# the SNI/target-name override to it when none was supplied, tolerating
|
||||
# dial-by-IP or hostname mismatch.
|
||||
return ChannelSecurity(
|
||||
credentials=grpc.ssl_channel_credentials(root_certificates=presented.encode("ascii")),
|
||||
target_name_override="localhost",
|
||||
)
|
||||
|
||||
|
||||
def create_channel(
|
||||
options: ClientOptions,
|
||||
*,
|
||||
security: ChannelSecurity | None = None,
|
||||
) -> grpc.aio.Channel:
|
||||
"""Create a plaintext or TLS `grpc.aio` channel from client options.
|
||||
|
||||
The TLS default is lenient: grpc-python has no per-channel skip-verify, so
|
||||
@@ -113,48 +178,29 @@ def create_channel(options: ClientOptions) -> grpc.aio.Channel:
|
||||
as the channel's only trust root (trust-on-first-use). Set
|
||||
`require_certificate_validation=True` to force system-trust verification, or
|
||||
pass `ca_file` to verify against a specific CA — both bypass the TOFU path.
|
||||
|
||||
Pass *security* to reuse a `ChannelSecurity` already resolved off the event
|
||||
loop by `resolve_channel_security`; omit it and this call resolves (and may
|
||||
block) inline. Must run on the thread owning the event loop the channel will
|
||||
be used from.
|
||||
"""
|
||||
|
||||
security = security if security is not None else resolve_channel_security(options)
|
||||
|
||||
channel_options: list[tuple[str, str | int]] = [
|
||||
("grpc.max_receive_message_length", options.max_grpc_message_bytes),
|
||||
("grpc.max_send_message_length", options.max_grpc_message_bytes),
|
||||
]
|
||||
if options.server_name_override:
|
||||
channel_options.append(("grpc.ssl_target_name_override", options.server_name_override))
|
||||
elif security.target_name_override:
|
||||
channel_options.append(("grpc.ssl_target_name_override", security.target_name_override))
|
||||
|
||||
if options.plaintext:
|
||||
if security.credentials is None:
|
||||
return grpc.aio.insecure_channel(options.endpoint, options=channel_options)
|
||||
|
||||
if options.ca_file:
|
||||
root_certificates = Path(options.ca_file).read_bytes()
|
||||
credentials = grpc.ssl_channel_credentials(root_certificates=root_certificates)
|
||||
elif options.require_certificate_validation:
|
||||
credentials = grpc.ssl_channel_credentials()
|
||||
else:
|
||||
# Lenient default: grpc-python has no per-channel skip-verify, so fetch the
|
||||
# server's certificate (unverified) and pin it for this channel (TOFU).
|
||||
# The probe opens a real blocking TCP+TLS socket, so it MUST be bounded —
|
||||
# a black-holed / firewall-drop host would otherwise hang on the OS default
|
||||
# connect timeout (minutes). Bound it by call_timeout (or a short fixed
|
||||
# fallback) so the dial fails fast as a transport error. The async
|
||||
# `connect` classmethods run this off the event loop (asyncio.to_thread).
|
||||
host, port = _split_authority(options.endpoint)
|
||||
probe_timeout = options.call_timeout if options.call_timeout else _TOFU_PROBE_TIMEOUT_SECONDS
|
||||
try:
|
||||
presented = ssl.get_server_certificate((host, port), timeout=probe_timeout)
|
||||
except OSError as error:
|
||||
raise MxGatewayTransportError(
|
||||
f"failed to fetch TLS certificate from {options.endpoint}: {error}"
|
||||
) from error
|
||||
credentials = grpc.ssl_channel_credentials(root_certificates=presented.encode("ascii"))
|
||||
# The gateway self-signed cert always carries a "localhost" SAN, so default
|
||||
# the SNI/target-name override to it when none was supplied, tolerating
|
||||
# dial-by-IP or hostname mismatch.
|
||||
if not options.server_name_override:
|
||||
channel_options.append(("grpc.ssl_target_name_override", "localhost"))
|
||||
|
||||
return grpc.aio.secure_channel(
|
||||
options.endpoint,
|
||||
credentials,
|
||||
security.credentials,
|
||||
options=channel_options,
|
||||
)
|
||||
|
||||
@@ -12,6 +12,7 @@ from zb_mom_ww_mxgateway import client as client_module
|
||||
from zb_mom_ww_mxgateway import galaxy as galaxy_module
|
||||
from zb_mom_ww_mxgateway.galaxy import GalaxyRepositoryClient
|
||||
from zb_mom_ww_mxgateway.generated import mxaccess_gateway_pb2 as pb
|
||||
from zb_mom_ww_mxgateway.options import ChannelSecurity
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -21,11 +22,12 @@ async def test_gateway_connect_forwards_require_certificate_validation(
|
||||
"""The connect convenience kwarg must reach ClientOptions (Client.Python-027)."""
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def fake_create_channel(options: ClientOptions) -> object:
|
||||
def fake_resolve(options: ClientOptions) -> ChannelSecurity:
|
||||
captured["options"] = options
|
||||
return object()
|
||||
return ChannelSecurity()
|
||||
|
||||
monkeypatch.setattr(client_module, "create_channel", fake_create_channel)
|
||||
monkeypatch.setattr(client_module, "resolve_channel_security", fake_resolve)
|
||||
monkeypatch.setattr(client_module, "create_channel", _stub_create_channel)
|
||||
monkeypatch.setattr(client_module.pb_grpc, "MxAccessGatewayStub", lambda channel: object())
|
||||
|
||||
await GatewayClient.connect(
|
||||
@@ -43,11 +45,12 @@ async def test_galaxy_connect_forwards_require_certificate_validation(
|
||||
"""GalaxyRepositoryClient.connect must thread the flag too (Client.Python-027)."""
|
||||
captured: dict[str, Any] = {}
|
||||
|
||||
def fake_create_channel(options: ClientOptions) -> object:
|
||||
def fake_resolve(options: ClientOptions) -> ChannelSecurity:
|
||||
captured["options"] = options
|
||||
return object()
|
||||
return ChannelSecurity()
|
||||
|
||||
monkeypatch.setattr(galaxy_module, "create_channel", fake_create_channel)
|
||||
monkeypatch.setattr(galaxy_module, "resolve_channel_security", fake_resolve)
|
||||
monkeypatch.setattr(galaxy_module, "create_channel", _stub_create_channel)
|
||||
monkeypatch.setattr(
|
||||
galaxy_module.galaxy_pb_grpc, "GalaxyRepositoryStub", lambda channel: object()
|
||||
)
|
||||
@@ -61,52 +64,67 @@ async def test_galaxy_connect_forwards_require_certificate_validation(
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_gateway_connect_runs_create_channel_off_the_event_loop(
|
||||
async def test_gateway_connect_splits_probe_off_loop_and_channel_on_loop(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""connect must run the blocking channel factory off the loop (Client.Python-028)."""
|
||||
ran_in_thread: dict[str, bool] = {}
|
||||
"""The blocking probe runs off the loop; the channel is built on it.
|
||||
|
||||
def fake_create_channel(options: ClientOptions) -> object:
|
||||
# If this runs on the event loop thread, get_running_loop() succeeds.
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
ran_in_thread["off_loop"] = False
|
||||
except RuntimeError:
|
||||
ran_in_thread["off_loop"] = True
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr(client_module, "create_channel", fake_create_channel)
|
||||
Client.Python-028 required the blocking TOFU probe off the event loop. The
|
||||
channel itself must nonetheless be constructed *on* the loop thread: a
|
||||
``grpc.aio`` channel binds to the loop current on the constructing thread,
|
||||
and a ``to_thread`` worker has none, so building it off-loop raises
|
||||
``RuntimeError: There is no current event loop``. Assert both halves.
|
||||
"""
|
||||
where = _record_connect_threads(monkeypatch, client_module)
|
||||
monkeypatch.setattr(client_module.pb_grpc, "MxAccessGatewayStub", lambda channel: object())
|
||||
|
||||
await GatewayClient.connect(endpoint="gateway.example:5001")
|
||||
|
||||
assert ran_in_thread["off_loop"] is True
|
||||
assert where == {"resolve_off_loop": True, "create_on_loop": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_galaxy_connect_runs_create_channel_off_the_event_loop(
|
||||
async def test_galaxy_connect_splits_probe_off_loop_and_channel_on_loop(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
"""GalaxyRepositoryClient.connect must also run the probe off the loop (Client.Python-028)."""
|
||||
ran_in_thread: dict[str, bool] = {}
|
||||
|
||||
def fake_create_channel(options: ClientOptions) -> object:
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
ran_in_thread["off_loop"] = False
|
||||
except RuntimeError:
|
||||
ran_in_thread["off_loop"] = True
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr(galaxy_module, "create_channel", fake_create_channel)
|
||||
"""GalaxyRepositoryClient.connect splits the probe and the channel the same way."""
|
||||
where = _record_connect_threads(monkeypatch, galaxy_module)
|
||||
monkeypatch.setattr(
|
||||
galaxy_module.galaxy_pb_grpc, "GalaxyRepositoryStub", lambda channel: object()
|
||||
)
|
||||
|
||||
await GalaxyRepositoryClient.connect(endpoint="gateway.example:5001")
|
||||
|
||||
assert ran_in_thread["off_loop"] is True
|
||||
assert where == {"resolve_off_loop": True, "create_on_loop": True}
|
||||
|
||||
|
||||
def _stub_create_channel(options: ClientOptions, *, security: ChannelSecurity) -> object:
|
||||
return object()
|
||||
|
||||
|
||||
def _on_event_loop_thread() -> bool:
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _record_connect_threads(monkeypatch: pytest.MonkeyPatch, module: Any) -> dict[str, bool]:
|
||||
"""Patch *module*'s channel helpers to record which thread each ran on."""
|
||||
where: dict[str, bool] = {}
|
||||
|
||||
def fake_resolve(options: ClientOptions) -> ChannelSecurity:
|
||||
where["resolve_off_loop"] = not _on_event_loop_thread()
|
||||
return ChannelSecurity()
|
||||
|
||||
def fake_create_channel(options: ClientOptions, *, security: ChannelSecurity) -> object:
|
||||
where["create_on_loop"] = _on_event_loop_thread()
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr(module, "resolve_channel_security", fake_resolve)
|
||||
monkeypatch.setattr(module, "create_channel", fake_create_channel)
|
||||
return where
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -0,0 +1,288 @@
|
||||
"""Wire-level tests: the Python client against a real localhost gRPC server.
|
||||
|
||||
Every other test in this suite substitutes a fake *stub* object for
|
||||
``pb_grpc.MxAccessGatewayStub``, so nothing between the client wrapper and the
|
||||
generated stub is exercised: no HTTP/2 framing, no protobuf serialization, no
|
||||
call metadata, no gRPC status translation. That leaves a class of contract break
|
||||
— a field the gateway populates but the client never decodes, metadata the
|
||||
client believes it sends but does not, a status code it maps differently once it
|
||||
arrives as a real ``grpc.RpcError`` — invisible to the default suite.
|
||||
|
||||
These tests close that gap by serving the real ``mxaccess_gateway.v1.MxAccessGateway``
|
||||
service from an in-process ``grpc.aio`` server bound to ``127.0.0.1:0`` and
|
||||
driving the ordinary public client API against it. The bytes on the wire are the
|
||||
real ones; only the gateway's *behavior* is canned. No MXAccess, no worker, no
|
||||
network beyond loopback, so this runs everywhere the normal suite runs.
|
||||
|
||||
See ``docs/GatewayTesting.md`` (Client Wire Tests) for the shared pattern and its
|
||||
counterpart in the .NET client.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable
|
||||
|
||||
import grpc
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from zb_mom_ww_mxgateway import ClientOptions, GatewayClient
|
||||
from zb_mom_ww_mxgateway.errors import MxGatewayAuthorizationError
|
||||
from zb_mom_ww_mxgateway.events import ReplayGap
|
||||
from zb_mom_ww_mxgateway.generated import mxaccess_gateway_pb2 as pb
|
||||
from zb_mom_ww_mxgateway.generated import mxaccess_gateway_pb2_grpc as pb_grpc
|
||||
|
||||
API_KEY = "mxgw_wiretest_secret"
|
||||
SESSION_ID = "wire-session-1"
|
||||
SERVER_HANDLE = 4242
|
||||
ITEM_HANDLE = 77
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
return int(sock.getsockname()[1])
|
||||
|
||||
|
||||
def _ok() -> pb.ProtocolStatus:
|
||||
return pb.ProtocolStatus(code=pb.PROTOCOL_STATUS_CODE_OK)
|
||||
|
||||
|
||||
class FakeGateway(pb_grpc.MxAccessGatewayServicer):
|
||||
"""Canned gateway serving the four session RPCs over a real transport.
|
||||
|
||||
Replies are shaped like the gateway's own: an OK ``ProtocolStatus``, the
|
||||
echoed session id, and the typed payload the client wrapper reads (for
|
||||
example ``RegisterReply.server_handle``). Set ``deny`` to make ``Invoke``
|
||||
abort with ``PERMISSION_DENIED`` so the client's gRPC-status mapping is
|
||||
exercised against a genuine ``grpc.RpcError`` rather than a hand-built one.
|
||||
"""
|
||||
|
||||
def __init__(self, *, deny: bool = False, replay_gap: pb.ReplayGap | None = None) -> None:
|
||||
self.deny = deny
|
||||
self.replay_gap = replay_gap
|
||||
self.endpoint = ""
|
||||
self.metadata_by_method: dict[str, str] = {}
|
||||
self.open_request: pb.OpenSessionRequest | None = None
|
||||
self.invoke_request: pb.MxCommandRequest | None = None
|
||||
self.stream_request: pb.StreamEventsRequest | None = None
|
||||
self.close_request: pb.CloseSessionRequest | None = None
|
||||
|
||||
def _record(self, method: str, context: grpc.aio.ServicerContext) -> None:
|
||||
for key, value in context.invocation_metadata() or ():
|
||||
if key == "authorization":
|
||||
self.metadata_by_method[method] = value
|
||||
|
||||
async def OpenSession( # noqa: N802 - generated gRPC method name
|
||||
self, request: pb.OpenSessionRequest, context: grpc.aio.ServicerContext
|
||||
) -> pb.OpenSessionReply:
|
||||
"""Answer ``OpenSession`` with a fully populated reply."""
|
||||
self._record("OpenSession", context)
|
||||
self.open_request = request
|
||||
return pb.OpenSessionReply(
|
||||
session_id=SESSION_ID,
|
||||
backend_name="fake-backend",
|
||||
worker_process_id=1234,
|
||||
worker_protocol_version=1,
|
||||
capabilities=["events", "invoke"],
|
||||
gateway_protocol_version=3,
|
||||
protocol_status=_ok(),
|
||||
)
|
||||
|
||||
async def Invoke( # noqa: N802 - generated gRPC method name
|
||||
self, request: pb.MxCommandRequest, context: grpc.aio.ServicerContext
|
||||
) -> pb.MxCommandReply:
|
||||
"""Answer ``Invoke`` with a Register reply, or deny when configured."""
|
||||
self._record("Invoke", context)
|
||||
self.invoke_request = request
|
||||
if self.deny:
|
||||
await context.abort(grpc.StatusCode.PERMISSION_DENIED, "invoke scope required")
|
||||
return pb.MxCommandReply(
|
||||
session_id=request.session_id,
|
||||
correlation_id=request.client_correlation_id,
|
||||
kind=request.command.kind,
|
||||
protocol_status=_ok(),
|
||||
hresult=0,
|
||||
register=pb.RegisterReply(server_handle=SERVER_HANDLE),
|
||||
)
|
||||
|
||||
async def StreamEvents( # noqa: N802 - generated gRPC method name
|
||||
self, request: pb.StreamEventsRequest, context: grpc.aio.ServicerContext
|
||||
) -> AsyncIterator[pb.MxEvent]:
|
||||
"""Stream an optional replay-gap sentinel followed by one data change."""
|
||||
self._record("StreamEvents", context)
|
||||
self.stream_request = request
|
||||
if self.replay_gap is not None:
|
||||
# The sentinel shape the gateway emits: family unspecified, body
|
||||
# unset, only replay_gap populated.
|
||||
yield pb.MxEvent(session_id=request.session_id, replay_gap=self.replay_gap)
|
||||
yield pb.MxEvent(
|
||||
session_id=request.session_id,
|
||||
family=pb.MX_EVENT_FAMILY_ON_DATA_CHANGE,
|
||||
server_handle=SERVER_HANDLE,
|
||||
item_handle=ITEM_HANDLE,
|
||||
value=pb.MxValue(int32_value=17),
|
||||
quality=192,
|
||||
worker_sequence=9,
|
||||
on_data_change=pb.OnDataChangeEvent(),
|
||||
)
|
||||
|
||||
async def CloseSession( # noqa: N802 - generated gRPC method name
|
||||
self, request: pb.CloseSessionRequest, context: grpc.aio.ServicerContext
|
||||
) -> pb.CloseSessionReply:
|
||||
"""Answer ``CloseSession`` with a closed final state."""
|
||||
self._record("CloseSession", context)
|
||||
self.close_request = request
|
||||
return pb.CloseSessionReply(
|
||||
session_id=request.session_id,
|
||||
final_state=pb.SESSION_STATE_CLOSED,
|
||||
protocol_status=_ok(),
|
||||
)
|
||||
|
||||
|
||||
ServeGateway = Callable[..., Awaitable[FakeGateway]]
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def serve_gateway() -> AsyncIterator[ServeGateway]:
|
||||
"""Yield a factory that serves a :class:`FakeGateway` on loopback.
|
||||
|
||||
Each call starts its own server on a free port and records it for teardown,
|
||||
so a test can serve a differently-configured gateway without a fixture per
|
||||
variant.
|
||||
"""
|
||||
servers: list[grpc.aio.Server] = []
|
||||
|
||||
async def _start(**kwargs: object) -> FakeGateway:
|
||||
fake = FakeGateway(**kwargs) # type: ignore[arg-type]
|
||||
server = grpc.aio.server()
|
||||
pb_grpc.add_MxAccessGatewayServicer_to_server(fake, server)
|
||||
port = _free_port()
|
||||
server.add_insecure_port(f"127.0.0.1:{port}")
|
||||
await server.start()
|
||||
servers.append(server)
|
||||
fake.endpoint = f"127.0.0.1:{port}"
|
||||
return fake
|
||||
|
||||
try:
|
||||
yield _start
|
||||
finally:
|
||||
for server in servers:
|
||||
await server.stop(grace=None)
|
||||
|
||||
|
||||
async def _connect(fake: FakeGateway) -> GatewayClient:
|
||||
return await GatewayClient.connect(
|
||||
ClientOptions(
|
||||
endpoint=fake.endpoint,
|
||||
api_key=API_KEY,
|
||||
plaintext=True,
|
||||
call_timeout=10.0,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_round_trip_decodes_real_wire_bytes(serve_gateway: ServeGateway) -> None:
|
||||
"""Open, invoke, stream, and close against a real server over loopback."""
|
||||
wire_gateway = await serve_gateway()
|
||||
client = await _connect(wire_gateway)
|
||||
try:
|
||||
session = await client.open_session(client_session_name="wire-test")
|
||||
assert session.session_id == SESSION_ID
|
||||
assert session.open_reply.backend_name == "fake-backend"
|
||||
assert list(session.open_reply.capabilities) == ["events", "invoke"]
|
||||
|
||||
server_handle = await session.register("wire-test-client")
|
||||
assert server_handle == SERVER_HANDLE
|
||||
assert wire_gateway.invoke_request is not None
|
||||
assert wire_gateway.invoke_request.command.kind == pb.MX_COMMAND_KIND_REGISTER
|
||||
assert wire_gateway.invoke_request.command.register.client_name == "wire-test-client"
|
||||
|
||||
events = [event async for event in session.stream_events()]
|
||||
assert len(events) == 1
|
||||
event = events[0]
|
||||
assert not isinstance(event, ReplayGap)
|
||||
assert event.family == pb.MX_EVENT_FAMILY_ON_DATA_CHANGE
|
||||
assert event.server_handle == SERVER_HANDLE
|
||||
assert event.item_handle == ITEM_HANDLE
|
||||
assert event.value.int32_value == 17
|
||||
assert event.quality == 192
|
||||
assert event.worker_sequence == 9
|
||||
assert event.HasField("on_data_change")
|
||||
|
||||
close_reply = await session.close()
|
||||
assert close_reply.final_state == pb.SESSION_STATE_CLOSED
|
||||
assert wire_gateway.close_request is not None
|
||||
assert wire_gateway.close_request.session_id == SESSION_ID
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_key_reaches_the_server_on_every_rpc(serve_gateway: ServeGateway) -> None:
|
||||
"""The bearer header is on the wire for unary and streaming calls alike.
|
||||
|
||||
Stub-substituting tests can only assert what the client *passes*; this
|
||||
asserts what the server *receives*, which is the property that matters.
|
||||
"""
|
||||
wire_gateway = await serve_gateway()
|
||||
client = await _connect(wire_gateway)
|
||||
try:
|
||||
session = await client.open_session(client_session_name="wire-test")
|
||||
await session.register("wire-test-client")
|
||||
async for _ in session.stream_events():
|
||||
break
|
||||
await session.close()
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
expected = f"Bearer {API_KEY}"
|
||||
assert wire_gateway.metadata_by_method == {
|
||||
"OpenSession": expected,
|
||||
"Invoke": expected,
|
||||
"StreamEvents": expected,
|
||||
"CloseSession": expected,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_replay_gap_sentinel_survives_the_wire(serve_gateway: ServeGateway) -> None:
|
||||
"""A resumed stream surfaces the gateway's sentinel as a typed ``ReplayGap``."""
|
||||
replay_gap_gateway = await serve_gateway(
|
||||
replay_gap=pb.ReplayGap(requested_after_sequence=3, oldest_available_sequence=8)
|
||||
)
|
||||
client = await _connect(replay_gap_gateway)
|
||||
try:
|
||||
session = await client.open_session(client_session_name="wire-test")
|
||||
items = [item async for item in session.stream_events(after_worker_sequence=3)]
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
assert len(items) == 2
|
||||
gap = items[0]
|
||||
assert isinstance(gap, ReplayGap)
|
||||
assert gap.requested_after_sequence == 3
|
||||
assert gap.oldest_available_sequence == 8
|
||||
assert gap.resume_after_worker_sequence == 7
|
||||
assert not isinstance(items[1], ReplayGap)
|
||||
assert items[1].family == pb.MX_EVENT_FAMILY_ON_DATA_CHANGE
|
||||
|
||||
assert replay_gap_gateway.stream_request is not None
|
||||
assert replay_gap_gateway.stream_request.after_worker_sequence == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_permission_denied_maps_to_authorization_error(
|
||||
serve_gateway: ServeGateway,
|
||||
) -> None:
|
||||
"""A real ``PERMISSION_DENIED`` status becomes the typed client error."""
|
||||
denying_gateway = await serve_gateway(deny=True)
|
||||
client = await _connect(denying_gateway)
|
||||
try:
|
||||
session = await client.open_session(client_session_name="wire-test")
|
||||
with pytest.raises(MxGatewayAuthorizationError):
|
||||
await session.register("wire-test-client")
|
||||
finally:
|
||||
await client.close()
|
||||
@@ -140,6 +140,26 @@ Two viable A.2 designs given the probe data:
|
||||
poll period; modest CPU floor because the call is cheap. Matches
|
||||
the heartbeat-style WM 0xC275 semantics — AVEVA itself runs a
|
||||
poll loop internally.
|
||||
|
||||
As shipped, this is the chosen design, and the cadence is **no
|
||||
longer fixed at 500 ms**: it is the 500 ms *default* of
|
||||
`MxGateway:Alarms:PollIntervalMilliseconds` (range 100 ms – 1 h),
|
||||
which the gateway hands the worker through the
|
||||
`MXGATEWAY_ALARM_POLL_INTERVAL_MS` environment variable. The
|
||||
per-fetch cap is likewise configurable
|
||||
(`MxGateway:Alarms:MaxAlarmsPerFetch`, default 1024).
|
||||
|
||||
One snapshot rule matters when reading the capture below: a fetch
|
||||
that returns exactly the cap is treated as **truncated**, and the
|
||||
worker *merges* it into the retained snapshot instead of replacing
|
||||
it. `GetXmlCurrentAlarms2` caps its reply with no "more available"
|
||||
flag, so a capped reply is authoritative about presence only —
|
||||
alarms it had no room to mention are retained rather than allowed
|
||||
to vanish, because their disappearance is what the gateway's
|
||||
reconcile pass reads as a clear. Only a sub-cap fetch replaces the
|
||||
snapshot wholesale and can therefore clear alarms. See
|
||||
`docs/DesignDecisions.md`, "Alarms — a capped snapshot fetch never
|
||||
implies a clear".
|
||||
2. **Hook AVEVA's internal window.** Discover AVEVA's own window
|
||||
(`hwnd=0x18032E` in the probe), `SetWindowsHookEx` or
|
||||
`SetWindowSubclass` on it, and intercept WM 0xC275 on AVEVA's
|
||||
|
||||
@@ -40,27 +40,24 @@ reports the next deliverable sequence rather than `0` (see [Sessions](Sessions.m
|
||||
The default smoke sequence opens a fresh stream (no cursor) and does not exercise
|
||||
the gap path; a resume-with-gap fixture case is tracked separately (TST-24).
|
||||
|
||||
The CLIs differ in how they *print* that library-level signal. Three of them consume
|
||||
the typed gap and emit a dedicated row rather than a degenerate event row; the other
|
||||
two hand the raw sentinel `MxEvent` straight to the formatter, so they print the
|
||||
sentinel itself, whose `replayGap` field carries the same cursors:
|
||||
All five CLIs consume the typed gap and emit a dedicated row rather than a
|
||||
degenerate event row (the .NET and Java halves were the last to convert — NEXT-02):
|
||||
|
||||
| CLI | Text mode | JSON mode |
|
||||
|-----|-----------|-----------|
|
||||
| `mxgw-rs` (Rust, canonical) | `REPLAY_GAP requested_after=<n> oldest_available=<n>` | `{"replayGap": {"requestedAfterSequence": <n>, "oldestAvailableSequence": <n>}}` as one entry of the `events` array |
|
||||
| `mxgw-go` (Go) | `REPLAY_GAP requested_after=<n> oldest_available=<n>` | one `{"replayGap": {"requestedAfterSequence": <n>, "oldestAvailableSequence": <n>}}` line, counted toward `-limit` like any other row |
|
||||
| `mxgw-py` (Python) | same JSON dump as `--json` | `{"replayGap": {"requestedAfterSequence": <n>, "oldestAvailableSequence": <n>}}` as one entry of the `events` array |
|
||||
| `mxgw-dotnet` (.NET) | the raw sentinel `MxEvent` as protobuf JSON, including its `replayGap` field | same, as one entry of the `events` array |
|
||||
| `mxgw-java` (Java) | the sentinel's `worker_sequence` and `family` (`0 MX_EVENT_FAMILY_UNSPECIFIED`) | the raw sentinel `MxEvent` as protobuf JSON, including its `replayGap` field |
|
||||
| `mxgw-dotnet` (.NET) | one `{"replayGap": {...}}` line (its "text" mode is JSON-per-line) | the same row — per line with `--jsonl`, as one entry of the `events` array with `--json` |
|
||||
| `mxgw-java` (Java) | `REPLAY_GAP requested_after=<n> oldest_available=<n>` | one `{"replayGap": {"requestedAfterSequence": <n>, "oldestAvailableSequence": <n>}}` line |
|
||||
|
||||
Rust, Go, and Python emit the same two key names and, deliberately, the same JSON
|
||||
value **types**: the cursors are JSON numbers (`7`), not strings. That is why the
|
||||
Go CLI types the row by hand instead of marshalling `ReplayGap` with `protojson` —
|
||||
the proto3 JSON mapping renders 64-bit integers as strings (`"7"`), which is also
|
||||
why the .NET and Java rows, which pass the sentinel through a protobuf JSON
|
||||
formatter, carry **quoted** cursors. A matrix runner must therefore compare parsed
|
||||
values, not raw bytes, and must not assume the same value type across all five
|
||||
CLIs.
|
||||
All five emit the same two key names and, deliberately, the same JSON value
|
||||
**types**: the cursors are JSON numbers (`7`), not strings. That is why every
|
||||
CLI types the row by hand instead of marshalling `ReplayGap` through its
|
||||
protobuf JSON formatter — the proto3 JSON mapping renders 64-bit integers as
|
||||
strings (`"7"`). Normal event rows still come from the protobuf formatters, so
|
||||
a matrix runner must still compare parsed values, not raw bytes, when it mixes
|
||||
gap rows with event rows.
|
||||
|
||||
Two further formatting differences among the three canonical CLIs, none of them
|
||||
semantic: Python sorts object keys and uses `", "` / `": "` separators
|
||||
|
||||
@@ -135,6 +135,76 @@ alarm state is gateway-wide, not session-scoped — every client wants the same
|
||||
current set plus updates, and forcing each to own a worker would multiply AVEVA
|
||||
polling load for no benefit.
|
||||
|
||||
### Alarms — a capped snapshot fetch never implies a clear
|
||||
|
||||
Decision (2026-08-15): when the worker's `GetXmlCurrentAlarms2` fetch comes back
|
||||
holding exactly `MxGateway:Alarms:MaxAlarmsPerFetch` records, the worker treats
|
||||
the snapshot as **truncated** and merges it into the retained snapshot instead
|
||||
of replacing it. Alarms the capped reply did carry update normally; alarms it
|
||||
had no room to mention are retained untouched.
|
||||
|
||||
The COM API caps its reply at `maxAlmCnt` and exposes no "more available" flag,
|
||||
so a reply sitting exactly on the cap is indistinguishable from a galaxy that
|
||||
happens to hold exactly that many active alarms. Both are treated as truncated,
|
||||
because the two error directions are not symmetric.
|
||||
|
||||
Nothing in the worker emits a Clear transition. The clear is an **inference**:
|
||||
`WnWrapAlarmConsumer.ComputeTransitions` produces no transition for an alarm
|
||||
that disappears from the snapshot, and `GatewayAlarmMonitor.ApplyReconcile`
|
||||
later diffs its cache against `SnapshotActiveAlarms()` and broadcasts a Clear
|
||||
for every cached alarm the worker no longer reports. Before this decision, a
|
||||
capped fetch shrank that snapshot, so every alarm past the cap was broadcast as
|
||||
cleared while still standing — a silent, galaxy-wide false clear on exactly the
|
||||
alarm floods where the cap is reached.
|
||||
|
||||
Consequences, and how this sits with the existing failover/reconcile design:
|
||||
|
||||
- **The suppression is an eviction guard, not a transition filter.** It lives in
|
||||
the snapshot update inside `PollOnce`, not in `ComputeTransitions`, which was
|
||||
never going to emit anything for a disappearance. The reconcile/dedup
|
||||
machinery (`_clearedByReconcile` tombstones, the NEXT-03 duplicate-Clear
|
||||
suppression) is untouched: it still sees the same shape of snapshot, only
|
||||
with the truncated poll's unmentionable alarms still present.
|
||||
- **It preserves at-least-once, idempotent application.** The failure mode
|
||||
becomes bounded staleness — a genuinely cleared alarm can linger until the
|
||||
first sub-cap fetch evicts it, and the reconcile then broadcasts its Clear
|
||||
late. A late Clear is repaired by the next complete poll; a Clear that never
|
||||
happened is broadcast to every `StreamAlarms` subscriber and cannot be taken
|
||||
back. Consumers already apply transitions as "set this alarm to this state",
|
||||
so a repeated or delayed Clear is absorbed.
|
||||
- **Under *sustained* truncation, some intermediate history is lost — end state
|
||||
is not.** For an alarm that stays outside the fetch window, a full
|
||||
clear→re-raise cycle that begins and ends between two sightings emits **no
|
||||
transitions at all**: the retained record is identical before and after, so
|
||||
the diff sees nothing to report. Consumers that render current state are
|
||||
correct; consumers that *count occurrences* lose an event. Likewise, an
|
||||
operator acknowledgement of an out-of-window alarm does not reach the feed
|
||||
until that alarm re-enters a fetch window, at which point the reconcile
|
||||
repairs the acked state. This is a strictly better failure than the
|
||||
pre-guard behaviour (which fabricated a Clear for every out-of-window alarm
|
||||
on every poll), but it is not lossless, and it is another reason a
|
||||
persistently truncating deployment is a configuration defect to fix rather
|
||||
than a mode to run in.
|
||||
- **It does not synthesize anything.** Suppressing an inference is the opposite
|
||||
of inventing an event; no transition is fabricated on a truncated poll.
|
||||
- **Failover is unaffected.** `FailoverAlarmConsumer` selects which
|
||||
`IMxAccessAlarmConsumer` is live; the guard is internal to the wnwrap
|
||||
consumer's own snapshot bookkeeping and changes neither the failure counting
|
||||
that triggers failover nor the subtag standby's snapshot, which is built from
|
||||
a bounded watch-list and has no per-fetch cap to hit.
|
||||
- **Operators get told, weakly.** A truncated poll logs a rate-limited (once
|
||||
per minute) `AlarmSnapshotTruncated` warning carrying the cap, the record
|
||||
counts, and the running truncated-fetch total — identifiers and counts only,
|
||||
never tag names, values, limits, or comments. Be honest about its reach: it
|
||||
goes to the worker's console/stderr, which is captured on dev hosts but is
|
||||
not a metric, not a dashboard tile, and not part of any session-status or
|
||||
alarm-feed payload, so a production deployment can truncate indefinitely
|
||||
without anyone noticing. Surfacing truncation as a **structural** degraded
|
||||
status (a field on the alarm-provider mode/status surface the dashboard and
|
||||
`StreamAlarms` consumers already read) is filed as a follow-up; until it
|
||||
lands, the log line is the only signal. A galaxy that truncates persistently
|
||||
is a configuration problem: raise `MxGateway:Alarms:MaxAlarmsPerFetch`.
|
||||
|
||||
## Session-Resilience Epic Scope
|
||||
|
||||
Decision (2026-07-09, archreview TST-04; migrated here 2026-08-07 from the retired
|
||||
@@ -228,6 +298,56 @@ Storage recommendation:
|
||||
administrators.
|
||||
- Require TLS when the gateway is reachable off-machine.
|
||||
|
||||
## Audit Pipeline
|
||||
|
||||
Decision: audit is asynchronous, bounded, and swept.
|
||||
|
||||
The canonical `IAuditWriter` contract has always been best-effort — a failed audit write is
|
||||
logged and swallowed so it cannot abort the action that produced it. The registered writer is
|
||||
`ChannelAuditWriter`, which makes the cost of that promise explicit: a producer enqueues onto a
|
||||
4096-event bounded channel and returns, and `AuditDrainService` commits up to 64 buffered events
|
||||
per transaction. This exists because constraint denials are emitted per denied tag inside bulk
|
||||
RPC loops: a partially denied 1,000-tag request previously awaited 1,000 sequential SQLite
|
||||
inserts — each re-running `CREATE TABLE IF NOT EXISTS` — against the same database file every
|
||||
authenticated call reads. The schema bootstrap now runs once, from the drain's `StartAsync`.
|
||||
|
||||
When the channel is full the newest event is dropped and counted rather than blocking the
|
||||
producer: a stalled audit database must cost audit completeness, not gateway availability. Drops
|
||||
are logged once and reported in aggregate on each sweep. Shutdown drains what is buffered under a
|
||||
2-second cap.
|
||||
|
||||
Every other failure mode degrades to synchronous writes rather than to silent loss. The writer
|
||||
falls back to the direct path whenever nothing is draining: before the drain attaches, after it
|
||||
detaches, where no hosted service runs at all (the `apikey` admin CLI), and when the channel has
|
||||
been completed — so no attach/detach sequence can leave producers filling a buffer with no reader.
|
||||
If the drain loop itself dies it detaches the writer on the way out, which reverts every producer
|
||||
to the direct path. A batch that will not commit is retried one event at a time, so an unwritable
|
||||
row costs only itself instead of the up-to-63 good events sharing its transaction.
|
||||
|
||||
**All** audit is channelled, including admin and CRUD records — dashboard key create/revoke/rotate,
|
||||
session Close/Kill, and the library-forwarded API-key lifecycle entries. The alternative considered
|
||||
was keeping those on the synchronous writer and channelling only high-volume denial audit. It was
|
||||
rejected because a single dashboard key-create emits two records through two different seams (the
|
||||
library's `create-key` via `IApiKeyAuditStore`, and the enriching `dashboard-create-key` via
|
||||
`IAuditWriter`); splitting them across two durability regimes gives an auditor a per-producer
|
||||
matrix to reason about instead of one rule. The residual exposure is explicit: **if the gateway
|
||||
process dies between the enqueue and the batch commit, buffered audit events are lost.** The window
|
||||
is bounded by drain latency — the drain wakes on every write and commits immediately, so it is
|
||||
sub-millisecond under normal load — and it does not apply to the `apikey` CLI, which writes
|
||||
synchronously. Audit is a best-effort record of what the gateway did, not a write-ahead log of what
|
||||
it is about to do; a deployment that needs crash-durable admin audit should ship the events off-box
|
||||
rather than rely on this table.
|
||||
|
||||
`MxGateway:Security:AuditRetentionDays` (default 90, minimum 1) bounds the table: the drain sweeps
|
||||
at startup and hourly, deleting older rows. Retention cannot be configured off. The sweep compares
|
||||
through SQLite's `datetime()` rather than on the stored ISO-8601 text. Text comparison is correct
|
||||
only while every row is UTC-normalized — which the canonical model guarantees for rows written
|
||||
through the store, but not for rows that entered the table any other way — and on a mixed-format
|
||||
column it silently deletes live audit, because `2026-05-17T09:00:00-05:00` is two hours after a
|
||||
`2026-05-17T12:00:00+00:00` cutoff yet sorts before it. Comparing instants is correct however the
|
||||
text got there, and a timestamp `datetime()` cannot parse yields NULL, so undateable audit is kept
|
||||
rather than swept.
|
||||
|
||||
## Authorization
|
||||
|
||||
Decision: start with scope checks by command category.
|
||||
|
||||
+56
-37
@@ -84,42 +84,36 @@ The names match the MXAccess command list in `gateway.md` exactly. `Write` and `
|
||||
|
||||
### API key redaction
|
||||
|
||||
`RedactApiKey` is built around the `mxgw_` API key format issued by the gateway. It preserves the bearer scheme and the key id segment so that operators can correlate a log entry to a specific principal, but always strips the secret tail:
|
||||
`RedactClientIdentity` is the single redaction path for identity-bearing values; `RedactApiKey` is a
|
||||
name-preserving alias for it. Redaction **fails closed**: the only value that survives with any of its
|
||||
content is a gateway-issued `mxgw_<key-id>_<secret>` key, whose key id is kept so operators can
|
||||
correlate a log entry to a specific principal.
|
||||
|
||||
```csharp
|
||||
public static string? RedactApiKey(string? authorizationHeader)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(authorizationHeader))
|
||||
{
|
||||
return authorizationHeader;
|
||||
}
|
||||
| Input | Output | Why |
|
||||
|-------|--------|-----|
|
||||
| `Bearer mxgw_operator01_super-secret` | `Bearer mxgw_operator01_[redacted]` | Recognized gateway key; key id identifies the principal |
|
||||
| `Bearer eyJhbGciOi…` (any foreign token) | `Bearer [redacted]` | Structure is unknown, so the whole credential goes |
|
||||
| `Basic dXNlcjpwYXNz` | `Basic [redacted]` | Same, for any recognized scheme |
|
||||
| `Bearer mxgw_operator01` (no secret separator) | `Bearer mxgw_[redacted]` | No trustworthy key-id boundary |
|
||||
| `Bearer` (scheme only), `anonymous`, `some junk` | `[redacted]` | No scheme/credential split that can be trusted |
|
||||
| `null`, `""`, whitespace | unchanged | Nothing to redact |
|
||||
|
||||
const string bearerPrefix = "Bearer ";
|
||||
if (!authorizationHeader.StartsWith(bearerPrefix, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return RedactedValue;
|
||||
}
|
||||
A scheme word survives only when it is one of the recognized authorization schemes (`Bearer`,
|
||||
`Basic`, `Digest`, `Negotiate`, `NTLM`, `ApiKey`, `Token`). An unrecognized leading word is as likely
|
||||
to be credential material as it is to be a scheme, so it is dropped along with the rest. The key id is
|
||||
also dropped when it runs longer than 64 characters — a long run before the first `_` is more likely to
|
||||
be secret material than an identifier. Neither key-creation path (`ApiKeyAdminCommandLineParser.IsValidKeyId`,
|
||||
`DashboardApiKeyManagementService.ValidateKeyId`) enforces a length, so this is a redaction heuristic
|
||||
rather than a guarantee: operators should keep key ids under 64 characters, or the id stops appearing
|
||||
in logs and only the `mxgw_[redacted]` shape survives. The direction of the failure is deliberate —
|
||||
losing an identifier is cheap, logging a secret is not.
|
||||
|
||||
string token = authorizationHeader[bearerPrefix.Length..].Trim();
|
||||
The parse is span-based (no regex, no `Split` allocation): the value is split once at the first space,
|
||||
and the key id is read up to the first `_` of the remainder.
|
||||
|
||||
if (!token.StartsWith("mxgw_", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return $"{bearerPrefix}{RedactedValue}";
|
||||
}
|
||||
|
||||
string[] tokenParts = token.Split('_', 3, StringSplitOptions.RemoveEmptyEntries);
|
||||
if (tokenParts.Length < 2)
|
||||
{
|
||||
return $"{bearerPrefix}mxgw_{RedactedValue}";
|
||||
}
|
||||
|
||||
return $"{bearerPrefix}mxgw_{tokenParts[1]}_{RedactedValue}";
|
||||
}
|
||||
```
|
||||
|
||||
The split uses `count: 3` because the secret portion may itself contain underscores; only the first two segments (`mxgw` and the key id) are kept verbatim. Authorization headers that are not bearer tokens are reduced to `[redacted]` rather than passed through, since the gateway cannot reason about their structure.
|
||||
|
||||
`RedactClientIdentity` is the entry point used by `GatewayLogScope` and `DashboardRedactor`. It only invokes `RedactApiKey` when the input contains the `mxgw_` marker, leaving non-key identities (for example, Windows account names) untouched.
|
||||
The consequence for callers is that a non-key identity (for example a Windows account name) reaching
|
||||
`RedactClientIdentity` is now replaced rather than passed through. `DashboardRedactor` routes only
|
||||
values containing the `mxgw_` marker here, so dashboard display names are unaffected.
|
||||
|
||||
### Command value redaction
|
||||
|
||||
@@ -160,12 +154,12 @@ public static IApplicationBuilder UseGatewayRequestLoggingScope(this IApplicatio
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(app);
|
||||
|
||||
ILogger logger = app.ApplicationServices
|
||||
.GetRequiredService<ILoggerFactory>()
|
||||
.CreateLogger("MxGateway.Request");
|
||||
|
||||
return app.Use(async (context, next) =>
|
||||
{
|
||||
ILogger logger = context.RequestServices
|
||||
.GetRequiredService<ILoggerFactory>()
|
||||
.CreateLogger("ZB.MOM.WW.MxGateway.Request");
|
||||
|
||||
using IDisposable? scope = logger.BeginGatewayScope(new GatewayLogScope(
|
||||
SessionId: ReadHeader(context, SessionIdHeaderName),
|
||||
WorkerProcessId: ReadInt32Header(context, WorkerProcessIdHeaderName),
|
||||
@@ -190,7 +184,7 @@ The scope is keyed off four custom headers and the standard `authorization` head
|
||||
|
||||
The numeric headers use `int.TryParse` and `ulong.TryParse`; missing or unparseable values become `null` and are dropped by `GatewayLogScope.ToDictionary`. This keeps the middleware tolerant of clients that do not yet emit every header, which matters because the earliest call in a session (`OpenSession`) has no `SessionId` to send.
|
||||
|
||||
The logger category is `ZB.MOM.WW.MxGateway.Request`, which lets operators filter the request scope events independently from per-component categories.
|
||||
The logger category is `MxGateway.Request`, which lets operators filter the request scope events independently from per-component categories. The logger is resolved once at registration rather than per request: the category is fixed, so a per-request `IServiceProvider` resolve and `ILoggerFactory.CreateLogger` (which takes the factory lock) bought nothing. Scope construction itself stays unconditional — gating it on `ILogger.IsEnabled` would drop scope state for providers and scope consumers registered after startup.
|
||||
|
||||
### Pipeline ordering
|
||||
|
||||
@@ -217,8 +211,33 @@ The order matters: putting the logging scope first ensures that authentication f
|
||||
- `DashboardRedactor.Redact` delegates to `RedactClientIdentity` for any value containing the `mxgw_` marker, then falls back to a marker-keyword check for fields like `password` or `token`. This keeps dashboard renders aligned with log redaction.
|
||||
- `ZB.MOM.WW.MxGateway.Tests/Diagnostics/GatewayLogRedactorTests.cs` covers each redaction branch, including the assertion that `WriteSecured` values stay redacted even when `valueLoggingEnabled` is true.
|
||||
|
||||
## Health Checks
|
||||
|
||||
The shared `ZB.MOM.WW.Health` package maps three endpoints — `/healthz` (live), `/health/ready`, and
|
||||
`/health/active` — and each registered check opts into a tier by tag. The gateway registers two:
|
||||
|
||||
| Check | Endpoint tier | Fails when |
|
||||
|---|---|---|
|
||||
| `auth-store` | `ready` | The SQLite auth store cannot be opened. Every gRPC call authenticates against it, so its reachability genuinely gates whether the process should receive traffic. |
|
||||
| `mxaccess-sessions` | `active` | Sessions exist and their workers have faulted. Reports `total` / `ready` / `faulted` / `starting` / `closing` as entry `data`. |
|
||||
|
||||
**Zero sessions is Healthy, and the tier choice follows from that.** The gateway opens an MXAccess
|
||||
session when a client asks for one and holds none otherwise, so an idle gateway is working normally,
|
||||
not broken. A count threshold ("unhealthy below N") would sit red forever on a host nothing dials
|
||||
yet, and a permanently red probe is one operators stop reading — which leaves them worse off than no
|
||||
probe at all. `mxaccess-sessions` is therefore graded on whether the sessions that exist are usable:
|
||||
|
||||
- nothing faulted → **Healthy** (including no sessions at all)
|
||||
- some faulted, some still ready or starting → **Degraded**
|
||||
- every session faulted → **Unhealthy**
|
||||
|
||||
For the same reason it is tagged `active` rather than `ready`. Readiness decides whether the process
|
||||
should be sent traffic, and a gateway with no sessions is ready to serve; failing readiness there
|
||||
would pull a working gateway out of rotation over a condition its clients create.
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [Identifying A Deployed Build](./runbooks/IdentifyingADeployedBuild.md) — mapping a running binary back to a commit, and why the `InformationalVersion` stamp cannot be trusted on Windows builds from 2026-07-09 to 2026-08-10
|
||||
- [Sessions](./Sessions.md)
|
||||
- [gRPC](./Grpc.md)
|
||||
- [Authentication](./Authentication.md)
|
||||
|
||||
@@ -91,7 +91,7 @@ Environment variables use the normal .NET double-underscore form. For example,
|
||||
| Option | Default | Description |
|
||||
|--------|---------|-------------|
|
||||
| `MxGateway:Authentication:Mode` | `ApiKey` | Selects public gRPC authentication. Supported values are `ApiKey` and `Disabled`. `Disabled` bypasses API-key verification and is for local development only. |
|
||||
| `MxGateway:Authentication:SqlitePath` | derived from `CommonApplicationData` (`C:\ProgramData\MxGateway\gateway-auth.db` on Windows, `/usr/share/MxGateway/gateway-auth.db` or the container equivalent elsewhere) | SQLite database path for API-key records and audit rows when API-key authentication is enabled. The code default is built from `Environment.GetFolderPath(SpecialFolder.CommonApplicationData)` so the credential store never lands in the launch working directory on a non-Windows host. `appsettings.json` no longer ships an explicit value (SEC-33): the removed Windows literal was byte-identical to the Windows code default, and a Windows-absolute literal is **not** rooted on a Unix host, so it would have resolved against the CWD there. Deployed hosts still override the path through the NSSM environment (`MxGateway__Authentication__SqlitePath`). |
|
||||
| `MxGateway:Authentication:SqlitePath` | derived from `CommonApplicationData` (`C:\ProgramData\MxGateway\gateway-auth.db` on Windows, `/usr/share/MxGateway/gateway-auth.db` or the container equivalent elsewhere) | SQLite database path for API-key records and audit rows when API-key authentication is enabled. The code default is built from `Environment.GetFolderPath(SpecialFolder.CommonApplicationData)` so the credential store never lands in the launch working directory on a non-Windows host. `appsettings.json` no longer ships an explicit value (SEC-33): the removed Windows literal was byte-identical to the Windows code default, and a Windows-absolute literal is **not** rooted on a Unix host, so it would have resolved against the CWD there. Deployed hosts still override the path through the NSSM environment (`MxGateway__Authentication__SqlitePath`). The validator additionally rejects a path **inside the application content root**, even an absolute one: the upgrade procedure renames that directory to `Server.bak.*`, which takes the credential store with it and silently starts an empty one. That is not hypothetical — it happened on a production host on 2026-08-09 and no gRPC client could authenticate for two days. |
|
||||
| `MxGateway:Authentication:PepperSecretName` | `MxGateway:ApiKeyPepper` | Configuration key used to read the HMAC pepper for API-key secret hashing. The dashboard effective configuration redacts this value. |
|
||||
| `MxGateway:Authentication:RunMigrationsOnStartup` | `true` | Runs SQLite auth schema migrations at gateway startup when API-key authentication is enabled. |
|
||||
|
||||
@@ -115,6 +115,7 @@ launch CWD (SEC-01, SEC-33).
|
||||
| `MxGateway:Worker:StartupProbeRetryDelayMilliseconds` | `250` | Delay between transient startup probe retry attempts. |
|
||||
| `MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds` | `2000` | Per-attempt timeout used by the worker named-pipe connect retry path. The overall pipe connection still stays under the startup budget. |
|
||||
| `MxGateway:Worker:WriteCompletionWaitMilliseconds` | `1500` | Bounded wait the worker holds a unary write reply (`Write`/`Write2`/`WriteSecured`/`WriteSecured2`; bulk writes excluded) for the matching MXAccess `OnWriteComplete` callback, so the reply's `statuses` carry the real commit outcome. `0` disables the wait (pure fire-and-forget replies). Must be `>= 0`. The gateway conveys the value to the worker via the `MXGATEWAY_WORKER_WRITE_COMPLETION_WAIT_MS` environment variable. Consumers that time their own writes must budget above this wait: OtOpcUa's GalaxyDriver wraps gateway writes in a 2 s Tier A resilience timeout, so a deployment raising this option past ~2000 must raise that driver `ResilienceConfig` write timeout in step or slow-but-successful commits surface as consumer-side failures. |
|
||||
| `MxGateway:Worker:EventQueueCapacity` | `10000` | Capacity, in events, of the worker's outbound MXAccess event queue. Must be between `1000` and `1000000`. This is burst headroom, not a throttle: the queue has no drop policy, so filling it records a `QueueOverflow` worker fault and faults the session. Raise it for sessions whose subscription set can outrun the drain loop (large advise sets, slow event consumers); the backing queue pre-allocates its slots, so the ceiling keeps a mistyped value from committing the 32-bit worker to an outsized allocation. The gateway conveys the value to the worker via the `MXGATEWAY_EVENT_QUEUE_CAPACITY` environment variable; a missing or unusable value leaves the worker on the 10000 default rather than failing the session. |
|
||||
| `MxGateway:Worker:ShutdownTimeoutSeconds` | `10` | Grace period for worker shutdown before the gateway treats shutdown as failed and may kill the worker process tree. |
|
||||
| `MxGateway:Worker:HeartbeatIntervalSeconds` | `5` | Worker heartbeat send interval and gateway heartbeat check cadence input. |
|
||||
| `MxGateway:Worker:HeartbeatGraceSeconds` | `15` | Maximum age of the last worker heartbeat before the gateway faults the worker. This must be greater than or equal to `HeartbeatIntervalSeconds`. |
|
||||
@@ -254,6 +255,7 @@ dev/test GLAuth posture (`glauth.md`), not a production posture.
|
||||
| `MxGateway:Ldap:UserNameAttribute` | `cn` | LDAP attribute holding the login user name. |
|
||||
| `MxGateway:Ldap:DisplayNameAttribute` | `cn` | LDAP attribute holding the display name. |
|
||||
| `MxGateway:Ldap:GroupAttribute` | `memberOf` | LDAP attribute enumerating group membership (mapped to dashboard roles via `MxGateway:Dashboard:GroupToRole`). |
|
||||
| `MxGateway:Ldap:FallbackServers` | *(empty)* | Ordered backup LDAP endpoints tried when the primary fails with a system-side error (connect/TLS, service-account bind, or search) — **not** when a user's credentials are simply wrong. Each entry is `host` (adopting `Port`) or `host:port`. Empty leaves single-endpoint behaviour exactly as before. Endpoint preference is sticky: the last endpoint that answered keeps being used until it fails. The `Transport` / `AllowInsecure` policy applies to every endpoint — a fallback is not a way to downgrade TLS. Entries are parsed at startup and a malformed one fails the boot, so a typo'd backup DC cannot lie dormant until the outage it exists to survive. Requires ZB.MOM.WW.Auth 0.2.0+. |
|
||||
|
||||
When LDAP is enabled, `Server`, `SearchBase`, `ServiceAccountDn`,
|
||||
`ServiceAccountPassword`, and the attribute names must be non-blank, and `Port`
|
||||
@@ -291,7 +293,7 @@ section (a sibling of `MxGateway`, not nested under it):
|
||||
|
||||
| Option | Default | Description |
|
||||
|--------|---------|-------------|
|
||||
| `Secrets:SqlitePath` | `mxgateway-secrets.db` | Path to the encrypted secrets store, resolved relative to the app content root when not rooted. |
|
||||
| `Secrets:SqlitePath` | `<CommonApplicationData>/MxGateway/mxgateway-secrets.db` | Path to the encrypted secrets store. The default is supplied in code when the key is unset (`C:\ProgramData\MxGateway\...` on Windows), not from `appsettings.json` — a store inside the application directory is renamed away by the upgrade procedure, taking the secrets with it. On non-Windows hosts the default location is usually not writable by a normal user, so a local run must set `Secrets__SqlitePath` explicitly. |
|
||||
| `Secrets:MasterKey:Source` | `Environment` | Key-encryption-key (KEK) provider. `Environment` reads a base64-encoded 32-byte key from an env var; `Dpapi` uses a machine-bound key file instead (see below). |
|
||||
| `Secrets:MasterKey:EnvVarName` | `ZB_SECRETS_MASTER_KEY` | Env var name the `Environment` provider reads the KEK from. |
|
||||
|
||||
@@ -392,6 +394,7 @@ model requires otherwise.
|
||||
| `MxGateway:Security:ApiKeyFailureWindowSeconds` | `60` | Sliding-window length, in seconds, over which API-key verification failures are counted, for both the per-partition and the per-key-id aggregate layer. Must be greater than zero. |
|
||||
| `MxGateway:Security:ApiKeyFailureAggregateLimit` | `30` | Failed verifications for one key id counted across **all** transport peers within `ApiKeyFailureWindowSeconds` before that key id enters probe mode. This second layer bounds a distributed or source-rotating sprayer that never trips any single `(peer, key id)` partition. `0` disables the aggregate layer, leaving only per-partition counting. Must be zero or greater. |
|
||||
| `MxGateway:Security:ApiKeyFailureProbeIntervalSeconds` | `5` | Minimum interval, in seconds, between probe admissions for an over-limit partition or key-id aggregate. An over-limit state is a valve rather than a wall: one request per interval reaches the real verifier — exactly one, even when a burst arrives together at the interval boundary — so the holder of the correct secret always gets through and clears the state, while everything else is still refused before the store read. `0` blocks absolutely instead — **not recommended**, because an unauthenticated peer can then deny the key to its holder for the whole window. Must be zero or greater. |
|
||||
| `MxGateway:Security:AuditRetentionDays` | `90` | Days of canonical audit history kept in the `audit_event` table. The audit drain sweeps once at startup and hourly thereafter, deleting rows older than this window; without it the table grows without bound inside the same SQLite file the authentication hot path reads. Rows whose timestamp SQLite cannot parse are never swept. Must be greater than zero — retention can be widened but not switched off. |
|
||||
| `MxGateway:Security:ApiKeyFailureTrackedPeers` | `4096` | Maximum distinct partitions tracked by the failure counter (a bounded LRU) so a spray of unique tokens cannot grow memory without limit. It cannot be used to flush an active block either: only a validly shaped `mxgw_<keyId>_<secret>` token mints a key-id partition (everything else lands on the sender's transport-peer partition), each address may mint at most 32 key-id partitions before the overflow collapses onto that address's fallback partition, and eviction prefers fully expired windows, never removing an over-limit partition until the map exceeds twice this cap. Must be greater than zero. |
|
||||
|
||||
## Galaxy Options
|
||||
@@ -402,7 +405,7 @@ model requires otherwise.
|
||||
| `MxGateway:Galaxy:CommandTimeoutSeconds` | `60` | Per-command SQL timeout for all Galaxy browse RPCs. |
|
||||
| `MxGateway:Galaxy:DashboardRefreshIntervalSeconds` | `30` | Interval between background refreshes of the dashboard Galaxy summary cache. SQL is hit at most once per interval regardless of dashboard render rate. |
|
||||
| `MxGateway:Galaxy:PersistSnapshot` | `true` | Persists the latest successful Galaxy browse dataset to disk. When `true`, the cache reloads that snapshot at startup so clients can still browse last-known data while the Galaxy database is unreachable. The restored data is served with `Stale` status until a live query confirms it. |
|
||||
| `MxGateway:Galaxy:SnapshotCachePath` | derived from `CommonApplicationData` (`C:\ProgramData\MxGateway\galaxy-snapshot.json` on Windows, `/usr/share/MxGateway/galaxy-snapshot.json` or the container equivalent elsewhere) | File path for the persisted Galaxy browse snapshot. Ignored when `PersistSnapshot` is `false`. The snapshot is written atomically (temp file plus rename). `appsettings.json` no longer ships an explicit value (SEC-33): the option is bound by the shared `ZB.MOM.WW.GalaxyRepository` package, so the gateway supplies the `CommonApplicationData`-derived default when the bound value is blank and registers `GalaxyRepositoryOptionsValidator` to enforce that — when `PersistSnapshot` is `true` — the path is non-blank, valid, and **rooted on the host running the gateway** (`Path.IsPathRooted`, current OS). A bare filename or a foreign-platform literal fails startup instead of resolving against the launch working directory (SEC-01, SEC-33). |
|
||||
| `MxGateway:Galaxy:SnapshotCachePath` | derived from `CommonApplicationData` (`C:\ProgramData\MxGateway\galaxy-snapshot.json` on Windows, `/usr/share/MxGateway/galaxy-snapshot.json` or the container equivalent elsewhere) | File path for the persisted Galaxy browse snapshot. Ignored when `PersistSnapshot` is `false`. The snapshot is written atomically (temp file plus rename). `appsettings.json` no longer ships an explicit value (SEC-33): the option is bound by the shared `ZB.MOM.WW.GalaxyRepository` package, so the gateway supplies the `CommonApplicationData`-derived default when the bound value is blank and registers `GalaxyRepositoryOptionsValidator` to enforce that — when `PersistSnapshot` is `true` — the path is non-blank, valid, and **rooted on the host running the gateway** (`Path.IsPathRooted`, current OS). A bare filename or a foreign-platform literal fails startup instead of resolving against the launch working directory (SEC-01, SEC-33). The same validator also rejects a path **inside the application content root**, because the upgrade procedure renames that directory away and the cached snapshot would be discarded on every deploy. |
|
||||
|
||||
See [Galaxy Repository Browse](./GalaxyRepository.md) for the RPC surface and
|
||||
behavior.
|
||||
@@ -415,6 +418,8 @@ behavior.
|
||||
| `MxGateway:Alarms:SubscriptionExpression` | _(empty)_ | AVEVA alarm-subscription expression the monitor subscribes on startup, in canonical `\\<machine>\Galaxy!<area>` form. The literal `Galaxy` provider is correct regardless of the Galaxy database name. When empty and `Enabled` is `true`, the gateway falls back to `\\<MachineName>\Galaxy!<DefaultArea>` if `DefaultArea` is set. |
|
||||
| `MxGateway:Alarms:DefaultArea` | _(empty)_ | Area name used to compose a default subscription when `SubscriptionExpression` is empty. If both are empty while `Enabled` is `true`, the monitor faults with a configuration diagnostic. |
|
||||
| `MxGateway:Alarms:ReconcileIntervalSeconds` | `30` | How often the monitor reconciles its in-process alarm cache against the worker's authoritative active-alarm snapshot, catching transitions the live poll-and-diff feed missed. Floored at 5 seconds. |
|
||||
| `MxGateway:Alarms:PollIntervalMilliseconds` | `500` | Cadence at which the worker's STA polls the AVEVA alarm consumer (`GetXmlCurrentAlarms2`) for the active-alarm snapshot the live feed diffs. Must be between `100` and `3600000` (one hour): every poll is a COM call plus an XML parse on the same STA that serves reads and writes, so a tighter cadence starves the command path, while a value above an hour stops being a cadence and silently disables alarm polling. The gateway conveys the value to the worker via the `MXGATEWAY_ALARM_POLL_INTERVAL_MS` environment variable; a missing or out-of-range value leaves the worker on the 500 ms default rather than failing the session. |
|
||||
| `MxGateway:Alarms:MaxAlarmsPerFetch` | `1024` | Cap the worker passes to `GetXmlCurrentAlarms2`'s `maxAlmCnt`. Must be between `64` and `65536` — the worker is a 32-bit process that materializes each reply as one BSTR plus a full `XmlDocument`, so an unbounded cap faults the STA with an out-of-memory rather than merely slowing it. It doubles as the **truncation threshold**: a fetch returning exactly this many records is treated as truncated, because the COM API caps its reply with no "more available" flag. On a truncated poll the worker retains the alarms the capped reply could not mention instead of letting their absence read as a clear, and logs a rate-limited `AlarmSnapshotTruncated` warning to its stderr (identifiers and counts only). **Remediation when you see that warning: raise this value** so the steady-state active-alarm count fits inside one fetch. A galaxy permanently above the cap holds stale entries in the snapshot until a sub-cap poll, and loses clear→re-raise cycles that happen entirely out of window (see `docs/DesignDecisions.md`). Conveyed to the worker via the `MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH` environment variable; a missing or out-of-range value leaves the worker on the 1024 default. |
|
||||
|
||||
The alarm monitor is independent of client sessions: `AcknowledgeAlarm` and
|
||||
`StreamAlarms` are session-less RPCs served by the monitor.
|
||||
@@ -663,7 +668,7 @@ See each client README for the as-built behavior.
|
||||
|
||||
Transport security here applies only to the public gRPC channel. The
|
||||
gateway↔worker link is a per-session **named pipe**
|
||||
(`mxaccess-gateway-{gatewayPid}-{sessionId}`), not a network socket. It is not
|
||||
(`mxgw-{gatewayPid}-{sessionUid}`), not a network socket. It is not
|
||||
TLS-encrypted and does not need to be: it never leaves the local Windows host and
|
||||
is secured by the OS pipe ACL. See [Worker Frame Protocol](./WorkerFrameProtocol.md).
|
||||
|
||||
|
||||
+228
-26
@@ -98,6 +98,7 @@ ZB.MOM.WW.MxGateway.Server
|
||||
StatusBadge.razor
|
||||
FaultList.razor
|
||||
DashboardSnapshotService.cs
|
||||
DashboardSnapshotFeed.cs
|
||||
DashboardAuthorizationHandler.cs
|
||||
DashboardAuthenticator.cs
|
||||
DashboardApiKeyAuthorization.cs
|
||||
@@ -110,9 +111,15 @@ ZB.MOM.WW.MxGateway.Server
|
||||
```
|
||||
|
||||
The dashboard exposes three named SignalR hubs in addition to Blazor Server's
|
||||
internal circuit; pages connect to those hubs from within the circuit via the
|
||||
`DashboardHubConnectionFactory` helper. The hubs publish snapshot, alarm, and
|
||||
per-session event updates that the pages render in place of polling.
|
||||
internal circuit. The hubs are the **remote** surface: they publish snapshot,
|
||||
alarm, and per-session event updates to clients outside the gateway process.
|
||||
Server-rendered Blazor pages do not use them. A page runs inside this process,
|
||||
so it consumes the producing services directly through in-process seams —
|
||||
`IDashboardSnapshotFeed`, `IDashboardSessionEventSubscriber`, and
|
||||
`IGatewayAlarmService` — instead of opening a loopback WebSocket back into its
|
||||
own heap. `DashboardHubConnectionFactory`, the helper a circuit used to open
|
||||
those connections, stays registered for out-of-tree consumers, but no in-repo
|
||||
page resolves it.
|
||||
|
||||
## Dashboard Data Source
|
||||
|
||||
@@ -159,37 +166,183 @@ gateway internals.
|
||||
|
||||
## Realtime Updates
|
||||
|
||||
Updates flow over three SignalR hubs, all guarded by the
|
||||
Realtime data reaches two audiences over two seams:
|
||||
|
||||
- **in-process**, for the server-rendered Blazor pages, which run inside the
|
||||
gateway process and read the producing services directly;
|
||||
- **SignalR hubs**, for clients outside the process.
|
||||
|
||||
Pages originally took the hub path too, which put a loopback WebSocket, a
|
||||
hub-token mint, and a serialize/deserialize round trip between a Blazor component
|
||||
and an object already in its own heap. The in-process seams remove that hop. The
|
||||
hubs stay for the audience that genuinely needs a wire.
|
||||
|
||||
### In-process page feeds
|
||||
|
||||
| Page | Seam | Producer |
|
||||
|---|---|---|
|
||||
| every page deriving from `DashboardPageBase` | `IDashboardSnapshotFeed.WatchAsync` | `DashboardSnapshotFeed` (singleton) multicasting one `IDashboardSnapshotService.WatchSnapshotsAsync` enumeration |
|
||||
| `SessionDetailsPage` | `IDashboardSessionEventSubscriber.Subscribe(sessionId)` | `DashboardEventBroadcaster` — the same singleton the session mirror publishes to, registered behind both interfaces |
|
||||
| `AlarmsPage` | `IGatewayAlarmService.StreamAsync` | the central alarm monitor, **provider status only**; the alarm rows still come from the 3 s `QueryAlarmsAsync` poll |
|
||||
|
||||
The snapshot feed multicasts rather than handing each page its own enumeration:
|
||||
`WatchSnapshotsAsync` is not multicast on its own — each enumeration owns a timer
|
||||
and builds its own snapshot per tick — so a subscription per page would multiply
|
||||
the snapshot cost by the number of open pages. Each subscriber reads through a
|
||||
capacity-1 drop-oldest channel, so a circuit that renders slowly skips snapshots
|
||||
instead of buffering without bound or stalling the pump.
|
||||
|
||||
`DashboardPageBase` seeds `Snapshot` synchronously from
|
||||
`IDashboardSnapshotService.GetSnapshot()` in `OnInitializedAsync` so the first
|
||||
render is non-empty, then calls `InvokeAsync(StateHasChanged)` for every snapshot
|
||||
the feed yields. On dispose it cancels the watch and waits at most **5 seconds**
|
||||
for the loop to drain, logging a warning on timeout. The bound is deliberate: the
|
||||
loop marshals renders through the renderer's dispatcher and disposal can run on
|
||||
that same dispatcher, so an unconditional wait would hang on a wedged dispatcher.
|
||||
The accepted cost is that an abandoned loop still holds its feed subscription — the
|
||||
feed's idle gate stays open until it unwinds — and the warning is the operator's
|
||||
only signal that a circuit teardown wedged.
|
||||
|
||||
`SessionDetailsPage` subscribes for the current session id and renders the most
|
||||
recent N events (default 50) in a "Recent events" table. Its pump drains everything
|
||||
queued and renders once per batch rather than once per event, and it re-checks
|
||||
**inside the renderer dispatch** — where the subscription field is written, making
|
||||
the check an unsynchronized read of dispatcher-owned state — that the batch's
|
||||
subscription is still the live one. A batch read before a session switch would
|
||||
otherwise render the previous session's events under the new session's heading.
|
||||
Detaching cancels the pump, disposes the subscription (which releases the viewer
|
||||
registration and completes the channel, so the pump has an exit even if
|
||||
cancellation is missed), then drains under its own timeout.
|
||||
|
||||
### SignalR hubs (remote clients)
|
||||
|
||||
Updates for out-of-process clients flow over three SignalR hubs, all guarded by the
|
||||
`MxGateway.Dashboard.HubClients` policy (cookie OR `MxGateway.Dashboard.HubToken`
|
||||
bearer). Each hub class is `[Authorize(Policy = HubClientsPolicy)]`.
|
||||
|
||||
| Hub | Path | Producer | Payload | Routing |
|
||||
|---|---|---|---|---|
|
||||
| `DashboardSnapshotHub` | `/hubs/snapshot` | `DashboardSnapshotPublisher` (BackgroundService consuming `IDashboardSnapshotService.WatchSnapshotsAsync`) | `DashboardSnapshot` | Sent to all connected clients on every snapshot tick; new connections receive the current snapshot synchronously in `OnConnectedAsync`. |
|
||||
| `DashboardSnapshotHub` | `/hubs/snapshot` | `DashboardSnapshotPublisher` (BackgroundService consuming `IDashboardSnapshotService.WatchSnapshotsAsync`) | `DashboardSnapshot` | Sent to all connected clients on every snapshot tick, but only while at least one client is connected (see "Idle gating" below); new connections receive the current snapshot synchronously in `OnConnectedAsync`. |
|
||||
| `AlarmsHub` | `/hubs/alarms` | `AlarmsHubPublisher` (BackgroundService consuming `IGatewayAlarmService.StreamAsync(filter: null)`) | `AlarmFeedMessage` (`active_alarm` / `snapshot_complete` / `transition`) | Connected clients auto-join `__alarms__`; all clients receive every message. Publisher auto-reconnects every 5s on stream faults. |
|
||||
| `EventsHub` | `/hubs/events` | `DashboardEventBroadcaster` invoked by each session's internal dashboard-mirror subscriber on its `SessionEventDistributor` (registered when the session becomes Ready) | `MxEvent` | Clients call `SubscribeSession(sessionId)` to join `session:{id}`. The dashboard is a first-class distributor subscriber, so it receives the session's events whether or not a gRPC client is streaming. It sees RAW session events — not the per-gRPC-subscriber `AfterWorkerSequence` filtering that `EventStreamService` applies at its own boundary — because the dashboard is a separate LDAP-authenticated monitoring view meant to show the session's full event activity. Tag values are stripped from the mirrored `MxEvent` copy by `DashboardEventBroadcaster` when `Dashboard:ShowTagValues` is false (the default) — event metadata (tag reference, quality, status, timestamps) still renders, but the value fields are blanked, so no value leaks through this seam. The per-session hub ACL that would scope a Viewer to specific sessions is still outstanding (SEC-25 / remediation roadmap item 12); the value redaction is the near-term hardening that closes the value-leak seam independently of that ACL. |
|
||||
| `EventsHub` | `/hubs/events` | `DashboardEventBroadcaster` invoked by each session's internal dashboard-mirror subscriber on its `SessionEventDistributor` (registered when the session becomes Ready) | `MxEvent` | Clients call `SubscribeSession(sessionId)` to join `session:{id}`, which also registers them in `EventsHubViewerRegistry` — the mirror is gated on that registry, which counts hub and in-process viewers alike (see "Mirror gating" below). The dashboard is a first-class distributor subscriber, so it receives the session's events whether or not a gRPC client is streaming. It sees RAW session events — not the per-gRPC-subscriber `AfterWorkerSequence` filtering that `EventStreamService` applies at its own boundary — because the dashboard is a separate LDAP-authenticated monitoring view meant to show the session's full event activity. Tag values are stripped from the mirrored `MxEvent` copy by `DashboardEventBroadcaster` when `Dashboard:ShowTagValues` is false (the default) — event metadata (tag reference, quality, status, timestamps) still renders, but the value fields are blanked, so no value leaks through this seam. The per-session ACL that would scope a Viewer to specific sessions is still outstanding for this seam and the in-process one alike (SEC-25 / remediation roadmap item 12); the value redaction is the near-term hardening that closes the value-leak seam independently of that ACL. |
|
||||
|
||||
`DashboardPageBase` opens a `DashboardSnapshotHub` connection via the connection
|
||||
factory in `OnInitializedAsync`, seeds `Snapshot` synchronously from
|
||||
`IDashboardSnapshotService.GetSnapshot()` so the first render is non-empty, and
|
||||
calls `InvokeAsync(StateHasChanged)` on every `SnapshotUpdated` push. SignalR's
|
||||
`WithAutomaticReconnect` handles transient disconnects.
|
||||
### Default cadences
|
||||
|
||||
`SessionDetailsPage` additionally opens an `EventsHub` connection for the
|
||||
current session id and renders the most recent N events (default 50) in a
|
||||
"Recent events" table with a live/offline connection pill.
|
||||
|
||||
Default cadences:
|
||||
Both seams consume the same producing services, so they share these cadences:
|
||||
|
||||
- snapshot service produces one snapshot per
|
||||
`MxGateway:Dashboard:SnapshotIntervalMilliseconds` (default 1s);
|
||||
- alarm publisher emits on each transition observed by the central monitor;
|
||||
- event publisher emits per event fanned by the session's `SessionEventDistributor`
|
||||
to its internal dashboard-mirror subscriber (independent of any gRPC `StreamEvents`).
|
||||
to its internal dashboard-mirror subscriber (independent of any gRPC `StreamEvents`);
|
||||
- the alarms page's provider-status badge resubscribes one second after its
|
||||
`IGatewayAlarmService.StreamAsync` enumeration ends — the monitor completes a
|
||||
subscriber's stream when it falls behind and again when it restarts, both
|
||||
recoverable by resubscribing — and holds its last value in between. The page's
|
||||
alarm rows are independent of that stream and refresh on the 3 s poll.
|
||||
|
||||
### Idle gating and snapshot cost
|
||||
|
||||
A snapshot is not free: each one takes a session-registry snapshot and sorts it,
|
||||
copies the metrics dictionaries under the global metrics lock, and projects
|
||||
sessions, workers, faults, and the Galaxy summary. Without gating that work ran
|
||||
once a second for the life of the process even when nothing was watching.
|
||||
|
||||
Gating is two-tier, because the two seams have independent audiences and each must
|
||||
be able to reach zero on its own.
|
||||
|
||||
**Hub tier.** `DashboardSnapshotHub` counts live connections into the singleton
|
||||
`DashboardSnapshotHubConnectionCounter` (`OnConnectedAsync` / `OnDisconnectedAsync`,
|
||||
clamped at zero). `DashboardSnapshotPublisher` reads that count before advancing the
|
||||
snapshot enumerator: while it is zero the publisher does not call `MoveNextAsync` at
|
||||
all, so the producing iterator stays suspended at its `yield` and builds nothing —
|
||||
the gate removes the snapshot *build*, not just the broadcast. The publisher
|
||||
re-checks once a second while idle, so the first client to connect resumes the tick
|
||||
within roughly one snapshot interval, and `OnConnectedAsync` pushes the current
|
||||
snapshot to that connection immediately. Now that no in-repo page connects to the
|
||||
hub, this tier stays idle unless a remote client connects.
|
||||
|
||||
**In-process tier.** `DashboardSnapshotFeed` gates its own pump on its subscriber
|
||||
list: the first subscriber starts the pump, the last one leaving cancels it and
|
||||
awaits it, so a gateway with no page open runs no timer and builds no snapshots on
|
||||
this seam either. Successive pumps are chained through the previous pump's task, so
|
||||
an unsubscribe immediately followed by a resubscribe restarts a fresh pump without
|
||||
ever running two enumerations at once. A page does not wait for the pump's first
|
||||
tick — `DashboardPageBase` seeds its first render synchronously from
|
||||
`IDashboardSnapshotService.GetSnapshot()`.
|
||||
|
||||
That gate is generation-scoped rather than a plain subscriber count. Each pump owns
|
||||
a generation, each subscriber is tagged with the generation it joined under, a pump
|
||||
ends its generation the instant its source faults or completes — before the possibly
|
||||
slow enumerator disposal — and a dying pump only ever detaches its own generation's
|
||||
subscribers. Two races motivate the extra state:
|
||||
|
||||
- a subscriber arriving mid-teardown must start a fresh generation rather than
|
||||
attach to a pump that is about to detach everybody and leave nobody watching;
|
||||
- an unsubscribe must compare its own generation against the live one before it
|
||||
cancels anything. Subscribers of an ending generation linger in the list until
|
||||
that pump's reset runs, so counting the whole list would let them hold the idle
|
||||
gate open, and cancelling on their behalf would stop a live pump that other
|
||||
viewers depend on.
|
||||
|
||||
Two per-tick costs inside the snapshot itself are bounded independently of either gate:
|
||||
|
||||
- the effective configuration (`EffectiveGatewayConfiguration`) is built once and
|
||||
cached. It is a projection of `IOptions<GatewayOptions>`, which the gateway binds
|
||||
at startup and never reloads, so rebuilding the whole option tree every tick
|
||||
produced an identical object;
|
||||
- the API key summaries are refreshed at most once every 15 seconds
|
||||
(`ApiKeySummaryRefreshInterval`) instead of on every tick. The list is a SQLite
|
||||
read whose content changes only when an operator creates, rotates, or revokes a
|
||||
key, so a key change reaches the dashboard within that interval. Only a
|
||||
*successful* refresh restarts the interval, so a failed or timed-out read is
|
||||
retried on the next tick and the previous summaries stay on screen.
|
||||
|
||||
Avoid pushing every MXAccess data-change event into a wider broadcast group.
|
||||
The current design routes events strictly through `session:{id}` groups; the
|
||||
snapshot hub continues to carry aggregate event counters and rates.
|
||||
Events are routed strictly per session (`session:{id}` groups on the hub,
|
||||
per-session subscriber lists in process); the snapshot seams continue to carry
|
||||
aggregate event counters and rates.
|
||||
|
||||
### Mirror gating
|
||||
|
||||
Each session's dashboard-mirror subscriber calls
|
||||
`DashboardEventBroadcaster.Publish` for every event the session produces,
|
||||
independently of whether anything is watching that session. `Publish` returns
|
||||
immediately when `EventsHubViewerRegistry.HasViewers(sessionId)` is false,
|
||||
**before** the redaction clone. That matters because redaction is on by default
|
||||
(`Dashboard:ShowTagValues` false), so the unwatched steady state — nobody on any
|
||||
session-details page — previously paid a deep protobuf clone plus a send to an
|
||||
empty group for every event of every session. Behaviour for a watched session is
|
||||
unchanged.
|
||||
|
||||
The registry counts both audiences, which is what lets one gate serve both seams.
|
||||
`EventsHub` mirrors its own `AddToGroup` / `RemoveFromGroup` calls into it —
|
||||
SignalR does not expose group membership, so the broadcaster cannot ask whether
|
||||
`session:{id}` is empty — and `OnDisconnectedAsync` releases every subscription a
|
||||
dropped connection held, the only reliable signal for a browser tab that closes
|
||||
without unsubscribing. An in-process subscription registers the same way under a
|
||||
synthetic `inproc-`-prefixed connection id, which cannot collide with a SignalR
|
||||
connection id and makes the origin obvious in a debugger; disposing it removes the
|
||||
viewer and releases the synthetic connection, because that id never reconnects and
|
||||
nothing else would ever release its per-connection entry. Both paths use the same
|
||||
ordering — register before becoming a delivery target, deregister after ceasing to
|
||||
be one — so the widest a race window opens is a redaction clone that reaches
|
||||
nobody, never a dropped event that was owed to a live viewer.
|
||||
|
||||
Redaction happens once per event, not once per audience: `Publish` produces a
|
||||
single redacted clone and hands that same instance to the in-process subscribers
|
||||
and to the hub group. In-process delivery runs first and synchronously — it cannot
|
||||
throw, and it must not be skipped by the guard clause around the hub send — into
|
||||
per-subscriber bounded drop-oldest channels, so a page that falls behind loses its
|
||||
oldest queued events rather than blocking the session's event pipeline.
|
||||
|
||||
The mirror subscriber itself is still registered on the `SessionEventDistributor`
|
||||
for the session's whole lifetime; only the per-event work is gated. Starting and
|
||||
stopping the mirror lease lazily with the first and last viewer was considered
|
||||
and deliberately not done — it entangles the dashboard with distributor
|
||||
subscribe/unsubscribe lifetime (and with the replay/sequence bookkeeping that
|
||||
attaching a subscriber mid-stream implies) for no additional saving beyond the
|
||||
clone and send this gate already removes.
|
||||
|
||||
## Pages
|
||||
|
||||
@@ -312,8 +465,9 @@ panel. The panel shows each subscribed tag's live value, MXAccess data type,
|
||||
quality and source timestamp, refreshed every two seconds. The subscription
|
||||
panel is the explicit opt-in tag-value surface: it always shows values
|
||||
regardless of `Dashboard:ShowTagValues`, which governs the diagnostic
|
||||
session/worker views and the per-session `EventsHub` mirror (values are
|
||||
redacted from the mirrored events when the flag is false).
|
||||
session/worker views and the per-session event mirror — both its hub and
|
||||
in-process audiences (values are redacted from the mirrored events when the flag
|
||||
is false).
|
||||
|
||||
### Alarms page
|
||||
|
||||
@@ -323,7 +477,11 @@ defaults to showing unacknowledged `Active` alarms; filters add acknowledged
|
||||
alarms and narrow by area, severity range, and a reference/source/description
|
||||
text search. Cleared alarms are not retained — the gateway holds no
|
||||
alarm-history store, so the page reflects only the live active set. The page is
|
||||
read-only; it does not acknowledge alarms. If `MxGateway:Alarms:Enabled` is
|
||||
read-only; it does not acknowledge alarms. A provider-status badge tracks the
|
||||
central monitor's health from `IGatewayAlarmService.StreamAsync` in process — the
|
||||
alarm service is already a multi-subscriber fan-out, so the badge needs no SignalR
|
||||
client, no loopback socket, and no hub token — while the alarm rows themselves
|
||||
still come from the three-second poll. If `MxGateway:Alarms:Enabled` is
|
||||
false the central monitor never starts, and the page says so instead of showing
|
||||
an empty list with no explanation.
|
||||
|
||||
@@ -337,6 +495,31 @@ its lease expires. One session means one worker process backs every dashboard
|
||||
circuit; all access is serialised so the worker sees one in-flight command at a
|
||||
time. Tag reads go through `GatewaySession.SubscribeBulkAsync` / `ReadBulkAsync`.
|
||||
|
||||
The advise set that backs those reads is capped at 256 tags (one browse page plus
|
||||
headroom) and evicted least-recently-read-first. Without the cap every tag any
|
||||
viewer ever inspected stayed advised on the single dashboard worker until the
|
||||
session faulted, so browsing a large galaxy accreted unbounded live MXAccess
|
||||
subscriptions — and the event churn they feed — on one x86 process. Reading a tag
|
||||
already in the set marks it most-recently-read; subscribing past the cap unadvises
|
||||
the oldest entries with `GatewaySession.UnsubscribeBulkAsync` in one batch before
|
||||
the new ones are advised. Tags read in the same call are never evicted to make
|
||||
room for each other. A failed unadvise does not fail the read: the tags are
|
||||
dropped from tracking anyway (they re-subscribe if read again), because the
|
||||
session-invalidation path already handles gateway/worker drift.
|
||||
|
||||
The cap is per-read, not absolute. A read may never evict a tag it is itself about
|
||||
to return, so one read of more distinct tags than the cap leaves the set that
|
||||
large; what the eviction pass guarantees is
|
||||
|
||||
> after any read, the advise set holds at most `max(256, distinct tags in that read)`
|
||||
> tags.
|
||||
|
||||
The overshoot is not sticky: the next read that subscribes anything measures the
|
||||
overflow against the oversized set and evicts the whole excess in one pass (a
|
||||
300-tag set plus one new tag evicts 45 and lands back at 256). A read that
|
||||
subscribes nothing new evicts nothing, but neither can it grow the set. A browse
|
||||
page requests far fewer tags than the cap, so in practice the set settles at 256.
|
||||
|
||||
The Alarms page does **not** use the dashboard session: alarm data comes from
|
||||
the gateway's always-on central monitor. `QueryAlarmsAsync` reads
|
||||
`IGatewayAlarmService.CurrentAlarms` — the monitor's in-process cache — so the
|
||||
@@ -454,6 +637,14 @@ Three authorization policies are registered:
|
||||
cookie OR a `MxGateway.Dashboard.HubToken` bearer (used by WebSocket upgrades
|
||||
where the cookie can't be forwarded).
|
||||
|
||||
The in-process page feeds carry no authentication of their own, and need none:
|
||||
`MapRazorComponents<App>()` applies `RequireAuthorization(ViewerPolicy)` to the
|
||||
component endpoints, so a page can only run inside a circuit whose principal is
|
||||
already an authorized Viewer. The hub-token flow below therefore covers only the
|
||||
remote hub surface. Neither seam scopes a Viewer to particular sessions — SEC-25
|
||||
(the per-session ACL) is outstanding for both, and the mirror's value redaction
|
||||
remains the near-term mitigation, unchanged by the move in-process.
|
||||
|
||||
Two environmental bypasses still apply, both scoped to **read-only** access:
|
||||
`MxGateway:Authentication:Mode = Disabled` and `MxGateway:Dashboard:AllowAnonymousLocalhost`
|
||||
(default `true`, loopback only) each satisfy a requirement that includes the Viewer
|
||||
@@ -498,12 +689,15 @@ surface is affected. Never enable in production.
|
||||
|
||||
### Hub bearer flow
|
||||
|
||||
This flow serves remote hub clients only; in-process pages are authorized by the
|
||||
component endpoint's `ViewerPolicy` and never mint a token.
|
||||
|
||||
SignalR connections cannot reuse the `__Host-` cookie when the JS client
|
||||
upgrades to WebSocket — the cookie's `SameSite=Strict; Path=/` keeps it from
|
||||
being forwarded by the browser's WebSocket layer in some edge cases. The
|
||||
dashboard mints short-lived bearer tokens for the connection:
|
||||
|
||||
1. The cookie-authenticated Blazor page calls `GET /hubs/token`
|
||||
1. The cookie-authenticated client calls `GET /hubs/token`
|
||||
(gated by `ViewerPolicy`, cookie-only).
|
||||
2. `HubTokenService.Issue(user)` serializes the user's name, NameIdentifier,
|
||||
and role claims to JSON, encrypts with the ASP.NET Core data-protection
|
||||
@@ -523,7 +717,9 @@ dashboard mints short-lived bearer tokens for the connection:
|
||||
|
||||
`DashboardHubConnectionFactory` (scoped to the Blazor circuit) wraps the
|
||||
HubConnectionBuilder and supplies a fresh token via `AccessTokenProvider` on
|
||||
every (re)connect, so the short 5-minute lifetime is transparent to clients.
|
||||
every (re)connect, so the short 5-minute lifetime is transparent to whoever uses
|
||||
it. It remains registered, but no in-repo page opens a hub connection any more;
|
||||
external clients implement the equivalent refresh themselves.
|
||||
|
||||
Caveat — logout does not revoke outstanding tokens. Logout clears the dashboard
|
||||
cookie, but hub bearer tokens are self-contained, data-protection-encrypted, and
|
||||
@@ -618,8 +814,14 @@ Integration tests should verify:
|
||||
- a user in a Viewer-mapped LDAP group can render every page but cannot
|
||||
invoke the Admin-only management actions,
|
||||
- a user with no mapped LDAP group cannot sign in at all,
|
||||
- live snapshot updates when a fake session changes state are delivered
|
||||
via the `/hubs/snapshot` push, not by polling.
|
||||
- live snapshot updates when a fake session changes state reach a page through
|
||||
the in-process `IDashboardSnapshotFeed` and reach a remote client through the
|
||||
`/hubs/snapshot` push — neither by polling;
|
||||
- the snapshot feed's idle gate: no subscribers means no pump, the last
|
||||
subscriber of the live generation stops it, and a subscriber arriving
|
||||
mid-teardown gets a fresh generation rather than a dead one;
|
||||
- the event mirror's viewer gate counts in-process subscriptions as well as hub
|
||||
connections, and a disposed in-process subscription releases its viewer count.
|
||||
|
||||
## Initial Implementation Slice
|
||||
|
||||
|
||||
@@ -418,9 +418,14 @@ The gateway creates the pipe server before launching the worker.
|
||||
Pipe name:
|
||||
|
||||
```text
|
||||
mxaccess-gateway-{gatewayProcessId}-{sessionId}
|
||||
mxgw-{gatewayProcessId}-{sessionUid}
|
||||
```
|
||||
|
||||
`sessionUid` is the session id's guid hex without the `session-` prefix. The
|
||||
short form keeps the Unix-domain-socket path .NET uses for named pipes on
|
||||
macOS/Linux (`$TMPDIR/CoreFxPipe_{name}`) inside the 104-byte macOS `sun_path`
|
||||
limit under the default per-user `TMPDIR`.
|
||||
|
||||
Message framing:
|
||||
|
||||
```text
|
||||
@@ -588,6 +593,28 @@ Pending command handling:
|
||||
Timeouts should not assume the COM call stopped. A timed-out command may still
|
||||
finish inside the worker.
|
||||
|
||||
On timeout the client also forwards a `WorkerCancel` carrying the abandoned
|
||||
correlation id, best-effort: the gateway has stopped waiting, but the worker has
|
||||
not stopped working, and the worker owns a single STA. `WorkerPipeSession` routes
|
||||
the cancel to `CancelCommand`, which drops the correlation from the STA queue if
|
||||
it has not started and replies `Canceled` for it. A cancel that arrives after the
|
||||
command reached MXAccess is a no-op — there is no way to abort an in-flight COM
|
||||
call — so this shortens the STA backlog rather than freeing a call already
|
||||
running on it, and the rule above still holds. A command whose envelope is still
|
||||
in the gateway's outbound queue needs no special handling: the queue is FIFO, so
|
||||
the worker reads the command and then its cancel and drops it before execution.
|
||||
|
||||
Cancels ride the same outbound channel as commands, whose capacity is
|
||||
`MaxPendingCommands + 4`: the reserve above the pending-command limit is what
|
||||
absorbs them, so a burst of timeouts stays bounded and cannot deadlock the
|
||||
enqueue path. Failing to send the cancel is logged at debug and never replaces
|
||||
the `CommandTimeout` the caller is owed.
|
||||
|
||||
Cancellation outranks the deadline. When a caller's token is canceled around the
|
||||
same time the timeout fires, the command is reported as canceled
|
||||
(`GatewayShutdown`, `OperationCanceledException`), not as `CommandTimeout`, and
|
||||
no cancel is forwarded.
|
||||
|
||||
## Fault Model
|
||||
|
||||
Fault categories:
|
||||
|
||||
+190
-1
@@ -82,7 +82,17 @@ fake-worker tests cannot validate:
|
||||
when the rig does not drive sample-bearing buffered batches on demand.
|
||||
|
||||
All eight tests are gated by the same `MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`
|
||||
opt-in variable.
|
||||
opt-in variable. Opt-in does not mean unscheduled: the `nightly-windev` job runs
|
||||
this suite on windev every night at 06:00 UTC and files a Gitea issue when it goes
|
||||
red (see [Continuous Integration](#continuous-integration)), so the smoke no longer
|
||||
depends on someone remembering to set the variable.
|
||||
|
||||
Known coverage gap: the suite reaches all six late-added MXAccess **COM** commands
|
||||
but none of the five **control** commands (`Ping`, `GetSessionState`,
|
||||
`GetWorkerInfo`, `DrainEvents`, `ShutdownWorker`). Those are implemented off-STA in
|
||||
`Worker/Ipc/WorkerPipeSession.cs` and are asserted only against
|
||||
`FakeWorkerHarness`'s canned replies, so no test proves the *real* worker answers
|
||||
them. Closing that is the residual half of archreview TST-05.
|
||||
|
||||
Build the worker before running the smoke:
|
||||
|
||||
@@ -267,6 +277,49 @@ $env:MxGateway__Ldap__ServiceAccountPassword = "<service-account-password>"
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.IntegrationTests/ZB.MOM.WW.MxGateway.IntegrationTests.csproj --filter FullyQualifiedName~DashboardLdapLiveTests
|
||||
```
|
||||
|
||||
## Client Wire Tests
|
||||
|
||||
Each client's own suite drives the client's public API against a **fake gateway
|
||||
served over a real gRPC transport** — an in-process or loopback server
|
||||
implementing `mxaccess_gateway.v1.MxAccessGateway`. Only the gateway's *behaviour*
|
||||
is canned; the HTTP/2 framing, protobuf serialization, call metadata, and gRPC
|
||||
status codes are genuine. That is the difference from the per-client mocks: a mock
|
||||
substituted for the generated stub (or, in .NET, for `IMxGatewayClientTransport`)
|
||||
proves what the client *intends* to send, never what a server *receives*, so a
|
||||
field the client fails to decode or a header it never actually attaches passes
|
||||
every mock-based test. These tests need no MXAccess, no worker, and no network
|
||||
beyond loopback, so they run in the default suite on every host.
|
||||
|
||||
The shared shape each client's wire test covers:
|
||||
|
||||
- **Round trip** — `OpenSession` → `Invoke` (a `Register`, asserting the decoded
|
||||
`RegisterReply.server_handle`) → `StreamEvents` (asserting the decoded
|
||||
`OnDataChange` fields) → `CloseSession`.
|
||||
- **Auth on the wire** — the `authorization: Bearer <key>` header is asserted as
|
||||
*observed by the server*, on the streaming RPC as well as the unary ones.
|
||||
- **Replay-gap sentinel** — a stream resumed with `after_worker_sequence` opens
|
||||
with the gateway's `replay_gap` sentinel, and the client surfaces it as its
|
||||
typed, non-terminal replay-gap signal rather than a normal event.
|
||||
- **Status mapping** — a real `PERMISSION_DENIED` from the server becomes the
|
||||
client's typed authorization error, not a bare transport exception.
|
||||
|
||||
Per-client harness and command:
|
||||
|
||||
| Client | Harness | Command |
|
||||
|---|---|---|
|
||||
| .NET | `WireFakeGatewayServer` (Kestrel h2c on `127.0.0.1:0`, `MxAccessGatewayBase`) — `MxGatewayClientWireTests` | `dotnet test clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/ZB.MOM.WW.MxGateway.Client.Tests.csproj` |
|
||||
| Python | `FakeGateway` + `serve_gateway` fixture (`grpc.aio` server on `127.0.0.1:0`) — `tests/test_wire_fake_gateway.py` | `python -m pytest` from `clients/python` |
|
||||
| Go | `fakeGatewayServer` + `newBufconnClient` (`grpc.NewServer` over `bufconn`) — `mxgateway/client_session_test.go` | `go test ./...` from `clients/go` |
|
||||
| Rust | `spawn_fake_gateway` (tonic `Server` over a loopback `TcpListener`) — `tests/client_behavior.rs` | `cargo test --workspace` from `clients/rust` |
|
||||
| Java | `TestGatewayService` + `InProcessGateway` (`InProcessServerBuilder`) — `MxGatewayClientSessionTests`; plus `InProcessGatewayHarness` for the CLI tests | `gradle test` from `clients/java` |
|
||||
|
||||
All five run in CI: Go, Rust, Python, and the .NET client tests in the `portable`
|
||||
job, Java in the `java` job.
|
||||
|
||||
Adding an RPC to `mxaccess_gateway.proto` does not automatically extend these —
|
||||
the fake gateways implement only the four session RPCs. Extend the fake in the
|
||||
client whose behaviour changed rather than adding a parallel harness.
|
||||
|
||||
## Client E2E Scripts
|
||||
|
||||
`scripts/discover-testmachine-tags.ps1` queries the ZB Galaxy Repository for the
|
||||
@@ -432,6 +485,142 @@ Run the gateway test project after shared gateway test infrastructure changes:
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj
|
||||
```
|
||||
|
||||
## Running the Gateway Suite on windev
|
||||
|
||||
The gateway suite (`ZB.MOM.WW.MxGateway.Tests`, net10.0/x64) is not part of the CI
|
||||
Windows tier — `windows-x86` and `nightly-windev` run only the x86 Worker build and
|
||||
`Worker.Tests`. It is still run on windev by hand when a change needs Windows
|
||||
confirmation, and that run has three Windows-specific characteristics worth knowing
|
||||
before results are interpreted.
|
||||
|
||||
Run it from an isolated clone under `C:\build` checked out to the SHA under test — never
|
||||
the dirty Desktop checkout, and never the CI clone `C:\build\mxaccessgw-ci`, whose worktree
|
||||
lock belongs to the Worker tier.
|
||||
|
||||
Baseline on an otherwise idle windev (2026-08-10): **879 passed, 0 failed, 31 s** — the same
|
||||
879 the macOS box runs, with nothing gated away. Any failure is therefore a real signal, but
|
||||
read the load caveat below before acting on one.
|
||||
|
||||
Runs before the pipe-buffer fix below reported 855, which was long read as "windev runs a
|
||||
smaller suite because some cases are gated to Unix". It was not: 855 is simply what had been
|
||||
flushed when the wedged host was torn down. Do not treat a short count on this suite as
|
||||
platform gating.
|
||||
|
||||
### Two long-standing "windev-environmental" failures were test bugs, not the environment
|
||||
|
||||
Both were dismissed as environmental for months and are now fixed. Neither depended on
|
||||
anything installed on windev; both failed on **any** Windows host:
|
||||
|
||||
- `SelfSignedCertificateProviderTests.GenerateCertificate_HasExpectedSansEkuAndValidity`
|
||||
asserted SAN content by substring-matching `X509Extension.Format(false)`. That string is
|
||||
produced by the platform crypto library: Windows' `CryptFormatObject` renders the IPv6
|
||||
loopback fully expanded (`IP Address=0000:0000:0000:0000:0000:0000:0000:0001`) while the
|
||||
managed formatter used on macOS/Linux renders `::1`, so the loopback assertion failed on
|
||||
Windows only. The test now decodes the extension with `X509SubjectAlternativeNameExtension`
|
||||
and compares parsed `IPAddress` values and DNS names (case-insensitively, as DNS names
|
||||
are), which is platform-independent.
|
||||
- `SessionManagerTests.OpenSessionAsync_PipeNameIsShortAndUniquePerPidAndSession` guards the
|
||||
104-byte macOS `sun_path` budget that NEXT-01 shortened the pipe name to fit. It padded the
|
||||
measured name up to a five-digit pid but never substituted that worst case *downward*, so a
|
||||
six-digit pid — routine on Windows, impossible on macOS, where pids stop at 99999 — made the
|
||||
name one character "too long" against a budget that does not apply to the host running the
|
||||
test. The check now replaces the running pid's digit count with the five-digit macOS worst
|
||||
case, so it measures the name *format* rather than the current process's pid.
|
||||
|
||||
### The real-pipe suites are load-sensitive
|
||||
|
||||
These suites drive real named pipes against a five-second worker startup timeout and start
|
||||
failing when windev is busy — most often when the x86 Worker tier is building or testing at
|
||||
the same time. All five passed in the idle baseline above and all five failed in a run taken
|
||||
while an x86 build and `Worker.Tests` were in flight (that run also took 2 m 21 s against the
|
||||
idle half-minute):
|
||||
|
||||
- `GatewayEndToEndFakeWorkerSmokeTests`, `GatewayEndToEndMultiSubscriberTests`,
|
||||
`GatewayEndToEndReconnectReplayTests` — fail as
|
||||
`RpcException Status(StatusCode="Unavailable", Detail="Failed to open session …")`.
|
||||
- `SessionWorkerClientFactoryFakeWorkerTests.CreateAsync_WhenFakeWorkerStartupFails_ThrowsWorkerClientException`
|
||||
— the startup timeout beats the protocol violation the test is asserting, so the observed
|
||||
exception is `TimeoutException` instead of `WorkerClientException`.
|
||||
- `WorkerClientTests.InvokeAsync_WhenCommandExceedsFrameMax_FailsOnlyThatCommandAndStaysReady`.
|
||||
- `EventStreamServiceTests.StreamEventsAsync_WithConcurrentStreams_TracksAggregateQueueDepth`
|
||||
— polls a metric against a five-second deadline. Its helper now reports the unmet condition
|
||||
rather than letting a bare `TaskCanceledException` escape, so a load-induced timeout here
|
||||
names what it was waiting for instead of looking like an unexplained cancellation.
|
||||
|
||||
A failure in that list is evidence about machine load, not about the change under test. Check
|
||||
for a concurrent x86 build/test (`Get-Process dotnet, testhost, testhost.net48.x86,
|
||||
MSBuild, VBCSCompiler`) and re-run the affected class on its own before treating it as real.
|
||||
windev has 36 logical CPUs and `xunit.runner.json` sets `maxParallelThreads: -1`, so the
|
||||
suite runs far wider there than on the macOS dev box — that width is what turns these
|
||||
real-clock deadlines into failures.
|
||||
|
||||
### Two more findings from the 2026-08-15 windev gate
|
||||
|
||||
- `SecretsStorePathGuardTests.CreateBuilder_AcceptsSecretsStoreOutsideContentRoot_AndCreatesIt`
|
||||
used to fail deterministically on Windows: creating the builder opens `secrets.db`,
|
||||
`Microsoft.Data.Sqlite`'s connection pool kept the file handle alive past the test body, and
|
||||
the cleanup's recursive directory delete hit a sharing violation Windows enforces and Unix
|
||||
does not. The cleanup now clears the SQLite connection pool before deleting the temp
|
||||
directory (the same pattern as `TempDatabaseDirectory` and `PreHostSecretExpansionTests`),
|
||||
so the test passes on Windows and macOS alike — count it as a pass on both.
|
||||
- The `StaWaitHelper` timing tests (`WaitForSignalOrMessages_*`) flake on a loaded box with a
|
||||
signature that reads like a broken wait but is not: the helper wakes on *input being
|
||||
present*, so a message posted to the test thread ends the wait early. That is the helper
|
||||
doing exactly what the STA pump needs. The tests drain the queue with
|
||||
`PumpPendingMessages()` first for that reason; a failure here means the box was busy enough
|
||||
to queue a message mid-test, not that the wait stopped honouring its handle or its timeout.
|
||||
Re-run the class on its own before treating it as real, per the load caveat above.
|
||||
|
||||
### The full-suite testhost hang was a zero-buffer named pipe (fixed)
|
||||
|
||||
For months a full-suite run on windev reported `855 passed, 0 failed` and then never
|
||||
returned: the x64 `testhost` stopped consuming CPU but stayed alive indefinitely, and the run
|
||||
had to be killed with `--blame-hang`. That guard is no longer needed — run the suite plainly:
|
||||
|
||||
```powershell
|
||||
dotnet test src\ZB.MOM.WW.MxGateway.Tests\ZB.MOM.WW.MxGateway.Tests.csproj
|
||||
```
|
||||
|
||||
The cause is worth recording because the shape of it is easy to hit again.
|
||||
|
||||
`dotnet-stack report` on the wedged host showed no thread running test code; xUnit's
|
||||
`RunTestsInAssembly` was simply parked on `WaitHandle.WaitOne()` waiting for the
|
||||
assembly-finished event. The wait was therefore in a suspended async state machine, which only
|
||||
`dotnet-dump analyze <dump> -c dumpasync` can see. It named the exact frame:
|
||||
`WorkerClientTests.StagingChannelOverflowFaultsWorkerWithoutWaitingForFullModeTimeout`
|
||||
awaiting `WorkerFrameWriter.WriteAsync` — a 63-byte pipe write that never completed. The `855`
|
||||
was never the whole suite: the same clone now reports 879, so the wedge was also costing 24
|
||||
results, and the summary still looked clean because the hung test is not counted as a failure.
|
||||
|
||||
That test pushes events past the worker client's staging bound to prove the client faults, and
|
||||
after the fault the client's read loop stops reading by design. The test-side pipe was created
|
||||
through `NamedPipeServerStream(string, PipeDirection, int, PipeTransmissionMode, PipeOptions)`,
|
||||
whose omitted buffer arguments become `inBufferSize: 0` / `outBufferSize: 0`. On Windows that
|
||||
reserves *no* buffer: a write completes only when the peer reads it. Measured directly on
|
||||
windev, that pipe absorbed **0 bytes** before blocking against a non-reading peer, while the
|
||||
same pipe declared with 64 KiB buffers absorbed **65 520**. On macOS and Linux .NET backs named
|
||||
pipes with Unix domain sockets, whose socket buffer swallows a few kilobytes regardless — which
|
||||
is why the identical test never hung there, and why the bug read as "a windev thing".
|
||||
|
||||
Two changes make it structural rather than incidental:
|
||||
|
||||
- Test-owned server pipes are created through `TestSupport/TestNamedPipe.CreateServer`, which
|
||||
declares explicit 64 KiB buffers, in both the gateway and worker test projects. This scopes
|
||||
those tests to the backpressure they are actually asserting — the gateway's staging and event
|
||||
queues — instead of the OS pipe's flow control.
|
||||
- Every fake-worker write in `WorkerClientTests` goes through `PipePair.WriteAsync`, which
|
||||
bounds the write by the class's five-second `TestTimeout` and fails with a message naming the
|
||||
stopped reader. A blocked write is now a named test failure rather than a silent wedge.
|
||||
|
||||
The severity came from the second point being missing, not the first. A test method that never
|
||||
returns keeps xUnit from raising `ITestAssemblyFinished`, so the runner waits forever and
|
||||
`testhost` never exits — one unbounded `await` in one test costs the entire suite its result.
|
||||
Any new test that writes to a pipe whose reader may stop must bound the write.
|
||||
|
||||
The gateway's production pipe in `SessionWorkerClientFactory.CreatePipe` deliberately keeps the
|
||||
unbuffered declaration: both ends run continuous read loops and every write there is bounded by
|
||||
the worker client's `_stopCts`, so a stalled peer cancels the write instead of blocking on it.
|
||||
|
||||
## Continuous Integration
|
||||
|
||||
CI runs on Gitea Actions (`.gitea/workflows/ci.yml`; origin is Gitea at
|
||||
|
||||
+3
-3
@@ -72,7 +72,7 @@ Observable gauges are pull-based; the `Meter` invokes the supplied callback when
|
||||
|------------|--------------|-------------|
|
||||
| `mxgateway.sessions.open` | `_openSessions` | Currently open sessions tracked by `SessionManager`. |
|
||||
| `mxgateway.workers.running` | `_workersRunning` | Worker clients in a running state. |
|
||||
| `mxgateway.events.worker_queue.depth` | `_workerEventQueueDepth` | Undelivered worker events held by `WorkerClient` — staged *and* queued (GWC-24). Incremented when the read loop stages an event, decremented when the consumer reads it, so a backlog stuck in the staging channel is visible rather than invisible. |
|
||||
| `mxgateway.events.worker_queue.depth` | `_workerEventQueueDepthSources` (summed on demand) | Undelivered worker events held by `WorkerClient` — staged *and* queued (GWC-24) — summed across every live client at collection time (GWC-30). Each client owns an interlocked counter incremented when the read loop stages an event and decremented when the consumer reads it, and registers it as a gauge source for its lifetime, so a backlog stuck in a staging channel is visible and concurrent sessions add up instead of overwriting one another. |
|
||||
| `mxgateway.events.grpc_stream_queue.depth` | `_eventStreamBacklogSources` (summed on demand) | Live backlog buffered across every active `EventStreamService` subscriber, summed from the subscribers' channel `Count` at collection time. |
|
||||
|
||||
## Snapshot Shape
|
||||
@@ -111,7 +111,7 @@ The scalar fields mirror the counters and gauges. The four dictionaries provide
|
||||
- `EventsBySession` keys by `sessionId`; entries are removed via `RemoveSessionEvents` when a session closes so the map does not grow without bound.
|
||||
- `RetryAttemptsByArea` keys by the resilience `area` tag, e.g. `worker_startup`.
|
||||
|
||||
`EventsReceived` is read with `Interlocked.Read(ref _eventsReceived)` because `EventReceived` increments it via `Interlocked.Increment` outside the lock to keep the event-ingestion path non-blocking.
|
||||
`EventsReceived` is read with `Interlocked.Read(ref _eventsReceived)` because `EventReceived` increments it via `Interlocked.Increment` outside the lock to keep the event-ingestion path non-blocking. `CommandsStarted`, `CommandsSucceeded`, `CommandsFailed`, and `CommandFailuresByMethod` are read the same way: the command counters run two-to-three times per gRPC call, so they are recorded with `Interlocked` and a `ConcurrentDictionary` rather than under `_syncRoot` (GWC-30). The two queue depths are pulled from their registered sources before the lock is taken, since those delegates reach into subscriber channels and worker clients.
|
||||
|
||||
## Recording Sites
|
||||
|
||||
@@ -146,7 +146,7 @@ _metrics.RemoveSessionEvents(session.SessionId);
|
||||
- `RecordWorkerStoppedOnce` calls `WorkerStopped(reason)` exactly once per worker, guarding against double-counting on simultaneous fault and exit signals.
|
||||
- `WorkerKilled(reason)` when the client forcibly terminates the worker.
|
||||
- `HeartbeatFailed(SessionId)` per missed heartbeat.
|
||||
- `SetWorkerEventQueueDepth(queueDepth)` when the read loop stages an event and when the consumer reads one, so the gauge tracks staged + queued events.
|
||||
- `RegisterWorkerEventQueueDepthSource(...)` once at construction, disposed in `DisposeAsync`. The client's own `_eventQueueDepth` is incremented when the read loop stages an event and decremented when the consumer reads one, so the gauge tracks staged + queued events without either hot-path step calling into `GatewayMetrics`.
|
||||
- `EventReceived(SessionId, workerEvent.Event.Family.ToString())` for each worker event.
|
||||
- `QueueOverflow("worker-events")` when the timed write into the bounded consumer channel exceeds `EventChannelFullModeTimeout`, and `QueueOverflow("worker-event-staging")` when the staging channel is full at its `2 × EventChannelCapacity` bound. The two labels distinguish a stalled consumer from one that merely drains too slowly; both fault the session with `ProtocolViolation`.
|
||||
|
||||
|
||||
@@ -261,6 +261,62 @@ is still responsive. Shutdown marks the runtime as closing, wakes the pump,
|
||||
rejects new commands, cancels queued work, uninitializes COM on the STA, and
|
||||
waits for the thread to exit.
|
||||
|
||||
### Inner Completion Waits
|
||||
|
||||
Two commands hold the STA while waiting for a COM event they just provoked: the
|
||||
unary write path waits for its `OnWriteComplete`
|
||||
(`MxAccessWriteCompletionCache.TryWaitForCompletion`), and `ReadBulk` waits per
|
||||
tag for the first `OnDataChange` (`MxAccessValueCache.TryWaitForUpdate`). Both
|
||||
run the same loop shape as the outer pump, for the same reason — the event they
|
||||
are waiting for *is* a Windows message, so the thread must keep dispatching to
|
||||
receive it:
|
||||
|
||||
```text
|
||||
loop:
|
||||
pumpStep() # PeekMessage / TranslateMessage / DispatchMessage
|
||||
if cache entry newer than baseline: return it
|
||||
if now >= deadline: return the timed-out shape
|
||||
|
||||
MsgWaitForMultipleObjectsEx(
|
||||
cache_update_event,
|
||||
min(remaining, 50 ms),
|
||||
QS_ALLINPUT,
|
||||
MWMO_INPUTAVAILABLE)
|
||||
```
|
||||
|
||||
The idle slice is a Win32 wait (`StaWaitHelper.WaitForSignalOrMessages`), never
|
||||
`Thread.Sleep`. A sleeping STA pumps no messages, so a sleep-polled loop could
|
||||
only dispatch the awaited COM event at poll-tick granularity while stalling
|
||||
*every other* event for the same tick — up to 1.5 s for a write completion and
|
||||
up to `timeout_ms` per tag for `ReadBulk`. The Win32 wait returns the instant a
|
||||
message needs pumping, so the apartment dispatches continuously for the whole
|
||||
wait. Each cache also sets an `AutoResetEvent` from its update path (outside the
|
||||
cache lock) so a cross-thread producer wakes the waiter immediately; in the live
|
||||
worker the update arrives on the STA from inside `pumpStep` itself, and the
|
||||
message wake is what carries it.
|
||||
|
||||
`MWMO_INPUTAVAILABLE` makes the drain contract load-bearing: the wait wakes on
|
||||
input that is merely *present*, including input an earlier `PeekMessage` saw but
|
||||
did not remove. A `pumpStep` that drains only part of the queue — or a no-op one
|
||||
— therefore leaves a message that satisfies the wake condition forever, and the
|
||||
loop spins at 100% CPU until its deadline (deadline and reply shape still hold;
|
||||
it is a CPU fault, not a correctness one). Every `pumpStep` must drain to empty,
|
||||
as `StaRuntime.PumpPendingMessages` does.
|
||||
|
||||
The wait slice is capped at 50 ms so `pumpStep` runs periodically even when
|
||||
nothing wakes the wait — a process with no STA message queue (unit tests drive
|
||||
these caches from ordinary threads, standing in for the STA by updating the
|
||||
cache from a fake `pumpStep`) must not block for a full poll interval. Timeouts,
|
||||
deadline math, and return values are unchanged by the wait mechanism: an expired
|
||||
write wait still yields the empty-`statuses` unconfirmed reply, and an expired
|
||||
per-tag `ReadBulk` wait still reports its own timeout.
|
||||
|
||||
The write wait's budget is `MxGateway:Worker:WriteCompletionWaitMilliseconds`
|
||||
(default 1500). It is a bounded hold on the STA per unary write, so deployments
|
||||
whose write workload is effectively fire-and-forget — no consumer reads the
|
||||
reply's `statuses` — can lower it, or set `0` to skip the wait entirely and
|
||||
reply on acceptance alone.
|
||||
|
||||
## COM Creation
|
||||
|
||||
The MXAccess analysis source at `C:\Users\dohertj2\Desktop\mxaccess` identifies
|
||||
@@ -368,7 +424,11 @@ type on buffered events. `OperationComplete` is only emitted from the native
|
||||
`MxAccessEventQueue` is the bounded outbound event queue for one worker
|
||||
session. It assigns the monotonic `WorkerSequence` and `WorkerTimestamp` when an
|
||||
event is accepted, preserving the order in which MXAccess handlers enqueue
|
||||
events. The default capacity is `10000`. When the queue reaches capacity it
|
||||
events. The capacity is `10000` by default and comes from
|
||||
`MxGateway:Worker:EventQueueCapacity`, which the gateway stamps onto the worker
|
||||
launch environment as `MXGATEWAY_EVENT_QUEUE_CAPACITY`; a missing, unparseable,
|
||||
or out-of-range value (outside `1000`–`1000000`) leaves the worker on the
|
||||
default rather than failing the session. When the queue reaches capacity it
|
||||
records a `WorkerFaultCategory.QueueOverflow` fault and rejects further events.
|
||||
The event handler catches conversion and enqueue failures, records the first
|
||||
fault on the queue, and returns to the STA message pump instead of writing to
|
||||
@@ -378,16 +438,30 @@ If event conversion throws, catch it inside the event handler, record a
|
||||
structured `WorkerFault`, and keep the worker alive only if the fault policy
|
||||
allows it.
|
||||
|
||||
The event drain loop streams queued events as `WorkerEvent` frames. A single
|
||||
event whose envelope exceeds the negotiated frame maximum is **undeliverable end
|
||||
to end** — the pipe maximum sits only the envelope-overhead reserve above the
|
||||
public gRPC cap, so a frame the pipe rejects would also be rejected on the
|
||||
client-facing stream. The session therefore faults on it rather than dropping it
|
||||
(a silent drop makes the event stream unfaithful, and a synthesized placeholder
|
||||
is barred by the no-synthesized-events rule), but the death is structured: the
|
||||
worker logs the event's identity — family, handles, worker sequence, and sizes,
|
||||
never the value — writes a `WorkerFault` with category `ProtocolViolation` and
|
||||
command method `EventDrain` carrying the same identity, and only then exits.
|
||||
The event drain loop streams queued events as `WorkerEvent` frames. It is
|
||||
**signal-driven, not polled**: `MxAccessEventQueue` carries a wake signal that
|
||||
`Enqueue` and `RecordFault` release (outside the queue lock, so the STA's enqueue
|
||||
stays a lock acquire plus a non-blocking release), and a drain that comes back
|
||||
empty waits on that signal rather than sleeping. The signal is capped at one
|
||||
pending wake, so a burst coalesces into a single wake and the waiter re-drains
|
||||
everything that arrived — the loop must therefore re-check `DrainFault()` and
|
||||
re-drain after every wait, never treat a wake as "exactly one event". The 25 ms
|
||||
`EventDrainInterval` survives as the **fallback ceiling** on an unsignalled wait,
|
||||
not as a latency floor: an event arriving at an idle worker is framed at signal
|
||||
latency instead of waiting out a tick, an idle worker parks instead of waking 40
|
||||
times a second, and the interval only bounds how long the loop may sleep if some
|
||||
future path mutates the queue without signalling.
|
||||
|
||||
A single event whose envelope exceeds the negotiated frame maximum is
|
||||
**undeliverable end to end** — the pipe maximum sits only the envelope-overhead
|
||||
reserve above the public gRPC cap, so a frame the pipe rejects would also be
|
||||
rejected on the client-facing stream. The session therefore faults on it rather
|
||||
than dropping it (a silent drop makes the event stream unfaithful, and a
|
||||
synthesized placeholder is barred by the no-synthesized-events rule), but the
|
||||
death is structured: the worker logs the event's identity — family, handles,
|
||||
worker sequence, and sizes, never the value — writes a `WorkerFault` with
|
||||
category `ProtocolViolation` and command method `EventDrain` carrying the same
|
||||
identity, and only then exits.
|
||||
Operator remediation is configuration: raise `MxGateway:Worker:MaxMessageBytes`
|
||||
for that workload. Other per-frame rejection codes keep their previous behavior
|
||||
because they indicate worker bugs, not workload size.
|
||||
@@ -467,7 +541,11 @@ is bounded on **two** axes because no diagnostics command may be session-fatal:
|
||||
maximum less a 64 KiB envelope/reply-wrapper reserve, and the size decision
|
||||
happens inside the event queue's lock, so an event is dequeued only once it is
|
||||
known to fit. An event that does not fit stays at the head of the queue and is
|
||||
never lost.
|
||||
never lost. Each event's serialized size is *measured* once at enqueue, outside
|
||||
that lock, and stored beside it: the drain only compares memoized numbers, so a
|
||||
large drain never walks messages under the lock the STA needs to enqueue the
|
||||
next COM callback. The memoized size cannot go stale because an enqueued event
|
||||
is never mutated again (WRK-11).
|
||||
|
||||
Truncation is reported in the reply's existing `DiagnosticMessage`
|
||||
("N events returned, M remain; repeat DrainEvents for the rest") rather than in a
|
||||
@@ -809,6 +887,50 @@ Graceful shutdown sequence:
|
||||
If shutdown wedges, the gateway kills the process. The worker should be written
|
||||
so process kill does not corrupt other sessions.
|
||||
|
||||
### Ending the pipe read (net48)
|
||||
|
||||
Step 8 above cannot be done by cancellation. On .NET Framework 4.8
|
||||
`NamedPipeClientStream.ReadAsync` accepts a `CancellationToken` and then never
|
||||
wires it to the overlapped I/O, so a read parked waiting for gateway bytes stays
|
||||
parked no matter what the worker cancels. Closing the handle is the only thing
|
||||
that ends it.
|
||||
|
||||
`WorkerPipeSession.RunMessageLoopAsync` races one outstanding read against the
|
||||
heartbeat and event-drain loops, so every fault exit — an event-drain fault, an
|
||||
event too large to frame, a failed heartbeat write — unwinds while that read is
|
||||
still pending. The session therefore owns the transport: `RunAsync`'s outermost
|
||||
`finally` disposes the stream as its last teardown step and then awaits the read
|
||||
that disposal unblocks. Disposal comes last because in the ordinary case every
|
||||
frame the session will ever write is already complete by then — the frame writer
|
||||
signals a write only after it has been written *and* flushed. It is not last
|
||||
because that is guaranteed: the wait on the heartbeat and drain loops is
|
||||
budgeted, and a stream write is genuinely uncancellable, so an overrunning write
|
||||
can still be in flight against the stream being disposed. Disposal is
|
||||
consequently exception-*total*, catching anything the handle close throws and
|
||||
logging it, because nothing raised while releasing a handle is more actionable
|
||||
than the terminal exception that ended the session, and nothing may displace it.
|
||||
|
||||
Observation is unconditional; only the *logging* of it is budgeted.
|
||||
`ObserveBackgroundTaskStopAsync` waits `BackgroundTaskStopTimeout` for a task to
|
||||
stop and logs what it saw, but when it gives up it hands the task a
|
||||
fault-observing continuation before returning. Windows owes no deadline for a
|
||||
completion torn off a closed handle, so a bounded await on its own would reopen
|
||||
the very orphaning window it was added to close. The same helper — and so the
|
||||
same guarantee — covers the abandoned read, the heartbeat loop, and the
|
||||
event-drain loop. This matters because the worker installs no
|
||||
`TaskScheduler.UnobservedTaskException` handler: an unheld faulted task would
|
||||
otherwise surface only at finalization, still holding the reader's reused
|
||||
length-prefix buffer and its pooled payload buffer.
|
||||
|
||||
Two invariants follow. Nothing may call `WorkerFrameReader.ReadAsync` again once
|
||||
a read has been abandoned — a second read would race the first for those buffers
|
||||
and could return a pooled buffer twice — which `Debug.Assert`s at both read-issue
|
||||
sites guard. And `WorkerPipeClient`'s `using` on the pipe stays as a backstop for
|
||||
the paths the session never reaches (a session factory that throws), not as the
|
||||
primary owner; disposal is idempotent, so its second `Dispose` is a no-op.
|
||||
Graceful shutdown leaves no pending read at all, so the observation step is a
|
||||
no-op on that path.
|
||||
|
||||
`MxAccessStaSession.ShutdownGracefullyAsync` implements the current cleanup
|
||||
path. It first calls `StaCommandDispatcher.RequestShutdown()` so new commands
|
||||
are rejected and queued commands that have not started receive
|
||||
|
||||
+34
-12
@@ -14,7 +14,7 @@ All four interfaces (`ISessionManager`, `ISessionRegistry`, `ISessionWorkerClien
|
||||
|
||||
`GatewaySession` is a sealed class that holds the identity, configured timeouts, worker client reference, and current `SessionState` for one session. State is protected by a private `_syncRoot` lock so that property reads and transitions are observed atomically by concurrent gRPC calls and the lease sweeper.
|
||||
|
||||
The session id is an opaque string in the form `session-{guid:N}` and the per-session pipe name is `mxaccess-gateway-{ProcessId}-{SessionId}`. Encoding the gateway PID into the pipe name avoids collisions when an old gateway process leaks pipes that the OS has not yet reclaimed.
|
||||
The session id is an opaque string in the form `session-{guid:N}` and the per-session pipe name is `mxgw-{ProcessId}-{guid:N}` (the same guid hex, without the `session-` prefix). Encoding the gateway PID into the pipe name avoids collisions when an old gateway process leaks pipes that the OS has not yet reclaimed. The name is kept short because .NET named pipes on Unix-like hosts are Unix domain sockets at `$TMPDIR/CoreFxPipe_{name}`, and macOS caps that path at 104 bytes while its default per-user `TMPDIR` is already ~49 — the old `mxaccess-gateway-{pid}-{sessionId}` form overflowed it and broke the fake-worker/e2e tests on macOS.
|
||||
|
||||
`SessionState` itself is the protobuf-generated enum from `ZB.MOM.WW.MxGateway.Contracts.Proto`, so it is shared between the gateway and clients on the wire.
|
||||
|
||||
@@ -201,12 +201,26 @@ The single worker event channel has exactly one direct reader: the `SessionEvent
|
||||
|
||||
The monitor takes that lease **before** it issues `SubscribeAlarms`, so no transition window is missed: the pump has been running since `MarkReady` started the dashboard mirror, and the distributor only fans to subscribers registered at fan-out time, so a lease taken after the subscribe + first-reconcile round trips would lose every transition raised inside that window. Transitions arriving while the monitor subscribes and reconciles buffer in the lease's bounded channel and are applied after the reconcile, which is order-safe because a live transition can update an alarm the reconciled snapshot already holds.
|
||||
|
||||
The repair transitions the monitor's reconcile broadcasts on the alarm feed (Raise/Clear presence deltas and the acked-state delta) are **at-least-once**: a reconcile reads the worker's current state while the matching live transition may still be buffered in the lease, so both can be broadcast and the duplicates are indistinguishable — alarm-feed consumers must apply transitions idempotently. This is a property of the reconcile design, not of the buffering above.
|
||||
The repair transitions the monitor's reconcile broadcasts on the alarm feed (Raise/Clear presence deltas and the acked-state delta) are **at-least-once**: a reconcile reads the worker's current state while the matching live transition may still be buffered in the lease, so both can be broadcast and the duplicates are indistinguishable — alarm-feed consumers must apply transitions idempotently. This is a property of the reconcile design, not of the buffering above. The monitor dedups the common case best-effort (NEXT-03): a buffered live transition that positively matches the cache's worker timestamp and resulting state — or, for Clear, a tombstone keyed on the cleared instance's original raise timestamp — was already broadcast as a repair and is suppressed; unset timestamps never suppress, so the consumer contract is unchanged.
|
||||
|
||||
`AttachInternalEventSubscriber` enforces the same readiness gate as `AttachEventSubscriber` — a session (or worker) that is not `Ready` throws `SessionNotReady` *before* the distributor is constructed. A premature attach would otherwise start the pump against a source that throws, completing every subscriber with that error and latching the distributor for the session's whole lifetime.
|
||||
|
||||
Sessions open with `MxGateway:Sessions:DefaultLeaseSeconds` (default 1800) added to the open timestamp. Unary client activity refreshes the lease by the same duration. `ExtendLease` and `IsLeaseExpired` cooperate with `SessionManager.CloseExpiredLeasesAsync`, which iterates a registry snapshot and closes any session whose lease has expired with `LeaseExpiredReason`. `SessionLeaseMonitorHostedService` runs that sweep every `MxGateway:Sessions:LeaseSweepIntervalSeconds` seconds (default 30).
|
||||
|
||||
#### Teardown parallelism
|
||||
|
||||
A sweep pass is two phases. *Selection* stays a single sequential pass over the snapshot, because that is what gives the precedence rule (lease-expiry, then faulted, then detach-grace) and the `TryBeginCloseIfExpired` TOCTOU re-check their meaning. *Closing* then runs over the already-selected set with `Parallel.ForEachAsync` at `MaxParallelSessionCloses`, a compile-time constant of `4` in `SessionManager`. Each close is bounded by `MxGateway:Worker:ShutdownTimeoutSeconds` (default 10), so a one-at-a-time sweep lets a few hung workers serialize reaping and starve session slots for the rest. Parallel closing is safe because `TryBeginCloseIfExpired` already flipped each selected session to `Closing` under its own lock — that idempotent begin-close is the per-session exclusivity invariant, so no two teardowns can ever run against one session. The degree is a fixed constant rather than an option, and bounded rather than unlimited, because every concurrent close is one x86 worker process being shut down or killed; the fan-out exists to hide a few hung workers, not to tear the whole registry down at once.
|
||||
|
||||
Splitting the phases moves the TOCTOU re-check earlier, and that is an accepted trade rather than an unchanged behavior: selection now flips **every** chosen session to `Closing` up front, before any teardown runs, whereas the sequential sweep re-checked session *N* only after sessions *1..N-1* had finished closing. A client that re-attaches a subscriber while the close phase is running therefore loses a race it could previously win — the eligibility snapshot is taken at one instant for the whole pass. Expiry evaluation itself is unaffected, because `now` is a parameter and is not re-read per session.
|
||||
|
||||
A close that throws no longer abandons the rest of the selected set: the sweep attempts every selected session, captures the first failure, and rethrows it once the pass is done so `SessionLeaseMonitorHostedService` still logs the sweep failure as before. Because the closes run concurrently, *which* failure surfaces when several fail in one pass is nondeterministic; the log line is the diagnostic, not the identity of the exception.
|
||||
|
||||
`ShutdownAsync` drains sessions with the same bounded fan-out and the same per-session catch → `KillWorkerAsync` fallback, with two rules that keep a stop deadline from turning into leaked workers. First, its body is **exception-total** — nothing escapes it — because `Parallel.ForEachAsync` cancels the token handed to the sibling bodies as soon as one body throws, which would abort in-flight graceful shutdowns *and* make their kill fallback fail instantly on the freshly cancelled token. Second, the drain loop is deliberately **not** bound to the caller's `CancellationToken` and the kill fallback runs on `CancellationToken.None`: a cancelled `ParallelOptions` token stops dispatching the remaining sessions entirely, so the untried tail would be neither closed nor killed. This **fixes a leak the sequential drain also had**, rather than restoring the sequential drain's behavior — there the kill fallback ran on the caller's already-cancelled token, and `KillWorkerAsync`'s entry `ThrowIfCancellationRequested` threw out of the loop on the very first session, producing zero kills. The token is passed to the graceful close only, so a host stop deadline turns the drain into a kill sweep rather than into a leak. This matters because nothing reattaches to a leaked worker — a restarted gateway terminates orphans (see [Design Decisions](DesignDecisions.md)).
|
||||
|
||||
The asymmetry with `CloseExpiredLeasesAsync` — whose `ParallelOptions` *is* token-bound — is intentional: the sweep is periodic maintenance, so a pass abandoned on cancellation loses nothing permanently (the next pass re-selects, and `ShutdownAsync` backstops it), whereas the shutdown drain is terminal and must not be abandoned partway.
|
||||
|
||||
**Stranded-`Closing` bound.** A sweep pass that is cancelled after selection leaves its unclosed selections in `Closing` with close already started. `IsFaultedReapableCore` requires `state == Faulted`, so a session selected under `FaultedReason` and stranded this way is not re-selected as faulted; it is swept only when its normal lease expires (up to `MxGateway:Sessions:DefaultLeaseSeconds`, default 1800 s), since `IsLeaseExpiredCore` and `IsDetachGraceExpiredCore` are state-agnostic. This bound is documented rather than closed with a re-selection clause: the sweep's only caller cancels on the host's `stoppingToken`, so the very next thing that runs is `ShutdownAsync`, which drains (or kills) the whole registry — and any worker that still survives that is terminated as an orphan on the next gateway start. Adding a "`Closing` and close-started" re-selection clause would also have to distinguish an abandoned close from one that is merely still in flight, which would weaken the single invariant that makes the parallel close phase safe.
|
||||
|
||||
#### Detach-grace retention
|
||||
|
||||
`MxGateway:Sessions:DetachGraceSeconds` (default 30) is a bounded retention window kept after a session's *last external (gRPC) event-stream subscriber* drops, so a client can reconnect to the same session instead of having it torn down on the first stream disconnect. While the window is open the session stays `Ready` and fully usable — worker commands continue to work and a reconnecting subscriber re-attaches normally. Because retention is keyed on the *external* subscriber count (`_activeEventSubscriberCount`), and the gateway-owned internal dashboard mirror registers directly on the distributor with `isInternal: true` and is therefore *not* counted, a session whose only remaining subscriber is the dashboard mirror still enters detach-grace.
|
||||
@@ -276,12 +290,13 @@ If both graceful shutdown and the kill fall-back fail, the original and kill exc
|
||||
|
||||
## Shutdown Coordination
|
||||
|
||||
`SessionShutdownHostedService.StopAsync` calls `SessionManager.ShutdownAsync`, which closes every registered session with `GatewayShutdownReason`. The shutdown loop catches per-session exceptions, calls `KillWorker`, and removes the session so that one stuck worker cannot block the rest of the host:
|
||||
`SessionShutdownHostedService.StopAsync` calls `SessionManager.ShutdownAsync`, which closes every registered session with `GatewayShutdownReason`. Sessions are drained with the same bounded fan-out the lease sweep uses (`MaxParallelSessionCloses`), because a one-at-a-time drain of a full registry at a worst-case worker shutdown timeout each outruns any host stop-timeout and leaves the tail to the orphan killer. Each iteration catches its own exceptions — *every* exception, including from the fallback — calls `KillWorkerAsync` on an uncancellable token, and removes the session, so that neither one stuck worker nor one failing teardown can block or abort the rest of the host's drain:
|
||||
|
||||
```csharp
|
||||
public async Task ShutdownAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
foreach (GatewaySession session in _registry.Snapshot())
|
||||
await Parallel.ForEachAsync(
|
||||
_registry.Snapshot(),
|
||||
new ParallelOptions { MaxDegreeOfParallelism = MaxParallelSessionCloses },
|
||||
async (session, _) =>
|
||||
{
|
||||
try
|
||||
{
|
||||
@@ -293,17 +308,24 @@ public async Task ShutdownAsync(CancellationToken cancellationToken)
|
||||
exception,
|
||||
"Graceful shutdown failed for session {SessionId}; killing worker.",
|
||||
session.SessionId);
|
||||
if (_registry.TryGet(session.SessionId, out _))
|
||||
if (_registry.TryGet(session.SessionId, out GatewaySession? registeredSession)
|
||||
&& registeredSession is not null)
|
||||
{
|
||||
session.KillWorker(GatewayShutdownReason);
|
||||
await RemoveSessionAsync(session).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
// Not the caller's token: the kill is the last-resort orphan preventer.
|
||||
await KillWorkerAsync(session.SessionId, GatewayShutdownReason, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception killException)
|
||||
{
|
||||
_logger.LogWarning(killException, "Worker kill fallback failed for session {SessionId}.", session.SessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}).ConfigureAwait(false);
|
||||
```
|
||||
|
||||
Iterating over `Snapshot` rather than the live dictionary lets `RemoveSessionAsync` mutate the registry inside the loop without throwing.
|
||||
Iterating over `Snapshot` rather than the live dictionary lets `RemoveSessionAsync` mutate the registry from inside the loop without throwing, and gives the parallel drain a stable, already-materialized source.
|
||||
|
||||
## Dependency Injection
|
||||
|
||||
|
||||
@@ -37,6 +37,16 @@ $env:Path = [Environment]::GetEnvironmentVariable('Path','Machine') + ';' + [Env
|
||||
| C compiler x86 | 14.44.35207 | `C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\14.44.35207\bin\Hostx64\x86\cl.exe` |
|
||||
| Linker x86 | 14.44.35207 | `C:\Program Files (x86)\Microsoft Visual Studio\2022\BuildTools\VC\Tools\MSVC\14.44.35207\bin\Hostx64\x86\link.exe` |
|
||||
| LibMan CLI | 3.0.71 | `C:\Users\dohertj2\.dotnet\tools\libman.exe` |
|
||||
| dotnet-stack | 9.0.661903 | `C:\Users\dohertj2\.dotnet\tools\dotnet-stack.exe` |
|
||||
| dotnet-dump | 9.0.661903 | `C:\Users\dohertj2\.dotnet\tools\dotnet-dump.exe` |
|
||||
|
||||
`dotnet-stack` and `dotnet-dump` are the diagnostics pair for a process that stops
|
||||
making progress but does not exit. `dotnet-stack report -p <pid>` prints every
|
||||
managed thread's stack, which is enough when a *thread* is blocked; when nothing is
|
||||
on a thread the wait lives in a suspended async state machine, and only
|
||||
`dotnet-dump collect -p <pid>` followed by `dotnet-dump analyze <dump> -c dumpasync`
|
||||
reveals it. Both were installed user-local with `dotnet tool install -g` while
|
||||
root-causing the windev test-host hang described in `docs/GatewayTesting.md`.
|
||||
|
||||
Reference assemblies:
|
||||
|
||||
|
||||
+74
-11
@@ -88,7 +88,9 @@ priority order. A caller enqueues its frame into the control or event queue
|
||||
under a lock, then contends for a single write lock; whichever caller wins
|
||||
drains every frame queued at that moment, control frames first and each class
|
||||
in FIFO order, so a command reply, fault, heartbeat, or shutdown
|
||||
acknowledgement is never delayed behind a backlog of queued events. Priority
|
||||
acknowledgement is never delayed behind a backlog of queued events — neither
|
||||
in the bytes written nor in the flush that marks them delivered (see the
|
||||
class-boundary flush under flush coalescing below). Priority
|
||||
only reorders *which frame writes next* — it does not affect the sequence
|
||||
value a frame receives (see below), so a caller cannot infer priority class
|
||||
from the wire sequence.
|
||||
@@ -117,25 +119,62 @@ Two failure shapes are distinguished during a drain pass:
|
||||
and every frame still queued, then stops draining entirely so no caller
|
||||
waits forever on a stream that will not recover.
|
||||
|
||||
Flushes are coalesced across a drained batch: each frame in the batch is
|
||||
written to the stream without an individual flush, then one `FlushAsync`
|
||||
runs after the whole batch, and only then does every successfully-written
|
||||
frame's completion resolve — so a caller's `WriteAsync` still does not
|
||||
complete until its bytes are both written *and* flushed, but a batch that
|
||||
happened to contain several queued frames pays one flush instead of one per
|
||||
frame. The event drain loop (`WorkerPipeSession.RunEventDrainLoopAsync`)
|
||||
Flushes are coalesced across a *run of same-class frames* inside a drain
|
||||
pass: each frame in the run is written to the stream without an individual
|
||||
flush, then one `FlushAsync` runs — at the end of the pass, and additionally
|
||||
at every control-to-event boundary — and only then does every
|
||||
successfully-written frame of that run resolve its completion. A caller's
|
||||
`WriteAsync` therefore still does not complete until its bytes are both
|
||||
written *and* flushed; what changed is *when* that moment arrives
|
||||
for a control frame that a pass writes ahead of queued events. It used to be
|
||||
the end of the pass, so a heartbeat, command reply, fault, or shutdown
|
||||
acknowledgement was written first but only counted as delivered after up to a
|
||||
full event batch had been written and flushed behind it. The boundary flush
|
||||
closes the control run out before the events are written, so the priority
|
||||
class governs the frame's delivery point and not just its byte order. The
|
||||
cost stays bounded: a pure-event pass — the event hot path — still pays
|
||||
exactly one flush however many frames drain together, a run of control
|
||||
frames still pays one for the whole run (never one per heartbeat, the
|
||||
syscall-per-frame cost the coalescing removed), and only a pass that actually
|
||||
mixes both classes pays a second.
|
||||
|
||||
One consequence of the boundary flush is worth stating: a control frame whose
|
||||
run has already been flushed and completed is out of the drain's
|
||||
written-but-unflushed set, so a *later* failure in the same pass — a broken
|
||||
write, or a failed end-of-pass flush — no longer reaches back and fails it.
|
||||
That is the honest outcome: its bytes were flushed, so it was delivered. A
|
||||
failure of the boundary flush itself is treated exactly like a failed
|
||||
end-of-pass flush, and additionally fails the event frame the drain had
|
||||
already claimed off its queue (nothing else would ever complete it) along
|
||||
with every frame still queued.
|
||||
|
||||
Note the ordering all of this implies at the peer: the frames reach the pipe
|
||||
before the flush that follows them, so the gateway can read a whole batch
|
||||
while the writer has not yet flushed it. Anything observing the flush itself
|
||||
(a test counting flushes, for instance) must wait for the flush, not infer it
|
||||
from frames arriving. The event drain loop (`WorkerPipeSession.RunEventDrainLoopAsync`)
|
||||
submits a whole drained event batch through `WriteBatchAsync`, which enqueues
|
||||
every frame under one `_gate` acquisition, takes the write lock once, and
|
||||
drains them together, so a burst of N events costs one flush rather than N —
|
||||
the coalescing the batch machinery was built for now engages on the event hot
|
||||
path, not only when independent producers happen to queue behind a blocked
|
||||
write. Intra-batch order is preserved (FIFO enqueue under one lock), and a
|
||||
concurrently queued control frame is still drained ahead of the batch. A
|
||||
per-frame rejection inside a batch (for example one oversized event) surfaces
|
||||
from the batch's awaited completions as that frame's
|
||||
concurrently queued control frame is still drained — and now flushed and
|
||||
completed — ahead of the batch's remaining events, which is why a batch a
|
||||
control frame cuts into pays one extra flush while an uninterrupted batch
|
||||
still pays exactly one. A per-frame rejection inside a batch (for example one
|
||||
oversized event) surfaces from the batch's awaited completions as that frame's
|
||||
`WorkerFrameProtocolException`; the remaining completions are still observed
|
||||
so none faults unobserved.
|
||||
|
||||
The completion is the frame's delivery point, not necessarily the instant its
|
||||
caller returns. A caller that loses the race for the write lock only observes
|
||||
its own completion after the winning drainer releases the lock, so its return
|
||||
remains bounded by that drain pass even though its control frame was flushed
|
||||
and completed at the class boundary inside it. The boundary flush is what
|
||||
makes the delivery point honest; unparking a lock-race loser from the winner's
|
||||
pass would be a separate change to the enqueue-then-contend shape.
|
||||
|
||||
Cancellation of a `WriteAsync`/`WriteBatchAsync` call that is still waiting
|
||||
for the write lock when its token fires tombstones the queued frame: the
|
||||
cancelled caller marks its frame under `_gate`, and the draining lock-holder's
|
||||
@@ -147,6 +186,30 @@ so the caller observes `OperationCanceledException` while that one frame still
|
||||
reaches the wire. That residual window is by design: blocking the canceller
|
||||
behind the very write it is abandoning would defeat the point of cancellation.
|
||||
|
||||
Two hygiene notes on that residual (NEXT-04/NEXT-05). First, a frame the
|
||||
cancelled caller abandons — claimed mid-write, or already faulted by a
|
||||
concurrent queue-wide failure — completes on a task nobody awaits; the
|
||||
tombstone path attaches a fault-observing continuation to it so a later write
|
||||
failure never surfaces as a `TaskScheduler.UnobservedTaskException`. Second,
|
||||
tombstoned entries stay in the class queues until a future `DequeueNext` pops
|
||||
and skips them; that lazy purge is deliberate. Eagerly rebuilding a `Queue<T>`
|
||||
under `_gate` on every cancellation would add ordering-invariant surface next
|
||||
to the claim/cancel interlock for no real gain: any subsequent write of either
|
||||
class drains both queues to empty, and the heartbeat loop guarantees one
|
||||
arrives within a heartbeat interval, so worst-case residency is a few envelope
|
||||
references for seconds — not a leak.
|
||||
|
||||
## Pipe Buffers
|
||||
|
||||
The gateway creates each worker pipe with an explicit 128 KiB kernel buffer per
|
||||
direction (`SessionWorkerClientFactory.PipeBufferSizeBytes`) rather than the zero
|
||||
quota the short `NamedPipeServerStream` overloads request. A zero-quota byte-mode
|
||||
pipe makes every write rendezvous with a pending read, so a writer with no reader
|
||||
parked blocks until one arrives — the failure class behind the historical windev
|
||||
full-suite wedge. A real quota decouples writer latency from reader scheduling and
|
||||
lets the flush coalescing above actually pay off. On Unix hosts, where named pipes
|
||||
are Unix domain sockets, the sizes are advisory.
|
||||
|
||||
## Verification
|
||||
|
||||
The frame protocol lives in `ZB.MOM.WW.MxGateway.Worker.Ipc` (`WorkerFrameReader`,
|
||||
|
||||
@@ -0,0 +1,261 @@
|
||||
# Follow-Ups: windev Redeploy, LDAP Test Fixtures, Runner Hygiene — Implementation Plan
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers-extended-cc:subagent-driven-development
|
||||
> (opus implementers; controller verifies ops evidence; final review pass).
|
||||
|
||||
**Goal:** Close the five items surfaced by the 2026-08-07 live-actions cycle: repair windev's
|
||||
crash-looping gateway and finish the deferred SEC-36 dashboard verification (NEXT-07), fix the
|
||||
DashboardLdapLiveTests fixture drift (NEXT-06), resolve the unexpected macOS instance runner,
|
||||
harden runner-1's plaintext registration token, and verify the cargo Bearer fix.
|
||||
|
||||
**Architecture:** Three independent live streams (windev serial: 1→2→3; repo test fix: 4;
|
||||
Gitea/runner hygiene: 5, 6) run concurrently; task 7 closes out docs/trackers. No contract,
|
||||
gateway-logic, or client changes — one test-file edit (Task 4) plus live ops plus docs.
|
||||
|
||||
**Tech Stack:** SSH + PowerShell `-EncodedCommand` (windev 10.100.0.48), SSH + docker compose
|
||||
(10.100.0.35), Gitea admin API (`gitea.dohertylan.com`, token via `~/.zshenv` `GITEA_TOKEN`),
|
||||
NSSM, xUnit live-LDAP suite, GLAuth at `10.100.0.35:3893`.
|
||||
|
||||
---
|
||||
|
||||
## Preflight facts (verified before planning)
|
||||
|
||||
- Unpushed local mxaccessgw commits: `0566716`, `9760497`, `5b153da`, `41e8648` (all docs-only).
|
||||
**`origin/main` = `a346d51`** — contains all current code, so windev can build from
|
||||
`origin/main` without any push.
|
||||
- windev (`10.100.0.48`): NSSM service `MxAccessGw`; deployed Server build of 2026-06-25
|
||||
(Auth 0.1.2.0, supports auth-DB schema 2) crash-loops on
|
||||
`C:\ProgramData\MxGateway\gateway-auth.db` migrated to schema 3 on 2026-07-15
|
||||
(`AuthStoreMigrationException`, ~3.8k–10k Hosting-failed events/day). The NEW LDAP secret is
|
||||
already staged as the 10th `AppEnvironmentExtra` entry (SEC-36 Task 3) — preserve it.
|
||||
- `DashboardLdapLiveTests.cs` (`src/ZB.MOM.WW.MxGateway.IntegrationTests/`): uses
|
||||
`admin`/`admin123` (3 tests) and `readonly`/`readonly123`. Directory reality
|
||||
(`scadaproj/infra/glauth/config.toml`): `admin` exists, password is the standard dev test
|
||||
password (`password`, hash `5e884898…42d8` — same as `multi-role`), and IS in GwAdmin
|
||||
(othergroups `[5610, 5701]`); `readonly` does not exist; `gw-viewer` (primarygroup 5611 =
|
||||
GwReader, NOT GwAdmin) is the natural not-an-admin fixture. Test binds
|
||||
`MxGateway:Ldap` from `appsettings.json` (**`Server: localhost`**) + env overrides — so the
|
||||
live run needs `MxGateway__Ldap__Server=10.100.0.35` as well as
|
||||
`MxGateway__Ldap__ServiceAccountPassword` (from Mac user-secrets, never printed).
|
||||
- Gitea instance runners (`GET /api/v1/admin/actions/runners`): id 1 `gitea-runner` (cap 4),
|
||||
id 4 `macos-local-Josephs-MacBook-Pro` (**unexpected, online, labels overlap
|
||||
ubuntu-latest**), id 5 `gitea-runner-2` (cap 2).
|
||||
- `10.100.0.35:/opt/gitea/docker-compose.yml` (+ `docker-compose.yml.bak-tst30`): runner-1's
|
||||
registration token inline in plaintext env, file world-readable. runner-2 uses
|
||||
`GITEA_RUNNER_REGISTRATION_TOKEN_FILE: /run/secrets/runner_token` ← 0600
|
||||
`/opt/gitea/runner_token`. runner-1 data volume `/opt/gitea/runner:/data` (its `.runner`
|
||||
credential persists — the registration env is only needed for first registration).
|
||||
- Cargo Bearer fix already applied (`~/.zshenv`, backup `~/.zshenv.bak-cli39`) and documented
|
||||
(`docs/ClientPackaging.md`, commit `5b153da`). Task 7 verifies; no further action expected.
|
||||
|
||||
## Secret hygiene (binding, all tasks)
|
||||
|
||||
- Never print the LDAP service-account password, `GITEA_TOKEN`, cargo token, runner
|
||||
registration tokens, or API keys — not in commands, logs, commits, or reports. Read the LDAP
|
||||
password from `dotnet user-secrets list` into an env var without echoing
|
||||
(e.g. `export MxGateway__Ldap__ServiceAccountPassword="$(dotnet user-secrets list --project src/ZB.MOM.WW.MxGateway.Server | awk -F' = ' '/ServiceAccountPassword/ {print $2}')"`).
|
||||
- Documented dev **test users** (`multi-role`/`password`, `admin`/`password`,
|
||||
`gw-viewer`/`password`) are NOT secrets — glauth.md publishes them; fine in code/commits.
|
||||
- SSH→windev PowerShell: always `powershell -NoProfile -EncodedCommand <base64-UTF16LE>`;
|
||||
never put secrets inside EncodedCommand blobs or argv.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: NEXT-07 — Recon windev deployment layout + schema support
|
||||
|
||||
**Classification:** standard — read-only recon, but its output gates a service redeploy
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Task 4, Task 5, Task 6
|
||||
|
||||
**Files:** none edited. SSH recon on `10.100.0.48` + repo/scadaproj reads on the Mac.
|
||||
|
||||
Determine everything Task 2 needs, and confirm the fresh-deploy path is safe:
|
||||
|
||||
1. `nssm get MxAccessGw Application`, `AppDirectory`, `AppParameters`,
|
||||
`AppEnvironmentExtra` (count entries; do NOT print values of secret-bearing entries —
|
||||
names only).
|
||||
2. Inventory the deployed dir: path, `ZB.MOM.WW.MxGateway.Server.exe` timestamp, whether
|
||||
`appsettings.json`/`appsettings.Production.json` in the deploy dir differ from repo
|
||||
`origin/main` (diff; windev-specific config must survive the redeploy).
|
||||
3. Confirm build feasibility on windev: `dotnet --list-sdks` (need 10.x), locate an existing
|
||||
mxaccessgw checkout/worktree (CI uses `scripts/ci/windev-worker-ci.ps1` — find its
|
||||
worktree path) or pick a fresh clone location. Confirm `git fetch` reaches `origin/main`
|
||||
= `a346d514dd24e775640e5667aa7cd8e561fec68a`.
|
||||
4. Confirm current code supports auth-DB schema 3: find the auth-store supported-schema
|
||||
constant (ZB.MOM.WW.Auth packages — check the package version the Server at `origin/main`
|
||||
references, and/or the migration code in the shared scadaproj libs) and state the
|
||||
evidence. **If current code does NOT support schema 3, STOP — report, do not deploy.**
|
||||
5. Gateway endpoints for verification: bound URLs/ports (from deployed config/env), dashboard
|
||||
scheme (http vs https → cookie will be `MxGatewayDashboard` vs `__Host-…`).
|
||||
6. Check what migrated the DB to schema 3 on 2026-07-15 (event log / file timestamps) — only
|
||||
to confirm schema 3 is the shared-lib current version, not an anomaly.
|
||||
|
||||
**Step: report** all findings as structured text (no secrets); no changes, no commits.
|
||||
|
||||
### Task 2: NEXT-07 — Build current Server on windev and redeploy the service
|
||||
|
||||
**Classification:** high-risk — replaces a running (crash-looping) service's binaries
|
||||
**Estimated implement time:** ~10 min
|
||||
**Parallelizable with:** none (needs Task 1)
|
||||
|
||||
**Files:** none in repo. windev filesystem + NSSM only.
|
||||
|
||||
Using Task 1's facts:
|
||||
|
||||
1. On windev, fetch/checkout `origin/main` (`a346d51…`) in the build worktree/clone.
|
||||
2. `dotnet publish src/ZB.MOM.WW.MxGateway.Server -c Release` (match deployed layout/RID from
|
||||
Task 1; framework-dependent vs self-contained must match what NSSM `Application` points at).
|
||||
3. Stop the service (`nssm stop MxAccessGw`), confirm process exited.
|
||||
4. Backup: deployed dir → sibling `*.bak-next07` copy; copy
|
||||
`C:\ProgramData\MxGateway\gateway-auth.db` (+ `-wal`/`-shm` if present) to
|
||||
`gateway-auth.db.bak-next07`. **Never delete the live DB.**
|
||||
5. Deploy publish output over the deploy dir, then restore any windev-specific config files
|
||||
identified in Task 1 (do not clobber live overrides; NSSM env entries are untouched by
|
||||
file copies but verify count unchanged after start).
|
||||
6. `nssm start MxAccessGw`; verify: service state RUNNING and stable ≥60 s (no restart
|
||||
cycle), Application event log shows clean host start and **zero new
|
||||
`AuthStoreMigrationException` / `Hosting failed to start`** after the start timestamp,
|
||||
bound port answers (e.g. dashboard root or health endpoint returns HTTP).
|
||||
7. Rollback if unhealthy: stop, restore `*.bak-next07` dir, start, report.
|
||||
|
||||
**Step: report** deployed SHA, verification evidence, backup paths. No repo commits.
|
||||
|
||||
### Task 3: SEC-36 deferred verification + NEXT-07/runbook closeout
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~6 min
|
||||
**Parallelizable with:** none (needs Task 2)
|
||||
|
||||
**Files:**
|
||||
- Modify: `docs/runbooks/SEC-36-ldap-credential-rotation.md` (Correction 3 — mark the
|
||||
deferred dashboard check done, dated)
|
||||
- Modify: `archreview/2026-07-12/remediation/90-candidate-findings-next-cycle.md` (NEXT-07 →
|
||||
resolved 2026-08-07, evidence one-liner)
|
||||
|
||||
1. Complete SEC-36 step 4 against the repaired windev gateway: log in to the dashboard as
|
||||
`multi-role`/`password` end-to-end. Preferred: `curl` flow — GET `/login` (capture
|
||||
antiforgery token + cookie), POST credentials, expect success redirect + auth cookie
|
||||
(name per Task 1 scheme). If the login page resists scripting (Blazor circuit), report
|
||||
exactly why and fall back to asserting a fresh `DashboardLdapLiveTests` green run
|
||||
(Task 4) plus windev log evidence of successful LDAP bind on a manual attempt.
|
||||
2. Update the two docs; commit locally (`docs(sec-36,next-07): …`), do NOT push.
|
||||
|
||||
### Task 4: NEXT-06 — Fix DashboardLdapLiveTests fixtures to match the shared directory
|
||||
|
||||
**Classification:** small — one test file, but must go green against live GLAuth
|
||||
**Estimated implement time:** ~6 min
|
||||
**Parallelizable with:** Task 1, Task 5, Task 6
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.IntegrationTests/DashboardLdapLiveTests.cs`
|
||||
- Modify: `archreview/2026-07-12/remediation/90-candidate-findings-next-cycle.md` (NEXT-06 →
|
||||
resolved)
|
||||
- Possibly modify: `docs/GatewayTesting.md` (live-LDAP opt-in row: document the
|
||||
`MxGateway__Ldap__Server` override needed when GLAuth is not localhost)
|
||||
|
||||
1. Read `DashboardAuthenticator` first: confirm a user who binds successfully but maps to no
|
||||
role yields `Succeeded == false` (drives the gw-viewer fixture).
|
||||
2. Fix fixtures: `admin`/`admin123` → `admin`/`password` (positive + wrong-password +
|
||||
unreachable tests); `readonly`/`readonly123` → `gw-viewer`/`password` (exercises
|
||||
user-binds-but-lacks-GwAdmin; keep the no-password-leak assertion, updating the asserted
|
||||
literal). Update XML doc comments to match. Keep MXAccess-repo style rules
|
||||
(TreatWarningsAsErrors).
|
||||
3. Build: `dotnet build src/ZB.MOM.WW.MxGateway.IntegrationTests` (macOS OK — net10.0).
|
||||
4. Live run (env only, never echo the password):
|
||||
`MXGATEWAY_RUN_LIVE_LDAP_TESTS=1 MxGateway__Ldap__Server=10.100.0.35 MxGateway__Ldap__ServiceAccountPassword=<from user-secrets> dotnet test … --filter FullyQualifiedName~DashboardLdapLiveTests`
|
||||
→ expect **5/5 passed** (this is also positive live proof of the SEC-36 service-account
|
||||
bind).
|
||||
5. Update tracker row (+ GatewayTesting.md if the Server-override note is missing); commit
|
||||
locally (`test(ldap): …`), do NOT push.
|
||||
|
||||
### Task 5: Resolve the unexpected macOS instance runner (id 4)
|
||||
|
||||
**Classification:** standard — evidence-gated removal of a live runner registration
|
||||
**Estimated implement time:** ~6 min
|
||||
**Parallelizable with:** Task 1, Task 4, Task 6
|
||||
|
||||
**Files:**
|
||||
- Modify: `docs/runbooks/TST-30-second-ci-runner.md` (the Correction paragraph mentions
|
||||
"id 4 — an unrelated local macOS runner" — update to final state)
|
||||
|
||||
1. Evidence, local: `pgrep -fl act_runner`, `launchctl list | grep -i act`,
|
||||
`brew services list | grep -i act`, look for `~/.runner`/act_runner config dirs. Evidence,
|
||||
Gitea (token from `~/.zshenv`, never printed): runner detail for id 4 (labels, last
|
||||
online), and whether any recent runs' jobs report `runner_id == 4`
|
||||
(`GET /repos/{owner}/{repo}/actions/runs?…` → `…/runs/{id}/jobs` for both `mxaccessgw`
|
||||
and `lmxopcua` recent runs).
|
||||
2. Decision rule: the runner advertises ubuntu labels from a macOS host, so it can steal
|
||||
Linux container jobs → **remove it** unless evidence shows it deliberately serves jobs
|
||||
the docker runners cannot (none expected). Removal = stop the local act_runner process
|
||||
AND disable its autostart (launchd/brew), then `DELETE /api/v1/admin/actions/runners/4`.
|
||||
Keep the local config file (renamed `*.disabled-2026-08-07`) so re-registering with
|
||||
mac-specific labels stays easy; note the re-registration recipe in the runbook edit.
|
||||
3. Verify: admin runner list shows only ids 1 and 5, both online; no act_runner process
|
||||
locally; a `pgrep` after 60 s still empty (nothing respawned).
|
||||
4. Update the TST-30 runbook correction paragraph; commit locally, do NOT push.
|
||||
|
||||
### Task 6: Harden runner-1's registration token on 10.100.0.35
|
||||
|
||||
**Classification:** standard — touches the live CI stack's compose file
|
||||
**Estimated implement time:** ~7 min
|
||||
**Parallelizable with:** Task 1, Task 4, Task 5
|
||||
|
||||
**Files:** none in repo (host `/opt/gitea/` only; runbook note lands in Task 7 if needed).
|
||||
|
||||
1. Preconditions on the host: confirm runner-1's `/data/.runner` exists in its volume
|
||||
(registration credential persists → the registration env var is no longer needed);
|
||||
confirm both runners idle (no `act_runner`-spawned job containers, no in-progress runs
|
||||
via API) before recreating.
|
||||
2. Edit `/opt/gitea/docker-compose.yml` (backup first → `docker-compose.yml.bak-tst30b`):
|
||||
replace runner-1's inline `GITEA_RUNNER_REGISTRATION_TOKEN: <plaintext>` with the same
|
||||
`_FILE`/secrets pattern runner-2 uses (`/opt/gitea/runner_token`, 0600). Do NOT touch the
|
||||
`gitea` service definition.
|
||||
3. `docker compose up -d --no-deps` the runner-1 service only; verify it comes back online
|
||||
in the admin runner list and its `.runner` identity is unchanged (still id 1).
|
||||
4. Tighten perms: `chmod 600 /opt/gitea/docker-compose.yml docker-compose.yml.bak-tst30 docker-compose.yml.bak-tst30b`
|
||||
(verify compose stack still operable by the deploy user).
|
||||
5. Rotate the leaked registration token if the deployment allows:
|
||||
`docker exec … gitea actions generate-runner-token` (or admin API) — if Gitea offers no
|
||||
invalidation of the old value, say so explicitly in the report (residual risk: LAN actor
|
||||
could register a rogue runner until rotation) rather than claiming it rotated.
|
||||
6. Verify CI still works: trigger nothing; just confirm both runners online and the token
|
||||
file perms; a real push lands naturally later. Report evidence.
|
||||
|
||||
### Task 7: Closeout — cargo Bearer verification, docs/tracker sync, commits
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** none (needs Tasks 3, 4, 5, 6)
|
||||
|
||||
**Files:**
|
||||
- Modify: `archreview/2026-07-12/remediation/90-candidate-findings-next-cycle.md` (final
|
||||
state of NEXT-06/NEXT-07 rows if Tasks 3/4 left anything)
|
||||
- Possibly modify: `docs/GatewayTesting.md` / TST-30 runbook (runner topology now ids 1+5
|
||||
only; token-hardening note)
|
||||
|
||||
1. Cargo Bearer verification (no printing): assert `~/.zshenv` line matches
|
||||
`CARGO_REGISTRIES_DOHERTJ2_GITEA_TOKEN="Bearer …"` via `grep -c`, confirm
|
||||
`docs/ClientPackaging.md` note present (commit `5b153da`); check no other credential
|
||||
location (CI secrets, windev profiles) publishes to cargo — expected none.
|
||||
2. Sweep: every doc touched this cycle consistent (runbooks, trackers, GatewayTesting.md);
|
||||
`git grep` for stale phrases ("crash-loop… pending", "id 4", "admin123") and fix.
|
||||
3. Commit remaining doc changes locally; do NOT push. List the full unpushed stack in the
|
||||
report.
|
||||
|
||||
---
|
||||
|
||||
## Out of scope
|
||||
|
||||
- Pushing any mxaccessgw commits (user decides; stack listed at closeout).
|
||||
- The five next-cycle candidate findings other than NEXT-06/NEXT-07.
|
||||
- Auth-DB restore path for windev (fresh deploy chosen — preserves schema-3 data).
|
||||
- `ci.yml` changes (labels, concurrency groups).
|
||||
|
||||
## Dependency graph
|
||||
|
||||
```
|
||||
{1} → 2 → 3 ┐
|
||||
{4} ├→ 7
|
||||
{5} │
|
||||
{6} ─────────┘
|
||||
```
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"planPath": "docs/plans/2026-08-07-followups-windev-ldapfixtures-runners.md",
|
||||
"tasks": [
|
||||
{"id": 1, "subject": "Task 1: NEXT-07 — Recon windev deployment layout + schema support", "status": "completed"},
|
||||
{"id": 2, "subject": "Task 2: NEXT-07 — Build current Server on windev and redeploy the service", "status": "completed", "blockedBy": [1]},
|
||||
{"id": 3, "subject": "Task 3: SEC-36 deferred verification + NEXT-07/runbook closeout", "status": "completed", "blockedBy": [2]},
|
||||
{"id": 4, "subject": "Task 4: NEXT-06 — Fix DashboardLdapLiveTests fixtures to match the shared directory", "status": "completed"},
|
||||
{"id": 5, "subject": "Task 5: Resolve the unexpected macOS instance runner (id 4)", "status": "completed"},
|
||||
{"id": 6, "subject": "Task 6: Harden runner-1's registration token on 10.100.0.35", "status": "completed"},
|
||||
{"id": 7, "subject": "Task 7: Closeout — cargo Bearer verification, docs/tracker sync, commits", "status": "completed", "blockedBy": [3, 4, 5, 6]}
|
||||
],
|
||||
"lastUpdated": "2026-08-07 (all tasks executed; SEC-36 verification done during Task 2's foreground smoke test; 8 commits local on main, not pushed; one pending operator action: Gitea registration-token UI reset)"
|
||||
}
|
||||
@@ -0,0 +1,420 @@
|
||||
# Live Actions: SEC-36 Rotation, TST-30 Second Runner, Client Publish — Implementation Plan
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers-extended-cc:executing-plans to implement this plan task-by-task (or subagent-driven-development in-session).
|
||||
|
||||
**Goal:** Execute the three repo-complete-but-live-pending operator actions: rotate the dev GLAuth service-account credential (SEC-36), register a second Gitea Actions runner (TST-30), and publish the five client packages at 0.2.0 (Java 0.2.1).
|
||||
|
||||
**Architecture:** Three independent workstreams executed by subagents. SEC-36 is a strictly ordered cutover (pre-stage hosts → flip GLAuth → verify → finalize) with secret-hygiene rules. TST-30 is infra work on docker host 10.100.0.35 plus a concurrency verification. Publish runs the existing guarded `pack-clients.ps1 -Publish` + `tag-go-module.ps1` locally on macOS.
|
||||
|
||||
**Tech Stack:** ssh (BatchMode works to 10.100.0.35 and 10.100.0.48), PowerShell/nssm on windev, docker compose on 10.100.0.35, Gitea API (`~/.zshenv` has admin-scoped `GITEA_USERNAME`/`GITEA_TOKEN`), pwsh 7 on macOS.
|
||||
|
||||
---
|
||||
|
||||
## Preflight facts (verified 2026-08-07 from this macOS box)
|
||||
|
||||
- `ssh 10.100.0.35` OK. GLAuth container is **`zb-shared-glauth`**, compose working dir **`/home/dohertj2/zb-glauth`** (NOT the runbook's `~/Desktop/scadaproj/infra/glauth` — that path does not exist on the host; the runbook must be corrected in Task 5). Runner container **`gitea-runner`**, compose working dir **`/opt/gitea`**.
|
||||
- `ssh 10.100.0.48` (windev) OK; `powershell -NoProfile` works; `nssm` at `C:\Users\dohertj2\AppData\Local\Microsoft\WinGet\Links\nssm.exe`.
|
||||
- `wonder-app-vd03` does NOT resolve from macOS — check it from windev (Task 2).
|
||||
- Gitea API: token valid (`/api/v1/user` → 200), admin (`/api/v1/admin/users` → 200, `POST /api/v1/admin/actions/runners/registration-token` → 200).
|
||||
- Local `~/Desktop/scadaproj/infra/glauth/config.toml` exists (14 `passsha256` entries) — the git source of truth.
|
||||
- `pwsh` at `/usr/local/bin/pwsh`.
|
||||
|
||||
## Secret hygiene (SEC-36, binding for every task)
|
||||
|
||||
- The new plaintext password lives ONLY in `$SECRET_FILE = /private/tmp/claude-501/-Users-dohertj2-Desktop-MxAccessGateway/67849767-a07c-4afa-94e2-3ce4a39d8d23/scratchpad/sec36-new-secret` (chmod 600), created in Task 1 and shredded in Task 5.
|
||||
- **Never echo/cat the plaintext to stdout, never put it in a commit, a repo file, a log line, or a command whose text is captured verbatim.** Always load it into a shell variable from the file (`val=$(cat "$SECRET_FILE")`) and pass it via stdin or remote-side expansion, never inline in an `ssh "...literal..."` string where avoidable.
|
||||
- The `passsha256` hash MAY appear in `config.toml` commits — that is the established pattern (14 existing entries).
|
||||
- The OLD password must never be printed either. Its only uses are: GLAuth keeps honoring it until Task 4, and the single old-bind-must-fail probe in Task 4.
|
||||
|
||||
---
|
||||
|
||||
### Task 1: SEC-36 — Generate secret, stage GLAuth config change (repo + host copy diff)
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Task 2, Task 6, Task 9
|
||||
|
||||
**Files:**
|
||||
- Modify: `~/Desktop/scadaproj/infra/glauth/config.toml` (the `serviceaccount` user's `passsha256`) — DO NOT commit yet (Task 5 commits)
|
||||
- Create: `$SECRET_FILE` (scratchpad, chmod 600)
|
||||
|
||||
**Step 1: Generate the new secret and its hash**
|
||||
|
||||
```bash
|
||||
SECRET_FILE="/private/tmp/claude-501/-Users-dohertj2-Desktop-MxAccessGateway/67849767-a07c-4afa-94e2-3ce4a39d8d23/scratchpad/sec36-new-secret"
|
||||
umask 077
|
||||
openssl rand -base64 24 | tr -d '\n' > "$SECRET_FILE"
|
||||
chmod 600 "$SECRET_FILE"
|
||||
NEW_SHA=$(cat "$SECRET_FILE" | tr -d '\n' | shasum -a 256 | awk '{print $1}')
|
||||
echo "$NEW_SHA" # hash only — safe to display
|
||||
```
|
||||
Cross-check the hash recipe against `glauth.md` ("Generate `passsha256` from a plaintext password") in this repo and follow that recipe if it differs.
|
||||
|
||||
**Step 2: Diff host deployment config vs repo source of truth**
|
||||
|
||||
```bash
|
||||
ssh 10.100.0.35 'cat /home/dohertj2/zb-glauth/config.toml' > /tmp/host-glauth-config.toml 2>/dev/null || true
|
||||
diff ~/Desktop/scadaproj/infra/glauth/config.toml /tmp/host-glauth-config.toml
|
||||
```
|
||||
Small drift (comments, ports) is fine — note it. If the `serviceaccount` stanza differs structurally, STOP and surface before editing.
|
||||
|
||||
**Step 3: Edit the repo source of truth**
|
||||
|
||||
In `~/Desktop/scadaproj/infra/glauth/config.toml`, replace the `passsha256` value of the `[[users]]` entry whose `name`/`cn` is `serviceaccount` with `$NEW_SHA`. Edit ONLY that line. Do not `docker compose up` anything yet.
|
||||
|
||||
**Step 4: Record findings**
|
||||
|
||||
Report: hash staged (show hash, never plaintext), drift summary from step 2, and confirm `$SECRET_FILE` exists with mode 600.
|
||||
|
||||
---
|
||||
|
||||
### Task 2: SEC-36 — Determine wonder-app-vd03 LDAP status (via windev)
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~3 min
|
||||
**Parallelizable with:** Task 1, Task 6, Task 9
|
||||
|
||||
**Step 1: Try to reach vd03 from windev**
|
||||
|
||||
```bash
|
||||
ssh 10.100.0.48 'powershell -NoProfile -Command "Test-Connection wonder-app-vd03 -Count 1 -Quiet"'
|
||||
```
|
||||
|
||||
**Step 2: If reachable, read its gateway config for `MxGateway:Ldap:Enabled`**
|
||||
|
||||
Try (in order, stop at first success): `ssh` hop from windev; reading `\\wonder-app-vd03\c$\...` appsettings/environment via PowerShell remoting (`Invoke-Command -ComputerName wonder-app-vd03`); or `nssm get MxAccessGw AppEnvironmentExtra` remotely. Look for `MxGateway__Ldap__Enabled` / appsettings `Ldap:Enabled`.
|
||||
|
||||
**Step 3: Decide and record**
|
||||
|
||||
- `Enabled=false` or host unreachable/no gateway service → vd03 is OUT of scope; record why (runbook says its dashboard is disabled — `false` is the expected answer).
|
||||
- `Enabled=true` → vd03 is IN scope for Task 3 pre-staging; record the connection method that worked.
|
||||
|
||||
---
|
||||
|
||||
### Task 3: SEC-36 — Pre-stage the NEW value on LDAP-enabled deployed hosts
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** none (blocked by Tasks 1, 2)
|
||||
|
||||
**Step 1: Pre-stage windev (10.100.0.48)**
|
||||
|
||||
Load the secret locally, then set the env var remotely without leaking it into logged command text more than unavoidable (ssh arguments are not logged remotely by default; do NOT echo the value):
|
||||
|
||||
```bash
|
||||
SECRET_FILE="/private/tmp/claude-501/-Users-dohertj2-Desktop-MxAccessGateway/67849767-a07c-4afa-94e2-3ce4a39d8d23/scratchpad/sec36-new-secret"
|
||||
val=$(cat "$SECRET_FILE")
|
||||
ssh 10.100.0.48 'powershell -NoProfile -Command "$v = [Console]::In.ReadLine(); $cur = (& nssm get MxAccessGw AppEnvironmentExtra) -join \"`n\"; Write-Output (\"CURRENT: \" + ($cur -replace \"Password=.*\", \"Password=<redacted>\")); & nssm set MxAccessGw AppEnvironmentExtra (\"MxGateway__Ldap__ServiceAccountPassword=\" + $v)"' <<< "$val"
|
||||
```
|
||||
**CAUTION:** `nssm set AppEnvironmentExtra` REPLACES the whole extra-environment block. First inspect `nssm get MxAccessGw AppEnvironmentExtra` (redacting any `Password=` values); if other variables exist, preserve them in the new value (newline-separated). Adapt quoting as needed — verify with a redacted `nssm get` afterwards.
|
||||
|
||||
**Step 2: Restart the service**
|
||||
|
||||
```bash
|
||||
ssh 10.100.0.48 'nssm restart MxAccessGw'
|
||||
```
|
||||
Expected: service restarts. Binds against GLAuth now fail (old directory, new client value) — expected and brief; proceed immediately to Task 4.
|
||||
|
||||
**Step 3: vd03 (only if Task 2 said IN scope)** — same pre-stage + restart via the method Task 2 found.
|
||||
|
||||
---
|
||||
|
||||
### Task 4: SEC-36 — Rotate GLAuth and verify end-to-end
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** none (blocked by Task 3)
|
||||
|
||||
**Step 1: Back up current host config, sync the staged config, recreate**
|
||||
|
||||
```bash
|
||||
ssh 10.100.0.35 'cp /home/dohertj2/zb-glauth/config.toml /home/dohertj2/zb-glauth/config.toml.bak-sec36'
|
||||
scp ~/Desktop/scadaproj/infra/glauth/config.toml 10.100.0.35:/home/dohertj2/zb-glauth/config.toml
|
||||
```
|
||||
**If Task 1's diff showed host-vs-repo drift beyond the serviceaccount line:** do NOT wholesale-copy — instead edit only the serviceaccount `passsha256` line in the host copy (sed on the host), so unrelated host-local drift is preserved.
|
||||
|
||||
```bash
|
||||
ssh 10.100.0.35 'cd /home/dohertj2/zb-glauth && docker compose up -d --force-recreate && sleep 3 && docker compose logs --tail 30'
|
||||
```
|
||||
Expected: clean startup, no TOML parse error. On parse error: restore `.bak-sec36`, recreate, STOP, surface.
|
||||
|
||||
**Step 2: Verify new credential binds (from the glauth host, ldapsearch or python)**
|
||||
|
||||
```bash
|
||||
SECRET_FILE=".../sec36-new-secret" # full scratchpad path
|
||||
val=$(cat "$SECRET_FILE")
|
||||
ssh 10.100.0.35 'ldapsearch -x -H ldap://localhost:3893 -D "cn=serviceaccount,dc=zb,dc=local" -w "$(cat -)" -b "dc=zb,dc=local" "(cn=multi-role)" cn' <<< "$val"
|
||||
```
|
||||
Expected: search returns the `multi-role` entry. (If ldapsearch is missing on the host, run the equivalent from macOS against `10.100.0.35:3893`, or use `docker exec`.) Adjust the bind DN to match the actual `serviceaccount` DN in config.toml.
|
||||
|
||||
**Step 3: Verify the OLD value is dead — exactly ONE probe, from 10.100.0.35 itself**
|
||||
|
||||
One deliberately failing bind with the old password must return invalid credentials. **Only one attempt** (3-fail/10-min per-IP lockout; never probe from a shared-NAT box). The old value: recover it transiently from `config.toml.bak-sec36`'s hash? No — hash is not the plaintext. Instead: skip the plaintext probe if the old plaintext is not already known out-of-band; the hash replacement in config.toml is itself proof GLAuth no longer honors the old value (GLAuth compares against `passsha256` only). Record that reasoning instead of probing blind.
|
||||
|
||||
**Step 4: Verify dashboard login end-to-end on windev**
|
||||
|
||||
```bash
|
||||
curl -sk -o /dev/null -w '%{http_code}' -c /tmp/mxgw-cookies.txt https://10.100.0.48:5001/login
|
||||
```
|
||||
Find the actual dashboard port from windev config first (`nssm get`/appsettings; likely https). Then POST the login form as `multi-role`/`password` (the GLAuth TEST USER password, not the service account) and expect a redirect + `__Host-MxGatewayDashboard` (or `MxGatewayDashboard`) cookie:
|
||||
|
||||
```bash
|
||||
curl -sk -o /dev/null -w '%{http_code}\n' -b /tmp/mxgw-cookies.txt -c /tmp/mxgw-cookies.txt -d 'username=multi-role&password=password' <dashboard-base>/login
|
||||
grep -i mxgatewaydashboard /tmp/mxgw-cookies.txt
|
||||
```
|
||||
Inspect the login page HTML first for real form field names / antiforgery token; adapt. A successful `multi-role` login proves the service-account search bind works with the new credential end-to-end. If HTTP verification proves impractical (antiforgery), fall back to grepping the gateway log on windev for a successful LDAP bind/login line after attempting — or run the live-LDAP integration test from macOS:
|
||||
|
||||
```bash
|
||||
export MXGATEWAY_RUN_LIVE_LDAP_TESTS=1
|
||||
export MxGateway__Ldap__ServiceAccountPassword="$(cat "$SECRET_FILE")"
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.IntegrationTests/ZB.MOM.WW.MxGateway.IntegrationTests.csproj --filter FullyQualifiedName~DashboardLdapLiveTests
|
||||
```
|
||||
Expected: green. (This binds from macOS to 10.100.0.35:3893 directly — it verifies the credential, and the curl/log check verifies windev.)
|
||||
|
||||
**Step 5: Rollback (only on failure)** — restore `.bak-sec36` on the host, `docker compose up -d --force-recreate`, re-point windev's env var back (old value from where it was before — if unknown, STOP and surface), `nssm restart MxAccessGw`.
|
||||
|
||||
---
|
||||
|
||||
### Task 5: SEC-36 — Finalize: commit source of truth, dev secrets, runbook fix, tracker, cleanup
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** none (blocked by Task 4)
|
||||
|
||||
**Step 1: Commit and push the scadaproj glauth change (glauth paths ONLY)**
|
||||
|
||||
```bash
|
||||
cd ~/Desktop/scadaproj
|
||||
git add infra/glauth/config.toml
|
||||
git commit -m "sec(glauth): rotate serviceaccount passsha256 (mxaccessgw SEC-36)"
|
||||
git push
|
||||
```
|
||||
(`scadaproj` is a shared monorepo — stage only this path. If the worktree has unrelated staged changes, use `git commit -- infra/glauth/config.toml` style isolation.)
|
||||
|
||||
**Step 2: Set dev user-secrets on this macOS box**
|
||||
|
||||
```bash
|
||||
cd ~/Desktop/MxAccessGateway
|
||||
cat "$SECRET_FILE" | tr -d '\n' | dotnet user-secrets set "MxGateway:Ldap:ServiceAccountPassword" --project src/ZB.MOM.WW.MxGateway.Server/ZB.MOM.WW.MxGateway.Server.csproj
|
||||
```
|
||||
(Check `dotnet user-secrets set -h` for stdin support; if unsupported, pass via `"$(cat "$SECRET_FILE")"` — acceptable, it's a local process arg.)
|
||||
|
||||
**Step 3: Correct the runbook + flip tracker rows (mxaccessgw repo)**
|
||||
|
||||
- `docs/runbooks/SEC-36-ldap-credential-rotation.md`: fix the host deployment path (`/home/dohertj2/zb-glauth`, container `zb-shared-glauth`; repo source of truth remains `scadaproj/infra/glauth/`), and note vd03's actual status per Task 2.
|
||||
- Grep `archreview/2026-07-12/remediation/` for SEC-36 pending-operator rows; flip to Done citing the runbook + today's date.
|
||||
|
||||
```bash
|
||||
cd ~/Desktop/MxAccessGateway
|
||||
grep -rn "SEC-36" archreview/2026-07-12/remediation/ docs/ | grep -iv binary
|
||||
# edit the rows, then:
|
||||
git add -A docs archreview && git commit -m "docs(sec-36): record live rotation done; correct runbook host paths"
|
||||
```
|
||||
|
||||
**Step 4: Shred the secret file**
|
||||
|
||||
```bash
|
||||
rm -P "$SECRET_FILE" 2>/dev/null || rm "$SECRET_FILE"
|
||||
```
|
||||
|
||||
**Step 5: Done-criteria check** — walk the runbook's Done criteria list; report each as met/not-met.
|
||||
|
||||
---
|
||||
|
||||
### Task 6: TST-30 — Recon existing runner config on 10.100.0.35
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** Task 1, Task 2, Task 9
|
||||
|
||||
**Step 1: Inspect the existing runner**
|
||||
|
||||
```bash
|
||||
ssh 10.100.0.35 'cat /opt/gitea/docker-compose.yml 2>/dev/null || sudo cat /opt/gitea/docker-compose.yml; ls /opt/gitea'
|
||||
ssh 10.100.0.35 'docker inspect gitea-runner --format "{{json .Mounts}}"; docker exec gitea-runner cat /config.yaml 2>/dev/null || true'
|
||||
```
|
||||
Find: image/version, config file location (look for `container.network: traefik` and `capacity`/`maxParallel`), data volume, registration state file, docker socket mount, labels.
|
||||
|
||||
**Step 2: Check host capacity**
|
||||
|
||||
```bash
|
||||
ssh 10.100.0.35 'nproc; free -h; df -h / | tail -1'
|
||||
```
|
||||
|
||||
**Step 3: Decide (a)-variant** — second container vs raising `capacity` on the existing runner. Runbook prefers a second instance; if the existing runner's config shows a simple `capacity: 1` and resources are tight, raising capacity is the smaller change — but a second registered instance is the runbook default and survives one-runner wedge. Record the chosen variant, the exact compose/config snippets to reuse, and where the registration token goes.
|
||||
|
||||
---
|
||||
|
||||
### Task 7: TST-30 — Register and start the second runner
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** none (blocked by Task 6)
|
||||
|
||||
**Step 1: Mint an instance-level registration token**
|
||||
|
||||
```bash
|
||||
source ~/.zshenv
|
||||
curl -s -X POST -u "$GITEA_USERNAME:$GITEA_TOKEN" 'https://gitea.dohertylan.com/api/v1/admin/actions/runners/registration-token'
|
||||
```
|
||||
(Returns `{"token": "..."}` — a registration token, not a secret credential of lasting value; still avoid committing it.)
|
||||
|
||||
**Step 2: Create the second runner instance per Task 6's plan**
|
||||
|
||||
E.g. add a `gitea-runner-2` service to the compose (distinct name + data volume, same image, same `container.network: traefik`, same socket mount), inject the token via the runner's registration env (`GITEA_RUNNER_REGISTRATION_TOKEN`) or `act_runner register --no-interactive`, then `docker compose up -d gitea-runner-2` from `/opt/gitea`. Back up the compose file first (`cp docker-compose.yml docker-compose.yml.bak-tst30`). Do NOT touch the existing `gitea-runner` service definition.
|
||||
|
||||
**Step 3: Confirm both runners online**
|
||||
|
||||
```bash
|
||||
curl -s -u "$GITEA_USERNAME:$GITEA_TOKEN" 'https://gitea.dohertylan.com/api/v1/admin/actions/runners' | python3 -m json.tool
|
||||
```
|
||||
Expected: ≥2 runners, both online. Also check `docker logs` of the new container for a clean registration + poll loop.
|
||||
|
||||
---
|
||||
|
||||
### Task 8: TST-30 — Verify concurrency, gitea:3000 resolution, tracker
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** none (blocked by Task 7)
|
||||
|
||||
**Step 1: Trigger two concurrent runs**
|
||||
|
||||
Push two scratch branches to `mxaccessgw` back-to-back (empty commits off `main`, branch names `scratch/tst30-a`, `scratch/tst30-b`):
|
||||
|
||||
```bash
|
||||
cd ~/Desktop/MxAccessGateway
|
||||
git push origin main:refs/heads/scratch/tst30-a
|
||||
git commit --allow-empty -m "tst30 concurrency probe" && git push origin HEAD:refs/heads/scratch/tst30-b && git reset --hard HEAD~1
|
||||
```
|
||||
(Adapt: any two pushes that fan out jobs. Clean up branches after: `git push origin :scratch/tst30-a :scratch/tst30-b`.)
|
||||
|
||||
**Step 2: Confirm parallel execution**
|
||||
|
||||
Poll the runs API/UI: the second run's jobs must START before the first run finishes.
|
||||
|
||||
```bash
|
||||
curl -s -u "$GITEA_USERNAME:$GITEA_TOKEN" 'https://gitea.dohertylan.com/api/v1/repos/dohertj2/mxaccessgw/actions/tasks' | python3 -m json.tool | head -60
|
||||
```
|
||||
|
||||
**Step 3: Confirm `gitea:3000` resolves on the new runner** — verify a job scheduled on runner-2 succeeds at checkout (checkout hits `gitea:3000` over the traefik network); identify which runner took each job from the runs UI/API or runner logs.
|
||||
|
||||
**Step 4: Flip TST-30 tracker rows** in `archreview/2026-07-12/remediation/` (grep `TST-30`) to Done with today's date; confirm `docs/GatewayTesting.md` prose is still accurate (it should be — it already describes the bypass as valid regardless of runner count). Commit.
|
||||
|
||||
---
|
||||
|
||||
### Task 9: Publish — Preflight audit (versions, registry collisions, toolchains)
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** Task 1, Task 2, Task 6
|
||||
|
||||
**Step 1: Audit source versions**
|
||||
|
||||
```bash
|
||||
cd ~/Desktop/MxAccessGateway
|
||||
grep -n 'version' clients/rust/Cargo.toml | head -5
|
||||
grep -n 'version' clients/python/pyproject.toml clients/python/src/zb_mom_ww_mxgateway/version.py
|
||||
grep -n 'ClientVersion' clients/go/mxgateway/version.go
|
||||
grep -n '<Version>' clients/dotnet/ZB.MOM.WW.MxGateway.Client/ZB.MOM.WW.MxGateway.Client.csproj src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj
|
||||
grep -n 'version' clients/java/build.gradle | head -5
|
||||
grep -rn 'CLIENT_VERSION' clients/java --include=*.java | grep -i mxgatewayclientversion
|
||||
```
|
||||
Expected: Rust/Python/Go/.NET/Contracts = 0.2.0; Java build.gradle AND `MxGatewayClientVersion.CLIENT_VERSION` = 0.2.1. Any mismatch → STOP, surface (do not bump versions yourself; that's a scope change).
|
||||
|
||||
**Step 2: Query live registry for collisions**
|
||||
|
||||
```bash
|
||||
source ~/.zshenv
|
||||
for u in 'nuget/ZB.MOM.WW.MxGateway.Client/0.2.0' 'nuget/ZB.MOM.WW.MxGateway.Contracts/0.2.0' 'pypi/zb-mom-ww-mxaccess-gateway-client/0.2.0' 'cargo/zb-mom-ww-mxgateway-client/0.2.0' 'maven/com.zb.mom.ww.mxgateway-zb-mom-ww-mxgateway-client/0.2.1'; do
|
||||
echo "$u => $(curl -s -o /dev/null -w '%{http_code}' -u "$GITEA_USERNAME:$GITEA_TOKEN" "https://gitea.dohertylan.com/api/v1/packages/dohertj2/$u")"
|
||||
done
|
||||
```
|
||||
Expected: 404 for every target (unclaimed). Check the exact maven path convention against `pack-clients.ps1`'s own guard code and use its convention. 200 anywhere → STOP, surface.
|
||||
|
||||
**Step 3: Toolchain + workspace check**
|
||||
|
||||
```bash
|
||||
git -C ~/Desktop/MxAccessGateway status --porcelain # must be clean (publish from a clean tree at origin/main)
|
||||
for t in dotnet cargo go python3 gradle pwsh; do which $t; done
|
||||
```
|
||||
Also confirm `clients/go` module tag `clients/go/v0.2.0` does NOT already exist: `git ls-remote --tags origin 'clients/go/v*'`.
|
||||
|
||||
---
|
||||
|
||||
### Task 10: Publish — Run the guarded pack-and-publish
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min dispatch (script runtime longer)
|
||||
**Parallelizable with:** none (blocked by Task 9)
|
||||
|
||||
**Step 1: Run pack-clients with publish**
|
||||
|
||||
```bash
|
||||
cd ~/Desktop/MxAccessGateway
|
||||
source ~/.zshenv
|
||||
pwsh -NoProfile -File scripts/pack-clients.ps1 -Publish 2>&1 | tee /private/tmp/claude-501/-Users-dohertj2-Desktop-MxAccessGateway/67849767-a07c-4afa-94e2-3ce4a39d8d23/scratchpad/pack-clients-publish.log
|
||||
```
|
||||
Expected: per-language build+test+pack, collision guard prints "safe to publish" per artifact, uploads succeed. Timeout generously (Bash timeout 600000). If any language fails MID-loop, record exactly which artifacts pushed and which didn't — partial publish is the known failure mode; do not re-run blindly (re-run is safe only because the guard skips? NO — the guard ABORTS on existing versions. A re-run after partial publish will abort on the already-pushed artifact. If that happens, surface with the log; per-language `-Languages` selective re-run is the fix).
|
||||
If macOS cannot build a language (e.g. gradle/java env), use `-Languages` to publish what builds and surface the remainder — do not fake success.
|
||||
|
||||
**Step 2: Verify each artifact now exists (200)** — re-run Task 9 step 2's loop; expected 200 everywhere published.
|
||||
|
||||
---
|
||||
|
||||
### Task 11: Publish — Go module tag
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~3 min
|
||||
**Parallelizable with:** Task 10 (blocked by Task 9)
|
||||
|
||||
**Step 1: Tag via the guarded script**
|
||||
|
||||
```bash
|
||||
cd ~/Desktop/MxAccessGateway
|
||||
pwsh -NoProfile -File scripts/tag-go-module.ps1 -Version 0.2.0
|
||||
```
|
||||
Read the script's param block first (`-Version` name may differ; it validates semver and that `version.go` matches, then creates+pushes `clients/go/v0.2.0`). Expected: tag created and pushed to origin.
|
||||
|
||||
**Step 2: Verify**
|
||||
|
||||
```bash
|
||||
git ls-remote --tags origin 'clients/go/v0.2.0*'
|
||||
```
|
||||
Expected: exactly one tag. Optionally `GOPROXY=direct go list -m gitea.dohertylan.com/dohertj2/mxaccessgw/clients/go@v0.2.0` from a temp dir.
|
||||
|
||||
---
|
||||
|
||||
### Task 12: Publish — Docs/tracker closeout
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** none (blocked by Tasks 10, 11)
|
||||
|
||||
**Step 1:** Update `docs/ClientPackaging.md`'s versioning narrative if it claims 0.2.0/0.2.1 are unpublished (it currently records the maven 0.2.1 exception; add a dated line that 0.2.0 (Java 0.2.1) published on 2026-08-07). Grep `archreview/2026-07-12/remediation/` for publish/CLI-39 pending-operator rows and flip to Done.
|
||||
|
||||
**Step 2:** Commit:
|
||||
|
||||
```bash
|
||||
cd ~/Desktop/MxAccessGateway
|
||||
git add docs archreview && git commit -m "docs(clients): record 0.2.0/0.2.1 publish + close operator actions"
|
||||
```
|
||||
|
||||
**Step 3:** Report the full publish matrix (artifact → version → registry HTTP status).
|
||||
|
||||
---
|
||||
|
||||
## Dependency graph
|
||||
|
||||
```
|
||||
{T1, T2} ──▶ T3 ──▶ T4 ──▶ T5 (SEC-36, strictly serial after recon)
|
||||
T6 ──▶ T7 ──▶ T8 (TST-30)
|
||||
T9 ──▶ {T10, T11} ──▶ T12 (Publish)
|
||||
```
|
||||
The three streams are mutually independent and run concurrently. All subagents run with model=opus per operator instruction.
|
||||
|
||||
## Out of scope (explicitly)
|
||||
|
||||
- Option (b)/(c) runner topologies and the `concurrency:` ci.yml experiment (TST-30 runbook marks them escalation/optional).
|
||||
- The five next-cycle candidate findings in `archreview/2026-07-12/remediation/90-candidate-findings-next-cycle.md`.
|
||||
- Any client version bumps (versions are already landed; a mismatch is a STOP-and-surface).
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"planPath": "docs/plans/2026-08-07-live-actions-sec36-tst30-publish.md",
|
||||
"tasks": [
|
||||
{"id": 1, "subject": "Task 1: SEC-36 — Generate secret, stage GLAuth config change", "status": "completed"},
|
||||
{"id": 2, "subject": "Task 2: SEC-36 — Determine wonder-app-vd03 LDAP status via windev", "status": "completed"},
|
||||
{"id": 3, "subject": "Task 3: SEC-36 — Pre-stage NEW value on LDAP-enabled hosts (nssm + restart)", "status": "completed", "blockedBy": [1, 2]},
|
||||
{"id": 4, "subject": "Task 4: SEC-36 — Rotate GLAuth and verify end-to-end", "status": "completed", "blockedBy": [3]},
|
||||
{"id": 5, "subject": "Task 5: SEC-36 — Finalize: commit, dev secrets, runbook fix, tracker, cleanup", "status": "completed", "blockedBy": [4]},
|
||||
{"id": 6, "subject": "Task 6: TST-30 — Recon existing runner config on 10.100.0.35", "status": "completed"},
|
||||
{"id": 7, "subject": "Task 7: TST-30 — Register and start the second runner", "status": "completed", "blockedBy": [6]},
|
||||
{"id": 8, "subject": "Task 8: TST-30 — Verify concurrency + gitea:3000 + tracker", "status": "completed", "blockedBy": [7]},
|
||||
{"id": 9, "subject": "Task 9: Publish — Preflight audit (versions, collisions, toolchains)", "status": "completed"},
|
||||
{"id": 10, "subject": "Task 10: Publish — Run pack-clients.ps1 -Publish", "status": "completed", "blockedBy": [9]},
|
||||
{"id": 11, "subject": "Task 11: Publish — Go module tag clients/go/v0.2.0", "status": "completed", "blockedBy": [9]},
|
||||
{"id": 12, "subject": "Task 12: Publish — Docs/tracker closeout", "status": "completed", "blockedBy": [10, 11]}
|
||||
],
|
||||
"lastUpdated": "2026-08-07 (all tasks executed; 4 closeout commits local on main, not pushed)"
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
# Dashboard UI cleanup sweep (2026-08-11)
|
||||
|
||||
Runs the family admin-UI cleanup playbook (`../scadaproj/admin_ui_cleanup.md`) against the
|
||||
MXAccess Gateway Blazor dashboard. Third app in the family after the ignitionoee router and the
|
||||
OtOpcUa AdminUI.
|
||||
|
||||
## 0. Platform correction — this app is *not* Bootstrap-free
|
||||
|
||||
The umbrella index describes mxaccessgw as the family's Bootstrap-free app. That is wrong, and
|
||||
every Bootstrap-dependent recipe in the playbook applies here unchanged. What actually ships:
|
||||
|
||||
| Layer | Evidence |
|
||||
|---|---|
|
||||
| Bootstrap 5.3.3, self-hosted | `libman.json:5`; `wwwroot/lib/bootstrap/css/bootstrap.min.css` |
|
||||
| Linked first in the head | `Dashboard/Components/App.razor:7` |
|
||||
| `ZB.MOM.WW.Theme` 0.3.1 at discovery, **0.4.0 after §6** | `ZB.MOM.WW.MxGateway.Server.csproj` `<PackageReference>`; `<ThemeHead />` at `App.razor:8` |
|
||||
| App stylesheet, loaded last | `App.razor:9` → `wwwroot/css/site.css` |
|
||||
| Bootstrap JS bundle | `App.razor:14` |
|
||||
|
||||
What CLAUDE.md actually forbids is Blazor **component libraries** (MudBlazor, Radzen, FluentUI) —
|
||||
not Bootstrap's CSS/JS. No Bootstrap was introduced by this sweep.
|
||||
|
||||
Playbook §2 foundation item 1 (*app stylesheet after `<ThemeHead />` so it wins the cascade*) was
|
||||
therefore **already satisfied** before the sweep.
|
||||
|
||||
## 1. Discovery
|
||||
|
||||
### 1a. Foundation / CSS audit
|
||||
|
||||
**Shipped-vs-used class matrix.** Every class in `Dashboard/Components/**/*.razor` checked against
|
||||
theme 0.3.1 `staticwebassets/css/{theme,layout}.css`, `bootstrap.min.css`, and `site.css`. Three
|
||||
classes are defined nowhere:
|
||||
|
||||
| Ghost class | Site | Consequence | Verdict |
|
||||
|---|---|---|---|
|
||||
| `tree-load-status` | `Shared/BrowseTreeNodeView.razor:42`, `:49` | **Real visual defect.** The row's indent spacer is `<span class="tree-toggle tree-toggle-empty">`; `.tree-toggle` sets `flex:none;width:1.1rem`, which only takes effect on a flex item. `.tree-row`/`.tree-attr` are flex; this container is an undefined block, so the span stays inline and `width` is ignored — "⌛ Loading…" and "Failed to load: …" render flush left, out of alignment with every sibling row. | **Define it** |
|
||||
| `browse-stale-banner` | `Pages/BrowsePage.razor:78` | The banner carries `@onclick="ClearStaleBanner"` with no pointer affordance and no styling of its own — a click-to-dismiss control that does not look clickable. | **Define it** |
|
||||
| `tree-node` | `Shared/BrowseTreeNodeView.razor:15` | Structural wrapper only; nothing needs to style it. A no-op, but a deliberate one. | **Leave** |
|
||||
|
||||
(Reported as ghosts by the raw extractor but false positives: `h-100` at `Shared/MetricCard.razor:1`
|
||||
— Bootstrap-defined, mangled by the extractor's handling of the inline `@(...)` class expression.)
|
||||
|
||||
**Scoped-CSS bundle.** N/A — the project contains **zero** `*.razor.css` files, so no
|
||||
`*.bundle.scp.css` is emitted and nothing is missing from the head. (This was OtOpcUa's finding; it
|
||||
does not exist here.)
|
||||
|
||||
**Phantom CSS custom properties.** Zero. `Dashboard/Components/` contains no `var(--…)` at all —
|
||||
tokens are used only from `site.css`, and every one of them (`--ink`, `--ink-soft`, `--ink-faint`,
|
||||
`--card`, `--rule`, `--rule-strong`, `--mono`, `--accent`, `--accent-deep`, `--ok`, `--ok-bg`,
|
||||
`--bad`, `--bad-bg`, `--warn`, `--warn-bg`, `--idle`, `--idle-bg`) resolves against theme 0.3.1.
|
||||
This app has the OtOpcUa-clean result, not the router's.
|
||||
|
||||
**Button sizing — the one real foundation defect.** Theme 0.3.1 ships no `.btn` rule (confirmed:
|
||||
`.btn` appears in `layout.css` only inside a comment). But `site.css:187` does, and it sets
|
||||
`font-size` **directly** rather than through Bootstrap's variable:
|
||||
|
||||
```css
|
||||
.btn { border-radius: 5px; font-size: 0.82rem; font-weight: 500; white-space: nowrap; }
|
||||
```
|
||||
|
||||
Bootstrap renders `.btn { font-size: var(--bs-btn-font-size) }`, and `.btn-sm` /
|
||||
`.btn-group-sm > .btn` size themselves purely by *redefining that variable*
|
||||
(`.btn-sm{--bs-btn-font-size:0.875rem}`). A literal `font-size` on `.btn` at equal specificity,
|
||||
loaded later, wins over the variable-driven declaration for **every** button — so `btn-sm` and
|
||||
`btn-group-sm` are font-size no-ops app-wide and small buttons differ from full-size ones by
|
||||
padding alone. This is the same class of defect the other two apps hit from the opposite
|
||||
direction (no `.btn` rule at all), and it takes the same fix.
|
||||
|
||||
**Dark scheme.** Theme 0.3.1 is light-only; `site.css` makes no `prefers-color-scheme` /
|
||||
`data-bs-theme` claim, and its header comment ("Layers over theme.css … every colour resolves to a
|
||||
theme.css token") is accurate. Nothing to correct.
|
||||
|
||||
### 1b. Button inventory
|
||||
|
||||
`grep -rn "btn-group"` returns **three** — this app already uses the convention where it matters:
|
||||
|
||||
| Site | Members | State |
|
||||
|---|---|---|
|
||||
| `Pages/ApiKeysPage.razor:189` | Rotate / Revoke, or Delete | Correct — `btn-group btn-group-sm`, no per-button `btn-sm`, `@if` inside the group |
|
||||
| `Pages/SessionsPage.razor:90` | Close / Kill | Correct |
|
||||
| `Pages/SessionDetailsPage.razor:34` | Close session / Kill worker | Correct |
|
||||
|
||||
Adjacent related buttons **not** yet grouped (both are feet, the playbook's named case):
|
||||
|
||||
| Site | Members | Fix |
|
||||
|---|---|---|
|
||||
| `Shared/ConfirmDialog.razor:17,22` | Cancel + `@ConfirmButtonClass` confirm, in a `modal-footer` | Wrap in `btn-group` |
|
||||
| `Pages/ApiKeysPage.razor:136,137` | Save (`type="submit"`) + Cancel, in the create-key card body | Wrap in `btn-group btn-group-sm`; fold the two `btn-sm` and drop the `me-1` spacer |
|
||||
|
||||
Refused / not candidates:
|
||||
|
||||
- `Pages/WorkersPage.razor:74` — a lone Kill button. Nothing to group.
|
||||
- `Pages/ApiKeysPage.razor:22` — lone page-head "Create API Key".
|
||||
- `Pages/BrowsePage.razor:109` (`Clear all`) and `:167` (`Remove`) — lone buttons.
|
||||
- `Shared/BrowseTreeNodeView.razor:19` `.tree-toggle` — a bare expander, deliberately unstyled as a
|
||||
button; `MainLayout.razor:32` Sign Out / `:36` Sign In are the theme's `rail-btn`, one per
|
||||
auth branch and mutually exclusive.
|
||||
|
||||
**Row actions styled as links**: none. Every action in the app is already a real `<button>`; the
|
||||
only `<a>` in `Dashboard/Components/` is `MainLayout.razor:36`, which navigates. `btn-link`: zero
|
||||
uses. `·` action separators: zero — the `·` occurrences (`BrowsePage.razor:54,102,106,287`,
|
||||
`BrowseTreeNodeView.razor:71`, `AlarmsPage.razor:220`) all separate *metadata facts* or serve as a
|
||||
bullet glyph, never actions. Inline `width:` sizing hacks: zero. `py-0`: zero.
|
||||
|
||||
**Arm→confirm flows** (restyle-only; the two-step must survive):
|
||||
|
||||
| Page | Flow |
|
||||
|---|---|
|
||||
| `SessionsPage` | Close / Kill → `ConfirmDialog` → `ConfirmPendingAsync` |
|
||||
| `SessionDetailsPage` | Close session / Kill worker → `ConfirmDialog` |
|
||||
| `WorkersPage` | Kill → `ConfirmDialog` |
|
||||
| `ApiKeysPage` | Rotate / Revoke / Delete → `ConfirmDialog`; Create → modal form |
|
||||
|
||||
**Size mismatches within a group**: none.
|
||||
|
||||
### 1c. Prose inventory (DELETE / KEEP / RELOCATE)
|
||||
|
||||
| Site | Text | Verdict |
|
||||
|---|---|---|
|
||||
| `Pages/GalaxyPage.razor:134-138` | "Browse data is served by the `galaxy_repository.v1.GalaxyRepository` gRPC service. Clients call `DiscoverHierarchy` for the full tree and `GetLastDeployTime` to detect redeployments." | **DELETE** — unconditional client-API documentation on an operator page. Every fact is already in `docs/GalaxyRepository.md` (service name at :44, `GetLastDeployTime` at :49, `DiscoverHierarchy` at :50). Plain delete, no relocation needed. |
|
||||
| `Pages/AlarmsPage.razor:151-154` | "Cleared alarms are not retained — this list reflects only alarms currently Active or ActiveAcked, refreshed every 3 seconds." | KEEP — decodes what the list contains and does not contain; the operator cannot infer "cleared alarms are absent" from the data. |
|
||||
| `Pages/AlarmsPage.razor:26-30` | Alarms-disabled banner citing `MxGateway:Alarms:Enabled` | KEEP — conditional state banner. **Config key verified against the options class**: `GatewayOptions.Alarms` → `AlarmsOptions.Enabled`, present in `appsettings.json`. |
|
||||
| `Pages/BrowsePage.razor:33-35`, `:41`, `:120-124` | Empty states | KEEP |
|
||||
| `Pages/BrowsePage.razor:70`, `:90` | "Showing the first N matches — refine the filter." / "Double-click a tag, or right-click for the menu." | KEEP — truncation notice, and the only decoder of two interactions that have no visible affordance. |
|
||||
| `Pages/SessionDetailsPage.razor:115-118` | "Waiting for events. The dashboard mirrors the session's gRPC event stream — events appear here only while a gRPC client is also consuming this session's events." | KEEP — explains an empty state that otherwise reads as a bug. |
|
||||
| `Pages/GalaxyPage.razor:33-38` | Unknown-status empty state | KEEP |
|
||||
|
||||
**Stale-claim hunt** (wrong facts outrank style): none found. No milestone labels ("F8/F9 pending",
|
||||
"Batch 2"), no dead repo hyperlinks, no superseded architecture claims. Every config key cited in
|
||||
markup resolves — `MxGateway:Alarms:Enabled` (above) is the only one. Also checked and **cleared**:
|
||||
`SettingsPage.razor:32` renders `Ldap.ServiceAccountPassword`, but
|
||||
`Configuration/GatewayConfigurationProvider.cs:30` substitutes `RedactedValue` before the snapshot
|
||||
is built, so no credential reaches the page.
|
||||
|
||||
### 1d. Density / layout scan (per page, not per file)
|
||||
|
||||
Structural mitigations already present app-wide: `site.css:137` caps `.dashboard-table td` at
|
||||
`max-width: 26rem` with `overflow-wrap: break-word`, so the OtOpcUa `/hosts` failure mode (one long
|
||||
exception widens the table until the actions column scrolls off) **cannot happen here**. Every table
|
||||
is also inside `.table-responsive`. `panel-head` appears zero times, so the "sections running
|
||||
together" tell does not fire — each group is its own `section.dashboard-section` card already.
|
||||
|
||||
Residual finding — **unbounded free-text columns**. The 26rem cap converts the horizontal blowout
|
||||
into vertical blowout: a multi-line exception makes one row several times taller than its
|
||||
neighbours and pushes the rest of the table off-screen.
|
||||
|
||||
| Site | Column | Bound to |
|
||||
|---|---|---|
|
||||
| `Shared/FaultList.razor:28` | Message | `@fault.Message` — worker/COM fault text, uncontrolled |
|
||||
| `Pages/SessionsPage.razor:86` | Fault | `session.LastFault` |
|
||||
| `Pages/WorkersPage.razor:70` | Fault | `worker.LastFault` |
|
||||
| `Shared/BrowseTreeNodeView.razor:51` | (tree row) | `@Node.LoadError` — **worst case**: `.tree-attr`/`.tree-row` siblings are `white-space: nowrap` inside a fixed-height scroller, so a long browse error stretches the left pane horizontally |
|
||||
| `Pages/DashboardHome.razor:47` | Galaxy panel | `Snapshot.Galaxy.LastError` on the compact overview |
|
||||
|
||||
Deliberately **not** truncated (the full text must stay reachable — playbook rule):
|
||||
`SessionDetailsPage.razor:84` "Last fault" and `GalaxyPage.razor:47` "Last Error" are both
|
||||
full-width rows on the drill-down page each summary links to.
|
||||
|
||||
Not firing: **identity slam** — `SessionsPage.razor:75-81` puts a worker pid next to a status chip,
|
||||
which the playbook explicitly permits ("chips stay with the name line"); no cell renders two
|
||||
identifiers. **Inline full-width expander rows** — zero `colspan` detail rows in the app.
|
||||
|
||||
### 1e. Tree tables
|
||||
|
||||
**No adoption.** `BrowsePage` + `BrowseTreeNodeView` is a lazy-loading nav/picker tree with a
|
||||
context menu and no columns — the rubric's explicit "different animal, leave it alone". No table in
|
||||
the app flattens hierarchy through a path-prefix column, hand-rolled `rowspan`, indent-by-padding,
|
||||
or faked group-header rows; `Sessions`/`Workers`/`Events`/`Alarms`/`Galaxy` are flat fact lists and
|
||||
`Top Templates` is rank-ordered (a tree would destroy load-bearing ordering). Porting the router's
|
||||
`TreeTable` here would create an orphan.
|
||||
|
||||
### Guard tests
|
||||
|
||||
Checked for source-scan tests pinning dashboard markup: none. The only `.razor` reference in the
|
||||
test project is a prose comment (`Gateway/GatewayApplicationTests.cs:116`). Nothing to re-pin.
|
||||
|
||||
## 2. Changes
|
||||
|
||||
### Foundation
|
||||
|
||||
1. **`wwwroot/css/site.css` — button sizing via Bootstrap CSS variables.** Replace the literal
|
||||
`font-size` on `.btn` with `--bs-btn-*` overrides and add the `btn-sm` / `btn-group-sm` block, so
|
||||
the small-button distinction works again. Box properties are *not* redefined wholesale —
|
||||
`border-radius: 5px` stays literal because the three existing `btn-group`s already render seamed
|
||||
under it (`.btn-group > .btn:not(:first-child)` outranks `.btn`). Carries the header comment the
|
||||
other two apps carry: delete the local copy when a `ZB.MOM.WW.Theme` release ships a `.btn` rule.
|
||||
**That release shipped the same day — see §6.**
|
||||
2. **`site.css` — define `.tree-load-status`** as a flex row matching `.tree-row`, restoring the
|
||||
indent alignment of the tree's loading/error rows.
|
||||
3. **`site.css` — define `.browse-stale-banner`** with `cursor: pointer` and tightened padding, so
|
||||
the click-to-dismiss banner reads as clickable. No behavior change.
|
||||
4. **`Dashboard/Components/DashboardDisplay.cs` — add `Abbreviate(string?, int)`.** Safe helper
|
||||
(length-checked, never a raw `[..n]` slice — the OtOpcUa #504 failure), `-` for null/whitespace,
|
||||
ellipsis on truncation.
|
||||
|
||||
No prose relocation task: the single DELETE is already docs-covered.
|
||||
|
||||
### Page batches
|
||||
|
||||
**Batch A — free-text truncation** (`Shared/FaultList.razor`, `Pages/SessionsPage.razor`,
|
||||
`Pages/WorkersPage.razor`, `Shared/BrowseTreeNodeView.razor`, `Pages/DashboardHome.razor`):
|
||||
`Abbreviate` + full text on `title`, at the five sites in 1d.
|
||||
|
||||
**Batch B — button feet** (`Shared/ConfirmDialog.razor`, `Pages/ApiKeysPage.razor`): the two
|
||||
`btn-group` wraps from 1b. `@onclick`, `disabled`, `type="submit"`, and both arm→confirm flows
|
||||
unchanged.
|
||||
|
||||
**Batch C — prose** (`Pages/GalaxyPage.razor`): delete `:134-138`.
|
||||
|
||||
## 3. Verification
|
||||
|
||||
### Build + tests
|
||||
|
||||
- [x] `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx` — 0 warnings, 0 errors
|
||||
(`TreatWarningsAsErrors=true`).
|
||||
- [x] `dotnet test src/ZB.MOM.WW.MxGateway.Tests` — 879/879 passed, the macOS baseline.
|
||||
|
||||
### Post-merge greps
|
||||
|
||||
- [x] Zero phantom `var(--…)` usages (there were none to begin with).
|
||||
- [x] Zero ghost classes remaining except the deliberate `tree-node`.
|
||||
- [x] `btn-group` count 3 → 5, matching the 1b inventory.
|
||||
- [x] Zero `·` action separators, zero inline `width:` hacks (none existed).
|
||||
- [x] Scoped-bundle link: N/A, no `*.razor.css`.
|
||||
|
||||
### Live browser gate
|
||||
|
||||
Recorded in §4 below.
|
||||
|
||||
## 4. Live browser gate — results
|
||||
|
||||
Run against a local `dotnet run` of the gateway on macOS (`http://localhost:5120`, the launch
|
||||
profile's port), not windev — the sweep is CSS/markup-only and redeploying the shared NSSM service
|
||||
was not warranted. Rig configuration (env overrides only, no committed config touched):
|
||||
`Dashboard:DisableLogin=true` (to reach the Admin-only surfaces), `Ldap:Enabled=false`,
|
||||
`Authentication:Mode=Disabled`, SQLite + Galaxy snapshot paths under the session scratchpad,
|
||||
`ApiKeyPepper` set locally. The rig was stopped and its throwaway auth DB deleted afterwards.
|
||||
|
||||
**Realistic erroring data**: the Galaxy SQL Server (`localhost`, `ZB`) does not exist on macOS, so
|
||||
every refresh fails with a genuine 250-character `Microsoft.Data.SqlClient` SSPI exception, and the
|
||||
alarm monitor's auto-opened session fails on the missing x86 worker — both real, uncontrolled
|
||||
error text of exactly the kind §1d is about.
|
||||
|
||||
| # | Check | Result | Evidence |
|
||||
|---|---|---|---|
|
||||
| 1 | Computed-style probes | **PASS** | body `14.4px`; `.btn` `13.6px` (0.85rem); `.btn-sm` `12.48px` (0.78rem) — the small/full distinction is live, where before the fix both computed to `13.12px`. `btn-group-sm` members: both `12.48px`, first member's right radius `0px`, last member's `5px` → **seams intact**, confirming the literal `border-radius` does not break group machinery. `.tree-load-status` → `display: flex`; `.browse-stale-banner` → `cursor: pointer`. Stylesheet order: bootstrap → theme → layout → **site.css last**. The theme's own `.rail-toggle btn-sm` also picks up `12.48px`, so the block reaches theme-owned buttons, not just app markup. |
|
||||
| 2 | Every page leads with data, no unconditional doc block above the fold | **PASS** | All 9 routes' first blocks are `dashboard-page-header` → `metric-grid` / `dashboard-section` / state `alert`. `/galaxy` now ends at the Sync Info table — the deleted client-API paragraph is gone. |
|
||||
| 3 | Seamed `btn-group`s; destructive members red | **PASS** | `/apikeys` row actions render Rotate + Revoke as one seamed group with Revoke in `btn-outline-danger` red. The `ConfirmDialog` foot renders Cancel + a solid red Revoke seamed together. |
|
||||
| 4 | Arm→confirm: arm → confirm UI appears → **Cancel** → verify disarm | **PASS** | Revoke → "Revoke API key?" dialog → Cancel. Post-cancel probe: `dialogOpen:false`, `backdrops:0`, key status still `Active`, row action back to `btn-outline-danger` outline. No write completed. Save/Cancel in the create-key modal also verified: Cancel closes without creating; Save creates (validation message surfaced correctly on the first attempt with a missing display name). |
|
||||
| 5 | Anything the scoped bundle resurrected | **N/A** | No `*.razor.css` in the project — see 1a. |
|
||||
| 6 | KEEP-list legends / hints / empty states still render | **PASS** | `/browse`: both empty states plus the "Right-click a tag … Add to subscription panel" hint. `/alarms`: the "Cleared alarms are not retained …" legend under the table. `/galaxy`: Object Categories / Top Templates empty states. |
|
||||
| 7 | Density with realistic data; sections as separate cards; no horizontal scroll | **PASS** | `/` renders the SQL error truncated to one line with `…`; `/galaxy` renders the same error in full (the drill-down) — the intended split is visible side by side. `/apikeys` with a 3-entry `read_subtrees` constraint wraps inside the column with no table overflow. `documentElement.scrollWidth == clientWidth` on every page walked. Each group is its own card. |
|
||||
|
||||
**Not exercisable on this rig** (recorded, not claimed as passing): the `Sessions` / `Workers` /
|
||||
`Recent Faults` truncation call sites and the `BrowseTreeNodeView` load-error row need a *registered*
|
||||
session, which requires the x86 worker — on macOS the launch fails before the session is ever
|
||||
registered, so those tables stay empty. Their shared helper and the container CSS were both verified
|
||||
live by other means (the `/` Galaxy error for `Abbreviate`, the computed-style probe for
|
||||
`.tree-load-status`). Re-check them on windev the next time that service is redeployed.
|
||||
|
||||
**Side note surfaced, not acted on**: `SettingsPage` renders `LDAP service password` as
|
||||
`[redacted]` in the browser — confirming `GatewayConfigurationProvider`'s substitution end to end.
|
||||
|
||||
## 5. Follow-up gate: `/admin/secrets` delete modal (scadaproj#2)
|
||||
|
||||
Not part of the sweep. Run because the same stale "mxgw is Bootstrap-free" claim corrected in §0 was
|
||||
the stated reason this app was **excused** from the 2026-07-19 family-wide `/admin/secrets` modal
|
||||
sweep: `ConfirmDeleteModal` shipped a bare `class="modal"`, which Bootstrap 5's `.modal{display:none}`
|
||||
made permanently invisible on every Bootstrap host. The exemption's reasoning was false, so mxgw was
|
||||
in scope and its delete modal had never been live-gated. `8f7ee49` bumped `ZB.MOM.WW.Secrets.Ui` to
|
||||
`0.2.3`, which is supposed to carry the fix — but "supposed to" is what the original claim was.
|
||||
|
||||
**Static check** — decoding the UTF-16 literals out of `ZB.MOM.WW.Secrets.Ui.dll` 0.2.3 shows the
|
||||
component now inlines its own `<style>` under a private namespace: `.zb-secrets-modal` with
|
||||
`display: flex; position: fixed; inset: 0; z-index: 1081`, over a `.zb-secrets-modal-backdrop` at
|
||||
`z-index: 1080`. No bare `modal` class.
|
||||
|
||||
**Live check** — local rig, scratch secrets store (`Secrets:SqlitePath` + `ZB_SECRETS_MASTER_KEY`
|
||||
under the session scratchpad), throwaway secret `uisweep/throwaway`, delete armed but **never
|
||||
confirmed**:
|
||||
|
||||
| Probe | Result |
|
||||
|---|---|
|
||||
| Modal root classes | `zb-secrets-modal` — and `document.querySelector('.modal')` is `null`, so Bootstrap's `.modal{display:none}` has nothing to bite |
|
||||
| Modal computed style | `display: flex`, `position: fixed`, `z-index: 1081`, `visibility: visible`, `opacity: 1`, box `1600×827` |
|
||||
| Backdrop | `display: block`, `position: fixed`, `z-index: 1080`, `rgba(15,18,20,.45)`, covers the full viewport |
|
||||
| Confirm button hit-testable | `elementFromPoint` at its centre returns the button itself — not covered by the backdrop |
|
||||
| Cancel → disarm | modal and backdrop both removed; secret still listed; **no delete performed** |
|
||||
|
||||
**PASS.** The `0.2.3` fix holds on this Bootstrap host. The rig was stopped and both throwaway
|
||||
stores (secrets + auth) deleted.
|
||||
</content>
|
||||
|
||||
## 6. Follow-up: `ZB.MOM.WW.Theme` 0.3.1 → 0.4.0 (dependency bump, not UI cleanup)
|
||||
|
||||
0.4.0 upstreams the button-sizing block this sweep installed locally, written against the finding in
|
||||
§1a — the kit sizes **only** through `--bs-btn-*` variables and carries a comment in `layout.css`
|
||||
saying why, so nobody simplifies it back into literals.
|
||||
|
||||
- `ZB.MOM.WW.MxGateway.Server.csproj`: `0.3.1` → `0.4.0`. No `Directory.Packages.props` in this repo;
|
||||
the version lives on the `PackageReference`.
|
||||
- `site.css`: the four `--bs-btn-*` overrides and the whole `.btn-group-sm > .btn, .btn-sm` rule
|
||||
deleted — `layout.css` now supplies them and `<ThemeHead />` emits it before `site.css`.
|
||||
|
||||
**Trimmed, not deleted wholesale.** The local `.btn` rule also carried `border-radius: 5px`,
|
||||
`font-weight: 500`, `white-space: nowrap`, which predate the sweep and which 0.4.0 does **not**
|
||||
ship — it upstreamed sizing only. Removing the block entirely would have silently dropped three
|
||||
app-specific declarations. What remains is `.btn { border-radius: 5px; font-weight: 500;
|
||||
white-space: nowrap; }` — shape, not size. `border-radius` deliberately stays a literal rather than
|
||||
`--bs-btn-border-radius`, because `.btn-sm` redefines that variable and small buttons would shrink
|
||||
to the Bootstrap small radius.
|
||||
|
||||
**Verification**
|
||||
|
||||
- Build: 0 warnings, 0 errors. Suite: 879/879.
|
||||
- Computed-style probe re-run on 0.4.0 with the local block gone: `.btn` **13.6px**, `.btn-sm`
|
||||
**12.48px** — identical to the pre-bump numbers, so the kit rule reaches. `btn-group-sm` members
|
||||
both 12.48px with the seam intact (first member right radius `0px`, last `5px`). Retained
|
||||
declarations confirmed live on both sizes: radius `5px`, weight `500`, `nowrap`.
|
||||
- Cascade confirmed by enumerating `document.styleSheets`: `--bs-btn-font-size` is now declared in
|
||||
exactly two sheets — `bootstrap.min.css` (`1rem` / `0.875rem`) and `layout.css`
|
||||
(`.85rem` / `.78rem`). `site.css` no longer declares it, so the duplicate is gone.
|
||||
|
||||
**Tree-hygiene note.** The first suite run after the bump failed
|
||||
`GatewayTreeHygieneTests.SourceTree_ContainsNoSqliteDatabaseFiles` — unrelated to the bump. The §4
|
||||
rig's *first* start used the default **relative** `Secrets:SqlitePath` (`mxgateway-secrets.db`),
|
||||
which resolved against the server project directory and left a DB in the source tree; only the §5
|
||||
run redirected it to the scratchpad. The file was untracked, was deleted, and the suite went green.
|
||||
The §4/§5 cleanup notes were therefore incomplete as originally written — the scratchpad copies were
|
||||
removed but this one was missed. The hygiene test is what caught it, which is what it exists for.
|
||||
|
||||
## 7. Follow-up: `ZB.MOM.WW.Theme` 0.4.0 → 0.4.1 (pin-only, no rendering change here)
|
||||
|
||||
0.4.1 was published the same day to fix `.rail-btn-block`, a modifier shipped broken in 0.4.0
|
||||
(`display: block; width: auto` fills for an `<a>` but shrink-wraps a `<button>`; now
|
||||
`width: calc(100% - 1.2rem)` with `box-sizing: border-box`, accounting for `.rail-btn`'s side
|
||||
margins). Bumped for family-pin alignment.
|
||||
|
||||
**Why nothing needed re-verifying.** Confirmed rather than assumed:
|
||||
|
||||
- `diff` of the two restored packages: `theme.css` byte-identical; `layout.css` differs **only**
|
||||
inside the `.rail-btn-block` rule and its comment. The `.btn` sizing block is unchanged, so the
|
||||
§6 measurements (`.btn` 13.6px, `.btn-sm` 12.48px) still hold and the probe was not re-run.
|
||||
- This app does not use `rail-btn-block`. It does carry the exact element pair the bug turned on —
|
||||
`MainLayout.razor:32` is a form-submit `<button class="rail-btn">` (Sign Out) and
|
||||
`MainLayout.razor:36` an `<a class="rail-btn">` (Sign In) — but base `.rail-btn` is
|
||||
`display: inline-block`, which shrink-wraps both element types identically. The asymmetry only
|
||||
appears once the block modifier is applied, so this dashboard was never affected.
|
||||
|
||||
**Verification.** Build 0 warnings / 0 errors; suite **879/879**; `staticwebassets.build.json`
|
||||
resolves `zb.mom.ww.theme/0.4.1`. No stale-HTTP-cache clear was needed — restore picked 0.4.1
|
||||
directly.
|
||||
|
||||
## 8. Follow-up: role-gate the side rail's Secrets link (family-wide nav task)
|
||||
|
||||
Requested as a family-wide sweep: every app's UI should link to the Secrets management page, visible
|
||||
to Administrator-role users only.
|
||||
|
||||
**Found state.** The link already existed — `MainLayout.razor`, Admin section, `/admin/secrets`. What
|
||||
did not exist was any gate: the rail rendered every item for every visitor, including a Viewer and
|
||||
the anonymous-localhost read-only identity. The premise that there was an "existing role-gated nav
|
||||
pattern" to follow was false; the rail's only `AuthorizeView` was the footer's signed-in/signed-out
|
||||
split, so this introduces the pattern rather than extending it.
|
||||
|
||||
Not an access hole — the mounted page carries `[Authorize(Policy = "secrets:manage")]`, so a Viewer
|
||||
clicking through was denied. It was a dead link presented as a live one.
|
||||
|
||||
**Gate chosen: the policy, not the role.** `<AuthorizeView Policy="@SecretsAuthorization.ManagePolicy">`,
|
||||
i.e. the same policy the page itself enforces, so nav visibility cannot drift from page access. The
|
||||
sweep asked for a role literal (`DashboardRoles.Admin` = `"Administrator"`), and in this host the two
|
||||
are equivalent: `GatewayOptionsValidator` constrains `Dashboard:GroupToRole` values to
|
||||
`Administrator` or `Viewer`, so the shared library's other manage-granting roles (`secrets-manager`,
|
||||
`secrets-reveal`) are unreachable here. The policy form was preferred because it stays correct if
|
||||
that constraint ever relaxes — a role literal would then hide the link from users who can use the
|
||||
page.
|
||||
|
||||
**Deliberate asymmetry — API Keys stays ungated.** Its sibling item looks like the same case and is
|
||||
not. `ApiKeysPage` renders for a Viewer with write affordances hidden (`@if (CanManageApiKeys)`), so
|
||||
hiding its nav item would remove legitimate read access. The secrets page has no read-only mode. The
|
||||
rule is "gate the link when the page denies the role outright", not "gate everything under Admin".
|
||||
|
||||
**Coverage.** Three tests pin the policy's verdict per principal (Administrator admitted, Viewer
|
||||
refused, unauthenticated refused) in `SecretsNavGateTests`, and `/admin/secrets` joins the canonical
|
||||
route list in `GatewayApplicationTests` — it is the one nav destination mounted from an RCL rather
|
||||
than declared here, so a routing regression could remove it without touching this repo's pages.
|
||||
|
||||
### 8a. Correction: the policy tests could not detect a deleted gate
|
||||
|
||||
The coverage above shipped with a stated rationale — that rendering was disproportionate because the
|
||||
policy verdict "is the part that can actually be wrong". That rationale was wrong, and a review point
|
||||
from the OtOpcUa session identified why: the policy is library code this repo did not author, while
|
||||
the *wiring* is the only thing this change introduced. Worse, the check applies specifically to repos
|
||||
where the link already existed before gating — "an Administrator still sees it" is identical to the
|
||||
pre-change behaviour, so it cannot distinguish a working gate from an inert one. **Only the negative
|
||||
observation proves a gate exists at all.**
|
||||
|
||||
`SecretsNavRenderTests` now renders `MainLayout` through the framework's static `HtmlRenderer` — no
|
||||
component-testing package needed, since the assertion is about emitted markup, not interactivity —
|
||||
and asserts the Secrets item is absent for a Viewer and for an anonymous caller, present for an
|
||||
Administrator, and that the ungated API Keys sibling stays present for a Viewer (so a later
|
||||
"consistency fix" that hides it fails loudly).
|
||||
|
||||
**Confirmed non-vacuous by mutation**, which is the only thing that makes the absence assertions
|
||||
worth anything: with the `AuthorizeView` removed from the layout, `Rail_OmitsSecretsLink_ForViewer`
|
||||
and `Rail_OmitsSecretsLink_ForAnonymous` both go red — **and all three original policy tests stay
|
||||
green**, demonstrating the gap concretely rather than by argument. The Administrator case is retained
|
||||
as the control: without it, a rail that rendered no nav at all would satisfy both absence assertions
|
||||
and the suite would report a working gate over a blank page.
|
||||
|
||||
**Verification.** Build 0 warnings / 0 errors; suite **899/899** (895 + 4).
|
||||
@@ -0,0 +1,310 @@
|
||||
# Deferred-Findings Remediation Implementation Plan
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers-extended-cc:subagent-driven-development to implement this plan task-by-task (Opus implementers per the user's instruction).
|
||||
|
||||
**Goal:** Resolve the six findings the 2026-08-15 perf-review remediation explicitly deferred (`docs/plans/2026-08-15-perf-review-remediation.md:611-620`) plus the pre-existing Windows-only `SecretsStorePathGuardTests` failure, so the deferred table empties and windev returns to a clean 1046/1046 gateway suite.
|
||||
|
||||
**Architecture:** Two phases. Phase A is gateway-side (net10, fully verifiable on macOS): the secrets-test fix, the distributor dictionary swap, event-path iterator flattening, and the dashboard in-process refactor that removes the Blazor pages' loopback SignalR hop while preserving the idle gate, mirror viewer gating, and clone-then-redact invariants. Phase B is worker-side (net48 x86, verified on windev over ssh): control-frame completion decoupling in the two-class frame writer, pipe-read teardown restructuring, and value-cache clone removal per the completed aliasing audit.
|
||||
|
||||
**Tech Stack:** .NET 10 / ASP.NET Core / Blazor Server / System.Threading.Channels (gateway); .NET Framework 4.8 x86 (worker); xUnit; windev CI clone `C:\build\mxaccessgw-ci` via `ssh windev`.
|
||||
|
||||
**Branch:** `perf/deferred-remediation` off local `main` (`15f188e`).
|
||||
|
||||
---
|
||||
|
||||
## Ground rules for every implementer subagent
|
||||
|
||||
- Shared working tree at `/Users/dohertj2/Desktop/MxAccessGateway`. **NEVER run `git stash`, `git reset`, `git clean`, `git checkout <sha/branch>`, or any command that touches files outside your task's `Files:` list.** Commit with explicit pathspecs only (`git add <your files> && git commit`).
|
||||
- Build/test lock: before `dotnet build` or `dotnet test`, acquire the lock with `mkdir /private/tmp/claude-501/-Users-dohertj2-Desktop-MxAccessGateway/f36938ae-bbca-4245-b5c9-fac512d69e22/scratchpad/buildlock` (retry loop with sleep until it succeeds); `rmdir` it in ALL exit paths.
|
||||
- `TreatWarningsAsErrors=true`, `Nullable=enable` repo-wide. Follow `docs/style-guides/CSharpStyleGuide.md`: file-scoped namespaces, `sealed` by default, `Async` suffix, MXAccess-aligned names.
|
||||
- Worker projects (`ZB.MOM.WW.MxGateway.Worker*`) are net48/x86 and DO NOT COMPILE on macOS. For Phase B tasks: edit carefully, self-review for net48 compatibility (target-typed `new` and file-scoped namespaces ARE valid — `LangVersion=latest`; but no `Span`-based BCL overloads, no `IAsyncDisposable` on BCL types, `Channel` comes from System.Threading.Channels package which the worker already references). Compilation and tests happen at the Task 13 windev gate.
|
||||
- Update affected docs in the same commit as the source (repo rule), except the dashboard design doc which Task 8 consolidates (deliberate, to avoid parallel edits to one file).
|
||||
- MXAccess parity: never synthesize events, never mutate an event already handed to the outbound queue or wire.
|
||||
|
||||
---
|
||||
|
||||
## Phase A — gateway (macOS-verifiable)
|
||||
|
||||
### Task 1: Windows-safe cleanup in SecretsStorePathGuardTests
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~3 min
|
||||
**Parallelizable with:** Task 2, Task 3, Task 4
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Tests/Configuration/SecretsStorePathGuardTests.cs`
|
||||
- Modify: `docs/GatewayTesting.md` (lines ~557-565, the "fails deterministically on Windows" note)
|
||||
|
||||
**Why:** `CreateBuilder_AcceptsSecretsStoreOutsideContentRoot_AndCreatesIt` (lines 84-105) fails deterministically on Windows: `GatewayApplication.CreateBuilder` migrates the secrets store through `SecretsSqliteConnectionFactory` (`Pooling = true`, WAL), disposal returns the connection to the Microsoft.Data.Sqlite pool with the native handle open, and the `finally`'s `Directory.Delete(directory, recursive: true)` (line 103) hits a sharing violation. macOS passes only because Unix unlinks open files. The repo fixes this pattern twice already: `TestSupport/../TempDatabaseDirectory.cs:57` and `Configuration/PreHostSecretExpansionTests.cs:130-153`.
|
||||
|
||||
**Spec:**
|
||||
1. In the failing test's `finally`, before `Directory.Delete`: call `Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();` and wrap the delete in `try { ... } catch (IOException) { } catch (UnauthorizedAccessException) { }` (best-effort, mirroring `TempDatabaseDirectory.Dispose`). Add a comment mirroring the one in `PreHostSecretExpansionTests.cs:133-137` (WAL + pooling keeps the handle alive past dispose).
|
||||
2. Leave the rejection test alone (the guard means its file is never created).
|
||||
3. Update `docs/GatewayTesting.md`: replace the "subtract it from the expected pass count on Windows" paragraph with a short note that the test's cleanup now clears the SQLite pool first and the failure is fixed as of this branch.
|
||||
|
||||
**Steps:** edit → `dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~SecretsStorePathGuardTests"` (expect 2/2 on macOS; the real proof is the Task 13 windev gate) → commit `fix(tests): clear the SQLite pool before deleting the secrets path-guard temp dir — Windows sharing violation`.
|
||||
|
||||
---
|
||||
|
||||
### Task 2: SessionEventDistributor `_subscribers` → plain `Dictionary`
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~3 min
|
||||
**Parallelizable with:** Task 1, Task 3, Task 4
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs`
|
||||
|
||||
**Why:** All five access sites (`:365`, `:555`, `:771`, `:799`, `:813`) are inside `lock (_lifecycleLock)`; the lock-free hot path reads the copy-on-write `_subscriberSnapshot` array (`:958`, `:302`), never the dictionary. The concurrent type buys nothing. Audit confirmed no external/reflection access.
|
||||
|
||||
**Spec:** Change the field at `:107` to `Dictionary<long, Subscriber>`; `TryRemove(subscriber.Id, out _)` at `:799` becomes `Remove(subscriber.Id)`. Reword the type remarks at `:69-80`, `:111-123`, and `:298-300` where they name `ConcurrentDictionary` by design — the invariant to state is now: "the dictionary is only ever touched under `_lifecycleLock`; lock-free readers use `_subscriberSnapshot`."
|
||||
|
||||
**Steps:** edit → `dotnet test ... --filter "FullyQualifiedName~SessionEventDistributorTests"` (29 facts, expect all green) → commit `refactor(sessions): _subscribers to plain Dictionary — every access is under _lifecycleLock`.
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Merge the session event-source pass-through iterator
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Task 1, Task 2, Task 4
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Sessions/GatewaySession.cs` (`MapWorkerEventsAsync` ~:767-776, `ReadEventsAsync` ~:1517-1530)
|
||||
- Test: `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/GatewaySessionTests.cs` (existing)
|
||||
|
||||
**Why:** The worker→distributor source chain nests three compiler-generated async iterators per event: `WorkerClient.ReadEventsCoreAsync` → `GatewaySession.ReadEventsAsync` (pure pass-through: `TouchClientActivity(); yield return`) → `GatewaySession.MapWorkerEventsAsync` (`yield return mapper.MapEvent(...)`). The pass-through layer is two extra `MoveNextAsync` state-machine hops per event for no semantic value.
|
||||
|
||||
**Spec:**
|
||||
1. FIRST grep all callers of `ReadEventsAsync`. If `MapWorkerEventsAsync` is its only caller, inline it: `MapWorkerEventsAsync` calls `GetReadyWorkerClientAsync`, iterates `client.ReadEventsAsync(ct)` directly, calls `TouchClientActivity()` per event, and `yield return mapper.MapEvent(workerEvent)`. Delete `ReadEventsAsync`. If other callers exist, keep the method for them but make `MapWorkerEventsAsync` self-contained as above — do NOT change any caller outside this file; report the finding.
|
||||
2. Behavior must be byte-identical: same activity-touch cadence (per event), same exception propagation (WorkerClientException flows to the distributor pump unchanged), no event synthesis, worker order preserved.
|
||||
3. `WorkerClient.ReadEventsCoreAsync`'s single-reader claim (`_eventsReaderClaimed`) must still be exercised exactly once per attach — do not add a second call site.
|
||||
|
||||
**Steps:** grep callers → edit → `dotnet test ... --filter "FullyQualifiedName~GatewaySession"` and `--filter "FullyQualifiedName~SessionEventDistributorTests"` → commit `perf(sessions): fold the ReadEventsAsync pass-through into MapWorkerEventsAsync — one fewer iterator per event`.
|
||||
|
||||
---
|
||||
|
||||
### Task 4: EventStreamService direct channel reads in the live loop
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Task 1, Task 2, Task 3
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Grpc/EventStreamService.cs`
|
||||
- Test: `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/EventStreamServiceTests.cs` (existing 17 facts — must pass unchanged)
|
||||
|
||||
**Why:** The subscriber-side live loop materializes `subscriber.Reader.ReadAllAsync(ct).GetAsyncEnumerator(ct)` (`:109-111`) — a BCL async-iterator wrapper costing a state-machine hop per event on the hottest gateway path. Direct `ChannelReader` consumption (`WaitToReadAsync` + drain-with-`TryRead`) removes it.
|
||||
|
||||
**Spec:**
|
||||
1. Replace the enumerator with direct reads: `while (await reader.WaitToReadAsync(ct)) { while (reader.TryRead(out MxEvent? mxEvent)) { ...existing per-event body... } }`; loop ends when `WaitToReadAsync` returns false (channel completed).
|
||||
2. EVERY invariant in the current body survives, verbatim where possible:
|
||||
- ReplayGap sentinel emitted exactly once, first, only when `replayGap` (`:133-139`) — untouched, it precedes the live loop.
|
||||
- Replay batch stitching (`:141-150`) — untouched.
|
||||
- Per-RPC dedup watermark `if (mxEvent.WorkerSequence <= afterWorkerSequence) continue;` (`:179-182`) — must apply to every live event.
|
||||
- `WorkerClientException` catch → `session.MarkFaulted` → metrics → rethrow (`:164-174`): a completed-with-exception channel surfaces its exception from `WaitToReadAsync` — the catch must wrap the wait/read, preserving identical fault classification. Terminal `SessionManagerException(EventQueueOverflow)` propagates unchanged.
|
||||
- `finally` ordering (`:192-200`): with no enumerator to dispose, the remaining order is backlog-gauge registration disposal → lease disposal → `metrics.StreamDisconnected("Detached")`. Keep the comments explaining why.
|
||||
3. Cancellation: `WaitToReadAsync(ct)` throws `OperationCanceledException` on detach — must reach the same code path the enumerator's cancellation did (the gRPC layer treats it as client disconnect). Verify against `StreamEventsAsync_WhenCanceled_DetachesSubscriber`.
|
||||
4. No public-surface change; `MxAccessGatewayService` (`:151-179`) is untouched.
|
||||
|
||||
**Steps:** edit → run the full `EventStreamServiceTests` class + `GatewayEndToEndReconnectReplayTests` + `GatewayEndToEndMultiSubscriberTests` → commit `perf(grpc): consume the subscriber channel directly in StreamEventsAsync — drops the ReadAllAsync iterator hop`.
|
||||
|
||||
---
|
||||
|
||||
### Task 5: In-process dashboard snapshot feed + page switch
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~8 min (accepted overage; splitting further would split one invariant)
|
||||
**Parallelizable with:** Task 6, Task 7
|
||||
|
||||
**Files:**
|
||||
- Create: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/IDashboardSnapshotFeed.cs`
|
||||
- Create: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotFeed.cs`
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/DashboardPageBase.cs`
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardServiceCollectionExtensions.cs` (one `AddSingleton` line)
|
||||
- Create: `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotFeedTests.cs`
|
||||
|
||||
**Why:** Eight pages inherit `DashboardPageBase` and each opens a loopback `/hubs/snapshot` HubConnection (`DashboardPageBase.cs:62`) — a WebSocket round trip back into the same process per circuit. `IDashboardSnapshotService.WatchSnapshotsAsync` exists but is NOT multicast (each enumeration = its own `PeriodicTimer` + snapshot build), so pages must not call it directly; a shared feed does one enumeration and fans out.
|
||||
|
||||
**Spec:**
|
||||
1. `IDashboardSnapshotFeed` (singleton): `IAsyncEnumerable<DashboardSnapshot> WatchAsync(CancellationToken ct)`. Internally: per-subscriber `Channel<DashboardSnapshot>` with capacity 1 and `BoundedChannelFullMode.DropOldest` (a dashboard viewer only ever wants the latest snapshot; a slow circuit must never buffer unboundedly or stall others).
|
||||
2. **Idle gating (the invariant this task must not lose):** the feed enumerates `IDashboardSnapshotService.WatchSnapshotsAsync` on a background task started when the subscriber count goes 0→1 and cancelled when it goes 1→0. While zero subscribers, the feed holds no timer and builds no snapshot. Guard subscriber add/remove with a plain lock; restart cleanly on resubscribe (mirror the start/stop discipline of `GatewayAlarmMonitor.StreamAsync` registration, `GatewayAlarmMonitor.cs:739-752`). If the underlying watch throws or completes, complete all subscriber channels with the error and reset so the next subscriber restarts it (mirror `DashboardSnapshotPublisher.ExecuteAsync`'s reconnect-after-delay posture, but per-feed).
|
||||
3. `DashboardPageBase`: remove the HubConnection path (`:62` and the factory usage); keep the synchronous first render via `snapshotService.GetSnapshot()` (`:37`); then a background loop `await foreach (var s in feed.WatchAsync(_cts.Token)) { Snapshot = s; await InvokeAsync(StateHasChanged); }` started in `OnAfterRenderAsync(firstRender)` or `OnInitializedAsync` (match current lifecycle), cancelled + awaited in `DisposeAsync`. Update the class XML doc that narrates the hub subscription history (`:7-14`).
|
||||
4. Hubs, `DashboardSnapshotPublisher`, `DashboardSnapshotHubConnectionCounter`, `DashboardHubConnectionFactory`, and `/hubs/token` all stay — they remain the remote/external surface. Do not touch them.
|
||||
5. Auth: the pages are mapped behind `ViewerPolicy` (`DashboardEndpointRouteBuilderExtensions.cs:136`), which remains the gate for in-process consumption; add one comment on `WatchAsync` saying so.
|
||||
6. Tests (`DashboardSnapshotFeedTests`): (a) zero subscribers → underlying service's `WatchSnapshotsAsync` never enumerated (fake service counts enumerations/`MoveNextAsync`); (b) first subscriber starts exactly one enumeration; two subscribers share it; (c) last unsubscribe cancels it; resubscribe restarts it; (d) slow subscriber observes latest-wins (push 3 snapshots, read 1, it is the newest) while a fast subscriber sees all; (e) underlying fault completes subscribers with the error and a fresh subscriber restarts.
|
||||
|
||||
**Steps:** write feed tests first (fail) → implement feed → page switch → `dotnet test ... --filter "FullyQualifiedName~DashboardSnapshotFeed"` then `--filter "FullyQualifiedName~Dashboard"` (whole dashboard test folder) → commit `feat(dashboard): in-process snapshot feed replaces the pages' loopback /hubs/snapshot hop`.
|
||||
|
||||
---
|
||||
|
||||
### Task 6: In-process session event subscription + SessionDetailsPage switch
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~8 min
|
||||
**Parallelizable with:** Task 5, Task 7
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardEventBroadcaster.cs`
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHubViewerRegistry.cs` (only if a member is needed for synthetic connection ids; prefer reusing the existing API)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Pages/SessionDetailsPage.razor` (the `/hubs/events` connection at `:271,297`)
|
||||
- Test: `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardEventBroadcasterTests.cs` (extend)
|
||||
|
||||
**Why:** `SessionDetailsPage` opens a loopback `/hubs/events` connection. The broadcaster already short-circuits on `!viewerRegistry.HasViewers(sessionId)` BEFORE the redaction deep clone (`DashboardEventBroadcaster.cs:51-56`) — the mirror viewer gating shipped last round. An in-process subscription must keep feeding that registry or every unwatched session pays `MxEvent.Clone()` per event again.
|
||||
|
||||
**Spec:**
|
||||
1. Add to `DashboardEventBroadcaster` an in-process subscribe API: `IDashboardEventSubscription Subscribe(string sessionId)` returning a disposable that exposes `ChannelReader<MxEvent> Reader` (bounded, capacity ~256, `DropOldest` — this is a UI mirror, loss is acceptable and already documented for the hub path). On subscribe: register a synthetic connection id (e.g. `"inproc-" + Guid.NewGuid().ToString("N")`) with `EventsHubViewerRegistry.AddViewer(connectionId, sessionId)`; on dispose: `RemoveViewer` + `ReleaseConnection` in the order the hub uses (`EventsHub.cs:86,99`). Registry stays the single source of truth for `HasViewers`.
|
||||
2. `Publish` (`:39-86`): after the existing `HasViewers` check and the clone-then-redact (`RedactValues` `:97-109`), `TryWrite` the SAME redacted clone to each in-process subscriber of that session, in addition to the hub group send. The source `MxEvent` is shared with the gRPC stream and replay ring — the existing never-mutate-in-place rule holds; in-process subscribers receive the redacted clone only.
|
||||
3. `SessionDetailsPage`: replace the HubConnection + `SubscribeSession` invoke with `broadcaster.Subscribe(SessionId)` and a read loop marshalling to the renderer via `InvokeAsync(StateHasChanged)`; dispose the subscription in `DisposeAsync`. Keep the existing per-session ACL posture (any Viewer may watch any session — SEC-25 is tracked separately; do not widen or narrow it here).
|
||||
4. Tests to add in `DashboardEventBroadcasterTests`: (a) in-process subscriber receives the redacted event when `ShowTagValues=false` and the source event is not mutated; (b) subscribing flips `HasViewers` so `Publish` stops short-circuiting (proves mirror gating integration); (c) disposing the last in-process subscriber restores the no-viewers short-circuit (no clone, no send — reuse the existing `Publish_WithNoRegisteredViewers_DoesNotCloneOrSend` fake pattern); (d) hub viewers and in-process viewers are independently counted.
|
||||
|
||||
**Steps:** tests first → implement → `dotnet test ... --filter "FullyQualifiedName~DashboardEventBroadcaster"` + `--filter "FullyQualifiedName~EventsHubViewerRegistry"` + `--filter "FullyQualifiedName~GatewaySessionDashboardMirror"` → commit `feat(dashboard): in-process session event subscription feeds the viewer registry — SessionDetailsPage drops its /hubs/events hop`.
|
||||
|
||||
---
|
||||
|
||||
### Task 7: AlarmsPage provider-status via IGatewayAlarmService
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** Task 5, Task 6
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Pages/AlarmsPage.razor` (`:194` HubConnection, `:281-304` poll loop untouched)
|
||||
|
||||
**Why:** `AlarmsPage` opens `/hubs/alarms` but only consumes `ProviderStatus` payloads from it (alarm rows come from the 3 s `QueryAlarmsAsync` poll). `IGatewayAlarmService.StreamAsync` (`GatewayAlarmMonitor.cs:724-777`) is already a true multi-subscriber in-process fan-out.
|
||||
|
||||
**Spec:** Replace the HubConnection with a background loop over `alarmService.StreamAsync(alarmFilterPrefix: null, ct)`, handling only `PayloadOneofCase.ProviderStatus` (skip snapshot/live alarm payloads — the poll stays authoritative for rows). The monitor's drop policy completes a lagging subscriber's channel (`:700-712`): on completion or fault, delay ~1 s and resubscribe (matching the hub path's `WithAutomaticReconnect` posture). Dispose via the page's existing cancellation. Leave the poll loop alone.
|
||||
|
||||
**Steps:** edit → `dotnet build src/ZB.MOM.WW.MxGateway.Server` → `dotnet test ... --filter "FullyQualifiedName~Alarms"` → commit `feat(dashboard): AlarmsPage reads provider status from IGatewayAlarmService in-process`.
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Dashboard design-doc update (consolidated)
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** none (runs after 5, 6, 7 land)
|
||||
|
||||
**Files:**
|
||||
- Modify: `docs/GatewayDashboardDesign.md` (sections at ~:112-114, :162-178, :190-217, :228-247, :535-541, :581-595)
|
||||
|
||||
**Spec:** Rewrite the affected sections to describe: pages consume in-process seams (`IDashboardSnapshotFeed`, `DashboardEventBroadcaster.Subscribe`, `IGatewayAlarmService.StreamAsync`); the three hubs and `/hubs/token` remain as the remote/external surface; idle gating is now two-tier (hub connection counter gates the hub publisher; feed subscriber count gates the in-process pump — while nobody watches, neither builds a snapshot); mirror gating counts hub viewers AND in-process viewers through the one registry; clone-then-redact still happens once in the broadcaster before any delivery; ViewerPolicy on the component endpoint is the in-process auth gate; SEC-25 per-session ACL gap unchanged. Present tense, why-not-what, no marketing.
|
||||
|
||||
**Commit:** `docs(dashboard): in-process page feeds, two-tier idle gating, hubs as the external surface`
|
||||
|
||||
---
|
||||
|
||||
### Task 9: Phase A gate — full gateway suite on macOS
|
||||
|
||||
**Classification:** trivial (verification only)
|
||||
**Parallelizable with:** none (after Tasks 1-8)
|
||||
|
||||
Run `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx` (expect 0 warnings) and the full `dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj` (expect ≥1046 passed, 0 failed; new feed/broadcaster tests raise the count). Fix-forward any failure before Phase B.
|
||||
|
||||
---
|
||||
|
||||
## Phase B — worker (net48 x86, verified on windev)
|
||||
|
||||
### Task 10: Control-frame completion decoupling in WorkerFrameWriter
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~6 min
|
||||
**Parallelizable with:** Task 11, Task 12
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWriter.cs`
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerFrameProtocolTests.cs`
|
||||
- Modify: `docs/WorkerFrameProtocol.md` (~:120-131 completion-semantics paragraph)
|
||||
|
||||
**Why:** Wire ordering is already correct — `DequeueNext` (`:383-413`) re-checks `_controlFrames` before every frame. The coupling is completion latency: `DrainQueuedFramesAsync` (`:304-361`) defers the single `FlushAsync` and ALL `TrySetResult` calls to after the whole drain pass, so a heartbeat/command-reply/fault/shutdown-ack `Task` awaited by its writer does not resolve until up to 128 event frames behind it are written and flushed. The XML docs claim "never delayed behind an event backlog" — true of bytes, false of the awaited completion.
|
||||
|
||||
**Spec:**
|
||||
1. Record the priority class on `PendingFrame` (`:23-48`), set at construction in `WriteAsync` (`:109`) and `WriteBatchAsync` (`:192`).
|
||||
2. In `DrainQueuedFramesAsync`: when `DequeueNext` returns an `Event` frame while `written` contains one or more not-yet-completed `Control` frames, first `FlushAsync` + complete + clear `written`, then continue draining. Exit-path flush at `:339-360` unchanged. Net effect: a control frame's completion never waits on an event frame dequeued after it; the pure-event 128-batch hot path still pays exactly one flush (guarded by the existing `WriteAsync_WhenBatchDrainedTogether_FlushesOnce` and `EventBurst_DrainLoopCoalescesFlushes`); a pure-control burst still pays one flush. Do NOT flush per control frame unconditionally — that reintroduces the pre-WRK-12 syscall-per-heartbeat cost.
|
||||
3. Failure handling: `FailFrames(written, ...)` / `FailAllQueued` (`:327-336`) operate on the current `written` list; after an early flush+complete+clear, frames already completed must not be failable — verify the clear ordering makes that structurally true, and extend the fault-injection tests if the early-flush path adds a new failure window (a `FlushAsync` fault with a partially-completed pass).
|
||||
4. New test (use the existing `GatedWriteStream` harness ~`:880`): queue a control frame behind N gated event frames within one drain pass; assert the control frame's `WriteAsync` task completes before the last event write is released. Keep all 9 existing writer tests green — sequence stamping (`:431-483`), claim/tombstone interlock (`:244-274`), and wire order must be untouched.
|
||||
5. `docs/WorkerFrameProtocol.md`: update the completion-semantics paragraph — completion now resolves at the class-transition flush, still meaning "written AND flushed".
|
||||
|
||||
**Commit:** `perf(worker): control-frame completions resolve at the class-transition flush, not after the event batch`
|
||||
|
||||
---
|
||||
|
||||
### Task 11: Worker pipe-read teardown — dispose-to-unblock and observe the abandoned read
|
||||
|
||||
**Classification:** high-risk
|
||||
**Estimated implement time:** ~8 min
|
||||
**Parallelizable with:** Task 10, Task 12
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs` (`RunMessageLoopAsync` `:267-310`, ctor `:55-68`)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeClient.cs` (`:143-159`) — only if ownership must move; prefer not
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs`
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameReader.cs` (comment only)
|
||||
|
||||
**Why:** On net48, `NamedPipeClientStream.ReadAsync` ignores its `CancellationToken` (`WorkerFrameReader.cs:109-111`). Fault-path exits (event-drain fault, oversized event, watchdog, heartbeat write failure) leave `readTask` pending; it is unblocked only when `WorkerPipeClient`'s `using` disposes the pipe, at which point it faults with `ObjectDisposedException`/`IOException` on a Task nobody observes (the finally at `:303-309` awaits only heartbeat and drain). The frame-pooling change (GWC-30) makes this sharper: the abandoned read owns the per-instance `_lengthPrefix` buffer and possibly a rented ArrayPool payload — the reader's single-consumer invariant holds today only because nothing ever reads again after abandonment.
|
||||
|
||||
**Spec — constraints, implementer designs within them:**
|
||||
1. **No unobserved faulted Task.** After the stream is disposed, `readTask`'s fault must be awaited/observed (reuse `ObserveBackgroundTaskStopAsync`'s timeout-and-log shape, `:312-348`) before `WorkerPipeClient.RunAsync` returns.
|
||||
2. **Ordering: final writes complete before disposal.** The shutdown ack (`WriteShutdownAckAsync` `:1064-1069`) and fault frames (`TryWriteFaultAsync` `:1164+`) are written after the message loop exits on some paths — trace every exit path and place the stream disposal AFTER the last possible write on each. The clean design: `WorkerPipeSession` keeps a reference to the ctor `Stream`; `RunAsync`'s outermost finally (after runtime-session disposal and any fault write, `:133-145`) disposes the stream and then observes `readTask` (stored in a field by `RunMessageLoopAsync`). `WorkerPipeClient`'s `using` then double-disposes harmlessly. If the trace shows a fault write that happens in `WorkerPipeClient` after `session.RunAsync` returns (there is none known), fall back to moving observation into `WorkerPipeClient`.
|
||||
3. **Never a second read.** After abandonment, no code path may call `_reader.ReadAsync` again (pooled-buffer use-after-return). The message loop already guarantees this (`return` before reassignment on the graceful path); keep it structurally true and assert it in a comment on `_lengthPrefix` (`WorkerFrameReader.cs:23-25`).
|
||||
4. **Graceful path unchanged:** `WorkerShutdown`/`ShutdownWorker` exits have no pending read; disposal+observation must be a no-op there (observe a completed/absent task).
|
||||
5. Document the net48 token-ignoring fact where the read is issued (`RunMessageLoopAsync` and/or `ReadExactlyOrThrowAsync`) — the research found zero comments acknowledging it.
|
||||
6. Tests (net48 project, real `PipePair` harness `:2433-2485`): (a) fault-path exit (reuse the `RunAsync_EventFrameTooLarge_...` shape `:868`) — assert `RunAsync` completes within the existing 5 s bound AND, via a `TaskScheduler.UnobservedTaskException` hook armed in the test with a forced GC, that no unobserved exception leaks; (b) graceful shutdown still completes with no pending read; (c) the session disposes the stream (harness observes the gateway-side stream faulting its own pending read promptly rather than at `PipePair.Dispose`).
|
||||
|
||||
**Commit:** `fix(worker): session-owned stream disposal unblocks and observes the net48 pipe read at teardown`
|
||||
|
||||
---
|
||||
|
||||
### Task 12: Value-cache clone removal per the aliasing audit
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Task 10, Task 11
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessValueCache.cs` (`Set` `:82,83,97`; `CachedValue` `:275`)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessValueCacheTests.cs` (rewrite the `:58` test)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessCommandExecutorTests.cs` (add cached-read test)
|
||||
|
||||
**Why (audit result):** All three clones in `Set` — `Value.Clone()` (deep, recursive for arrays), `SourceTimestamp.Clone()`, `Statuses.Clone()` (container + N proxies) — are removable. The event is fully stamped BEFORE `Set` runs (`Enqueue` at `MxAccessBaseEventSink.cs:263` precedes `postPublish` at `:288`; sequence/timestamp stamped inside `Enqueue`, `MxAccessEventQueue.cs:269-270`) and the queue's ownership invariant forbids later mutation. The alias already exists on the read side: `SucceededRead` (`MxAccessSession.cs:1086,1091,1096`) hands the cache's own `Value`/`SourceTimestamp` instances into every `BulkReadResult`, which downstream only wraps and serializes. Worker↔gateway is a process boundary — no gateway consumer can alias.
|
||||
|
||||
**Spec:**
|
||||
1. Remove all three clones; `CachedValue` stores the event's own references.
|
||||
2. Ownership contract comment on `Set` and on `CachedValue`: the cache holds borrowed references into an enqueued, write-once `MxEvent`; consumers may read and serialize, never mutate; mutation would additionally invalidate `QueuedEvent.Size` — the enqueue-time memoized serialized size that the byte-budgeted `Drain` charges (`MxAccessEventQueue.cs:499-506`), so a grown message could overshoot the negotiated frame max and fault the session via `MessageTooLarge`.
|
||||
3. Rewrite `Set_StoresIndependentSnapshot_UnaffectedByLaterEventMutation` (`:58` — it codifies the invariant being reversed) into the aliasing contract: `Set` then `TryGet` returns the same `Value`/`SourceTimestamp`/`Statuses`-element instances (`Assert.Same`), with the doc comment explaining the write-once borrow.
|
||||
4. Add the missing cached-read-path test in `MxAccessCommandExecutorTests`: seed the cache, dispatch a `ReadBulk` that hits `TryGetCachedReadFor` → assert `WasCached == true` and `result.Value` is reference-equal to the cached instance (closing the coverage gap the audit found — nothing today exercises `WasCached == true` end-to-end in the worker).
|
||||
5. `MxAccessWriteCompletionCache.Record`'s parallel `statuses.Clone()` (`:76`) is left AS-IS deliberately (different lifecycle, not in the finding) — add one cross-reference comment there pointing at the value-cache ownership contract.
|
||||
|
||||
**Commit:** `perf(worker): value cache borrows the write-once event's instances — three clones per OnDataChange removed`
|
||||
|
||||
---
|
||||
|
||||
### Task 13: Phase B gate — windev full verification
|
||||
|
||||
**Classification:** trivial (verification only)
|
||||
**Parallelizable with:** none (after Tasks 10-12; Phase A gate must be green)
|
||||
|
||||
Push the branch to origin, then on windev (`ssh windev`, clone `C:\build\mxaccessgw-ci`): fetch + checkout the branch; `dotnet build src/ZB.MOM.WW.MxGateway.slnx` (0 warnings); `dotnet build src/ZB.MOM.WW.MxGateway.Worker/... -p:Platform=x86`; `dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/... -p:Platform=x86` (expect 501+ passed, 0 failed — new tests raise the count); `dotnet test src/ZB.MOM.WW.MxGateway.Tests/...` (expect **0 failed including SecretsStorePathGuardTests** — the Task 1 proof). Known caveat: the reconnect-replay test is load-sensitive on windev; re-run isolated before treating it as a regression (documented in `docs/GatewayTesting.md`).
|
||||
|
||||
---
|
||||
|
||||
### Task 14: Wrap-up — deferred table closure, docs sweep, final review
|
||||
|
||||
**Classification:** small
|
||||
**Parallelizable with:** none (last)
|
||||
|
||||
- Append a closure note to `docs/plans/2026-08-15-perf-review-remediation.md`'s deferred table (one line: resolved by this plan, date, branch).
|
||||
- Sweep: `gateway.md` / `docs/WorkerFrameProtocol.md` / `docs/GatewayDashboardDesign.md` / `docs/GatewayTesting.md` consistency with as-built behavior; record any accepted deviations in THIS plan's "As-built notes" section (add it).
|
||||
- Update `.tasks.json` statuses; update auto-memory (`perf-remediation-branch.md` or successor) with the branch state.
|
||||
- Dispatch the final integration code review (Opus) over `git diff main..perf/deferred-remediation` before reporting done. Merge remains the user's decision.
|
||||
|
||||
---
|
||||
|
||||
## Explicitly out of scope
|
||||
|
||||
| Item | Why |
|
||||
|---|---|
|
||||
| wnwrap alarm GUID identity semantics; `ALARM_RECORDS/@COUNT` probe | Need live alarms on windev — external state this plan cannot provide. Still tracked in the prior plan's follow-ups. |
|
||||
| Structural alarm-truncation degraded-status signal | Contract-level design (proto change candidate) — separate effort. |
|
||||
| SEC-25 per-session dashboard event ACL | Security roadmap item; Task 6 deliberately preserves the current posture. |
|
||||
| `MxAccessWriteCompletionCache` clone | Different lifecycle than the value cache; consciously kept (Task 12.5). |
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"planPath": "docs/plans/2026-08-15-deferred-remediation.md",
|
||||
"tasks": [
|
||||
{ "id": 1, "subject": "Task 1: Windows-safe cleanup in SecretsStorePathGuardTests", "status": "completed" },
|
||||
{ "id": 2, "subject": "Task 2: SessionEventDistributor _subscribers to plain Dictionary", "status": "completed" },
|
||||
{ "id": 3, "subject": "Task 3: Merge the session event-source pass-through iterator", "status": "completed" },
|
||||
{ "id": 4, "subject": "Task 4: EventStreamService direct channel reads in the live loop", "status": "completed" },
|
||||
{ "id": 5, "subject": "Task 5: In-process dashboard snapshot feed + page switch", "status": "completed" },
|
||||
{ "id": 6, "subject": "Task 6: In-process session event subscription + SessionDetailsPage switch", "status": "completed" },
|
||||
{ "id": 7, "subject": "Task 7: AlarmsPage provider-status via IGatewayAlarmService", "status": "completed" },
|
||||
{ "id": 8, "subject": "Task 8: Dashboard design-doc update (consolidated)", "status": "completed", "blockedBy": [5, 6, 7] },
|
||||
{ "id": 9, "subject": "Task 9: Phase A gate — full gateway suite on macOS", "status": "completed", "blockedBy": [1, 2, 3, 4, 5, 6, 7, 8] },
|
||||
{ "id": 10, "subject": "Task 10: Control-frame completion decoupling in WorkerFrameWriter", "status": "completed", "blockedBy": [9] },
|
||||
{ "id": 11, "subject": "Task 11: Worker pipe-read teardown — dispose-to-unblock and observe", "status": "completed", "blockedBy": [9] },
|
||||
{ "id": 12, "subject": "Task 12: Value-cache clone removal per the aliasing audit", "status": "completed", "blockedBy": [9] },
|
||||
{ "id": 13, "subject": "Task 13: Phase B gate — windev full verification", "status": "pending", "blockedBy": [10, 11, 12] },
|
||||
{ "id": 14, "subject": "Task 14: Wrap-up — deferred table closure, docs sweep, final review", "status": "pending", "blockedBy": [13] }
|
||||
],
|
||||
"lastUpdated": "2026-08-15T00:00:00Z"
|
||||
}
|
||||
@@ -0,0 +1,627 @@
|
||||
# Performance Review Remediation Implementation Plan
|
||||
|
||||
> **For Claude:** REQUIRED SUB-SKILL: Use superpowers-extended-cc:executing-plans to implement this plan task-by-task (or superpowers-extended-cc:subagent-driven-development when executing in-session).
|
||||
|
||||
**Goal:** Resolve every actionable finding from the 2026-08-15 architectural performance review — six High findings, the Medium tier, and the worthwhile Low/hygiene items — without changing any MXAccess parity behavior or public contract.
|
||||
|
||||
**Architecture:** Two phases. Phase A is gateway-side (.NET 10, builds and tests locally on macOS via `NonWindows.slnx`); Phase B is worker-side (.NET Framework 4.8 x86, which does **not** compile on this Mac — Phase B tasks are edited here and verified in one consolidated pass on the windev box via the `psbridge` skill, Task 24). No `.proto` changes anywhere in this plan, so no client regeneration is needed. All work happens on branch `perf/review-remediation`.
|
||||
|
||||
**Tech Stack:** ASP.NET Core gRPC, System.Threading.Channels, SignalR, Microsoft.Data.Sqlite, .NET Framework 4.8 STA/COM interop, protobuf (Google.Protobuf).
|
||||
|
||||
---
|
||||
|
||||
## Ground rules for every implementer (read before your task)
|
||||
|
||||
- **Build gate:** `TreatWarningsAsErrors=true`, `Nullable=enable`, analyzers at latest. New warnings fail the build — fix them, never suppress.
|
||||
- **Style:** follow `docs/style-guides/CSharpStyleGuide.md` — file-scoped namespaces, `sealed` by default, `Async` suffix, MXAccess-aligned names. Match the comment density and idiom of the file you're editing.
|
||||
- **Parity is sacred:** do not change MXAccess-visible semantics (event ordering, `OperationComplete` behavior, write-completion reply shape, per-tag ReadBulk timeout meaning). These tasks change *mechanics* (waits, locks, allocations), never observable protocol behavior, except where a task explicitly says otherwise.
|
||||
- **Never synthesize events.** Nothing in this plan may fabricate an `MxEvent`.
|
||||
- **Docs in the same commit:** when a task changes configuration, event mechanics, security behavior, or lifecycle rules, the named docs must be updated in that task's commit.
|
||||
- **Worker code (Phase B) does not compile on this machine.** `LangVersion=latest` applies, so modern syntax is fine, but only net48-era BCL APIs exist (no `Span`-taking stream overloads, no `ArgumentNullException.ThrowIfNull` — check what the file already uses). Match the existing worker idioms exactly. Verification is Task 24.
|
||||
- **Tests:** gateway tests use the FakeWorkerHarness (`src/ZB.MOM.WW.MxGateway.Tests`), no MXAccess needed. Run only your task's filter, not the full suite (full suite runs once per phase).
|
||||
- **Commit after every task**, message style: `perf(<area>): <what>` (or `fix(...)` for the two correctness bugs).
|
||||
|
||||
Verification commands used throughout:
|
||||
|
||||
```bash
|
||||
# Gateway build (macOS-safe)
|
||||
dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx
|
||||
# Targeted gateway tests
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~<TestClass>"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
# Phase A — Gateway (local verification)
|
||||
|
||||
### Task 1: Named-pipe buffer sizes
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~3 min
|
||||
**Parallelizable with:** Tasks 2, 3, 4, 5, 6, 8, 10, 11, 12, 13, 14
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionWorkerClientFactory.cs` (`CreatePipe`, ~line 157)
|
||||
- Modify: `docs/WorkerFrameProtocol.md` (add a short "Pipe buffers" note)
|
||||
- Test: `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/` (existing factory/e2e tests must stay green; no new test — buffer size isn't observable through the .NET API)
|
||||
|
||||
**Why:** the current 5-arg `NamedPipeServerStream` overload passes `inBufferSize: 0, outBufferSize: 0`. A zero-quota byte-mode pipe forces every write to rendezvous with a pending read — lock-step IPC, and the exact failure class behind the historical windev suite wedge.
|
||||
|
||||
**Step 1: Change the overload**
|
||||
|
||||
```csharp
|
||||
private const int PipeBufferSizeBytes = 128 * 1024;
|
||||
|
||||
private static NamedPipeServerStream CreatePipe(string pipeName)
|
||||
{
|
||||
return new NamedPipeServerStream(
|
||||
pipeName,
|
||||
PipeDirection.InOut,
|
||||
maxNumberOfServerInstances: 1,
|
||||
PipeTransmissionMode.Byte,
|
||||
PipeOptions.Asynchronous,
|
||||
inBufferSize: PipeBufferSizeBytes,
|
||||
outBufferSize: PipeBufferSizeBytes);
|
||||
}
|
||||
```
|
||||
|
||||
Add a comment stating *why* (zero-quota rendezvous behavior; reference the windev wedge). Note: on Unix these sizes are advisory (Unix domain socket), which is fine — the fix targets Windows production.
|
||||
|
||||
**Step 2:** `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx` → 0 errors.
|
||||
**Step 3:** `dotnet test ... --filter "FullyQualifiedName~GatewayEndToEndFakeWorkerSmokeTests"` → PASS.
|
||||
**Step 4:** Update `docs/WorkerFrameProtocol.md` with a 3–4 line "Pipe buffers" paragraph. Commit: `perf(ipc): give worker pipes real OS buffers instead of zero-quota rendezvous`
|
||||
|
||||
---
|
||||
|
||||
### Task 2: Metrics — pull-gauge for worker queue depth, lock-free command counters
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Tasks 1, 3, 4, 5, 6, 8, 10, 11, 12, 13, 14 (NOT Task 7 — both edit `WorkerClient.cs`)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetrics.cs` (`SetWorkerEventQueueDepth` ~290; `CommandStarted/Succeeded/Failed` ~202–247; gauge wiring ~91; snapshot ~461–492)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs` (call sites ~303 and ~602)
|
||||
- Test: `src/ZB.MOM.WW.MxGateway.Tests/Metrics/` (extend the existing GatewayMetrics test class)
|
||||
|
||||
**Why:** `SetWorkerEventQueueDepth` takes the process-wide `_syncRoot` twice per event for every session, and the single scalar makes the gauge last-writer-wins across sessions (a correctness bug). The command counters take the same global lock 2–3× per RPC.
|
||||
|
||||
**Step 1 (failing test):** add a test that registers two worker-queue-depth sources reporting 3 and 4 and asserts the snapshot/gauge reports 7; add a test that `CommandStarted`×N from parallel tasks yields exactly N with no lock (behavioral: just correctness of count).
|
||||
|
||||
**Step 2 (implement):**
|
||||
- Mirror the existing GWC-15 pattern verbatim: add `RegisterWorkerEventQueueDepthSource(Func<int> depth)` returning an `IDisposable` handle, a `ConcurrentDictionary<long, Func<int>>` of sources, and make `GetWorkerEventQueueDepth` sum the sources (clamp negatives). Delete `SetWorkerEventQueueDepth` and the `_workerEventQueueDepth` field.
|
||||
- `WorkerClient`: at construction (or first use), register a source returning its staged+channel depth via `Volatile.Read` of a field the stage/consume paths maintain with `Interlocked` — the hot path does **no** metrics call at all anymore. Dispose the registration in `DisposeAsync`.
|
||||
- Command counters: `_commandsStarted/_commandsSucceeded/_commandsFailed` become `long` updated with `Interlocked.Increment`; `_commandFailuresByMethod` becomes `ConcurrentDictionary<string, long>` (follow the existing `EventReceived` pattern in the same file). Snapshot reads with `Interlocked.Read`.
|
||||
|
||||
**Step 3:** run the Metrics test filter → PASS. **Step 4:** grep the repo for `SetWorkerEventQueueDepth` — zero hits outside tests you updated.
|
||||
**Step 5:** Commit: `perf(metrics): pull-model worker queue gauge (fixes last-writer-wins), Interlocked command counters`
|
||||
|
||||
---
|
||||
|
||||
### Task 3: Distributor — copy-on-write subscriber snapshot
|
||||
|
||||
**Classification:** high-risk (core event fan-out concurrency)
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Tasks 1, 2, 4, 5, 6, 8, 10, 11, 12, 13, 14
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs` (pump loop ~600; register/unregister paths; the "snapshot-free enumerator" remark ~71)
|
||||
- Test: `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/` (existing SessionEventDistributor tests must stay green; add one test if a register-during-pump race test doesn't already exist)
|
||||
|
||||
**Why:** `_subscribers.Values` (the property) locks the whole `ConcurrentDictionary` and materializes a snapshot list **per event**, contradicting the adjacent comment.
|
||||
|
||||
**Step 1 (implement):** maintain a `volatile Subscriber[] _subscriberSnapshot` rebuilt inside the existing registration lock on every register/unregister (the set is tiny and mutates rarely). The pump iterates the array. Keep the dictionary if other paths use keyed lookup; the array is purely the fan-out view. Update the ~71 remark to describe the actual mechanism. Semantics to preserve exactly: a subscriber registered mid-iteration may miss the in-flight event ("late subscribers see events after they register") — the array snapshot preserves this naturally.
|
||||
|
||||
**Step 2:** run the distributor/replay test filters (`FullyQualifiedName~SessionEventDistributor`, `~Replay`) → PASS. The replay-handoff atomicity tests are the critical gate here.
|
||||
**Step 3:** Commit: `perf(events): copy-on-write subscriber snapshot in fan-out pump`
|
||||
|
||||
---
|
||||
|
||||
### Task 4: Dashboard event mirror — viewer gating
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Tasks 1, 2, 3, 5, 6, 8, 10, 11, 12, 13, 14
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHub.cs`
|
||||
- Create: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHubViewerRegistry.cs`
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardEventBroadcaster.cs` (Publish, ~39)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardServiceCollectionExtensions.cs` (register the registry)
|
||||
- Modify: `docs/GatewayDashboardDesign.md` (mirror gating paragraph)
|
||||
- Test: create `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/EventsHubViewerRegistryTests.cs` + extend the existing DashboardEventBroadcaster tests
|
||||
|
||||
**Why:** with `ShowTagValues=false` (default), `Publish` deep-clones every event and dispatches to a SignalR group that is empty in the steady state. No viewer gate exists anywhere on the path.
|
||||
|
||||
**Step 1 (failing test):** broadcaster with zero registered viewers for the session performs **no clone and no send** (assert via a counting fake hub-clients/`IHubContext` seam, matching however the existing broadcaster tests fake SignalR); with one viewer, behavior is unchanged (redacted clone sent).
|
||||
|
||||
**Step 2 (implement):**
|
||||
- `EventsHubViewerRegistry` (singleton): `ConcurrentDictionary<string, int>` session→viewer count, `Increment(sessionId)`, `Decrement(sessionId)`, `HasViewers(sessionId)`. Track per-connection subscribed sessions in a `ConcurrentDictionary<string, ConcurrentDictionary<string,byte>>` keyed by connection id so `OnDisconnectedAsync` can decrement everything that connection held.
|
||||
- `EventsHub`: `SubscribeSession`/`UnsubscribeSession` update the registry alongside the group add/remove; override `OnDisconnectedAsync` to release the connection's sessions. Keep the existing SEC-25 remark intact.
|
||||
- `DashboardEventBroadcaster.Publish`: first line after the null-guards becomes `if (!viewerRegistry.HasViewers(sessionId)) { return; }` — before the redact/clone.
|
||||
- Do **not** attempt lazy mirror-lease start in this task (it interacts with distributor lifecycle); the gate above removes ~all of the waste already. Note this decision in the doc paragraph.
|
||||
|
||||
**Step 3:** run Dashboard test filter → PASS. **Step 4:** update `docs/GatewayDashboardDesign.md`. Commit: `perf(dashboard): gate event mirror on live viewers — no clone, no send for unwatched sessions`
|
||||
|
||||
---
|
||||
|
||||
### Task 5: Snapshot pipeline — idle gating, cached config, keyed refresh cadence
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Tasks 1, 2, 3, 4, 6, 8, 10, 11, 12, 13, 14
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotPublisher.cs` (~69–83)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHub.cs` (connection counting)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs` (~103 config rebuild, ~163–164 + ~267 API-key refresh)
|
||||
- Modify: `docs/GatewayDashboardDesign.md`
|
||||
- Test: extend existing snapshot service/publisher tests under `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/`
|
||||
|
||||
**Why:** the 1 Hz tick runs an API-key SQLite read, a registry sort, a metrics snapshot, and a rebuild of the *static* effective-configuration record, broadcast to `Clients.All`, forever, with zero viewers.
|
||||
|
||||
**Step 1 (failing tests):** (a) effective configuration object is reference-identical across two snapshot builds; (b) API-key summaries refresh at most once per configured interval (inject `TimeProvider`, follow the file's existing time idiom); (c) publisher with zero connections does not enumerate the snapshot source (fake the hub context; count pulls).
|
||||
|
||||
**Step 2 (implement):**
|
||||
- Cache `EffectiveGatewayConfiguration` in a field on first build (it's startup-static; add a comment saying so).
|
||||
- `RefreshApiKeySummariesAsync`: skip unless `RefreshInterval` (new private constant, 15 s) has elapsed since the last successful refresh.
|
||||
- `DashboardSnapshotHub`: `OnConnectedAsync`/`OnDisconnectedAsync` maintain an `int` connection count on a small singleton (or reuse the Task 4 registry class with a well-known key — implementer's choice, keep it simple). Publisher checks the count each tick: zero connections → `await Task.Delay(interval)` and skip both the snapshot build and the broadcast. First connection after idle gets a fresh snapshot on its next tick (≤1 interval of staleness — acceptable; pages also seed from `IDashboardSnapshotService` directly on load).
|
||||
|
||||
**Step 3:** dashboard test filter → PASS. Docs paragraph. Commit: `perf(dashboard): idle-gate the snapshot tick; cache static config; bound key-list refresh`
|
||||
|
||||
---
|
||||
|
||||
### Task 6: Reply ownership transfer in `MapCommandReply`
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~3 min
|
||||
**Parallelizable with:** Tasks 1, 2, 3, 4, 5, 8, 10, 11, 12, 13, 14
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGrpcMapper.cs` (~74)
|
||||
- Test: existing mapper/service tests under `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/`
|
||||
|
||||
**Why:** every `WorkerCommandReply` is parsed fresh from one pipe frame and completed to exactly one awaiter; the gRPC handler is its only consumer. Events already got this treatment under GWC-07 — replies still deep-copy, which doubles the largest hot-path message on bulk reads.
|
||||
|
||||
**Step 1 (verify precondition, in-code):** confirm (grep) that no caller of `WorkerClient.InvokeAsync` retains `reply.Reply` after mapping — the review found the Invoke path clean; `GatewayAlarmMonitor` and `DashboardLiveDataService` own their separate replies. If you find a second consumer, STOP and surface it — that's a plan defect.
|
||||
|
||||
**Step 2 (implement):** `return reply.Reply.Clone();` → `return reply.Reply;` with a GWC-07-style ownership comment: the worker reply object is single-consumer by construction (one frame → one `PendingCommand` completion → one mapper call); the mapper transfers ownership to the gRPC response.
|
||||
|
||||
**Step 3:** run `FullyQualifiedName~MxAccessGrpcMapper` + the fake-worker smoke filter → PASS. Commit: `perf(grpc): transfer reply ownership instead of deep-cloning every worker reply`
|
||||
|
||||
---
|
||||
|
||||
### Task 7: WorkerClient — pooled-timer timeout, single sizing pass, `WorkerCancel` on timeout
|
||||
|
||||
**Classification:** high-risk (IPC concurrency + protocol behavior)
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Tasks 3, 4, 5, 8, 10, 11, 12, 13, 14 (NOT Task 2 — both edit `WorkerClient.cs`; run after Task 2)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs` (InvokeAsync ~226–270; timeout path)
|
||||
- Modify: `docs/GatewayProcessDesign.md` (command timeout → cancel-forwarding note)
|
||||
- Test: extend `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/` worker-client tests (the fakes there already exercise timeout paths)
|
||||
|
||||
**Why:** each Invoke churns a linked CTS + `Task.Delay` timer + `WhenAny`; `CalculateSize` runs twice (protobuf doesn't memoize); and on timeout the gateway never tells the worker, so a timed-out COM call keeps occupying the STA and an envelope still queued gets written anyway.
|
||||
|
||||
**Step 1 (failing test):** on command timeout, the client enqueues a `WorkerCancel` envelope carrying the timed-out correlation id (assert via the fake connection's written-frame log).
|
||||
|
||||
**Step 2 (implement):**
|
||||
- Replace the CTS/Delay/WhenAny block with `await pendingCommand.Task.WaitAsync(timeout, cancellationToken)` wrapped in a `try/catch (TimeoutException)` / `(OperationCanceledException)` mapping to the exact same `WorkerClientErrorCode`s and messages as today (tests depend on them).
|
||||
- On the timeout path, after `RemovePendingCommandAsFailed`, best-effort enqueue a `WorkerCancel` envelope for the correlation id (fire-and-forget with a swallow-and-log; never let cancel failure mask the timeout exception). The worker already handles `WorkerCancel` (`WorkerPipeSession` → `CancelCommand`).
|
||||
- Thread the already-computed `envelopeSize` into the frame write path if the writer API allows passing a known size; if the writer's public surface would have to change more than trivially, skip this sub-item and leave a `// PERF:` note — the timer and cancel fixes carry the task.
|
||||
|
||||
**Step 3:** worker-client test filter → PASS, including existing timeout tests unchanged. Docs note. Commit: `perf(ipc): WaitAsync command timeouts + forward WorkerCancel so a timed-out COM call frees the STA`
|
||||
|
||||
---
|
||||
|
||||
### Task 8: Audit pipeline — startup bootstrap, background writer, retention
|
||||
|
||||
**Classification:** high-risk (security/audit semantics)
|
||||
**Estimated implement time:** ~5 min (split if it runs long: 8a writer, 8b retention)
|
||||
**Parallelizable with:** Tasks 1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Audit/SqliteCanonicalAuditStore.cs` (per-op `EnsureTableAsync` ~52–54, ~94, ~131–136)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Audit/CanonicalAuditWriter.cs` (~35)
|
||||
- Create: `src/ZB.MOM.WW.MxGateway.Server/Security/Audit/ChannelAuditWriter.cs` (bounded channel + hosted drain)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/AuthStoreServiceCollectionExtensions.cs` (DI wiring + hosted service)
|
||||
- Modify: `docs/DesignDecisions.md` (audit is asynchronous best-effort, bounded, with retention)
|
||||
- Test: create `src/ZB.MOM.WW.MxGateway.Tests/Security/Audit/ChannelAuditWriterTests.cs`
|
||||
|
||||
**Why:** constraint denials await a SQLite insert inline per denied tag inside bulk RPC loops — sequential round-trips into the same DB file the auth store uses, each with a redundant `CREATE TABLE IF NOT EXISTS`, into a table with no retention.
|
||||
|
||||
**Step 1 (failing tests):** (a) `WriteAsync` returns without touching the store (enqueue-only) and the event lands in the store shortly after (drain); (b) when the bounded channel (capacity 4096) is full, `WriteAsync` drops (oldest or newest — pick drop-write/newest for simplicity) and increments a counter, never blocks; (c) retention sweep deletes rows older than the configured window.
|
||||
|
||||
**Step 2 (implement):**
|
||||
- `ChannelAuditWriter : ICanonicalAuditWriter` (or whatever the current writer interface is named — read `CanonicalAuditWriter.cs` first): bounded `Channel<CanonicalAuditEvent>` (`BoundedChannelFullMode.DropWrite`), a `BackgroundService` drain that batches up to 64 events into one transaction per drain pass. The audit contract is already documented best-effort — say so in the class doc.
|
||||
- Table bootstrap: run `EnsureTableAsync` once from the drain service's `StartAsync` (and from the store's first list call via a `Lazy`/latch); remove the per-insert and per-list calls.
|
||||
- Retention: in the same drain service, once per hour, `DELETE FROM audit_event WHERE timestamp < now - RetentionDays` (new `SecurityOptions`/audit option, default 90 days, validated ≥1 in `GatewayOptionsValidator`); document in `docs/GatewayConfiguration.md`.
|
||||
- Wire DI so `ConstraintEnforcer.RecordDenialAsync` transparently goes through the channel writer — **no signature changes** at the enforcer/service layer.
|
||||
- Flush-on-shutdown: drain the channel in `StopAsync` with a 2 s cap.
|
||||
|
||||
**Step 3:** audit test filter + `FullyQualifiedName~ConstraintEnforcer` → PASS. Docs (`DesignDecisions.md`, `GatewayConfiguration.md`). Commit: `perf(audit): bounded async audit writer with batched inserts, one-time bootstrap, retention sweep`
|
||||
|
||||
---
|
||||
|
||||
### Task 9: Parallel session teardown in sweep and shutdown
|
||||
|
||||
**Classification:** high-risk (lifecycle concurrency)
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** Tasks 10, 11, 12, 13, 14 (edits only `SessionManager.cs` + docs; run any time)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs` (`CloseExpiredLeasesAsync` ~256–296, `ShutdownAsync` ~301–329)
|
||||
- Modify: `docs/Sessions.md` (teardown parallelism note)
|
||||
- Test: extend `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/` session-manager tests
|
||||
|
||||
**Why:** both loops `await CloseSessionCoreAsync` strictly sequentially, each bounded by the 10 s worker-shutdown timeout — a mass expiry with hung workers stalls slot reclamation, and 50-session shutdown exceeds any host stop-timeout.
|
||||
|
||||
**Step 1 (failing test):** two sessions whose fake worker shutdowns each take T complete a sweep in ~T, not ~2T (the fake harness supports delayed shutdown; if not, add a delay knob to the fake).
|
||||
|
||||
**Step 2 (implement):** wrap both loops in `Parallel.ForEachAsync` with `MaxDegreeOfParallelism = 4` (named constant, comment why: bounded so a mass expiry can't stampede worker teardown). `TryBeginCloseIfExpired` already makes per-session close idempotent/exclusive — state that in a comment; that's the invariant making this safe. Preserve the existing sweep precedence (lease-expiry → faulted → detach-grace) by keeping the *selection* phase sequential and parallelizing only the close calls on the selected set.
|
||||
|
||||
**Step 3:** session-manager filter → PASS. Docs. Commit: `perf(sessions): bounded-parallel teardown in lease sweep and shutdown`
|
||||
|
||||
---
|
||||
|
||||
### Task 10: Dashboard live-data subscription cap
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** Tasks 1–9, 11, 12, 13, 14
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardLiveDataService.cs` (~61–70, `_subscribed`)
|
||||
- Modify: `docs/GatewayDashboardDesign.md`
|
||||
- Test: extend existing live-data tests under `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/`
|
||||
|
||||
**Why:** every tag any viewer ever inspected stays advised on the shared worker session forever.
|
||||
|
||||
**Step 1 (failing test):** subscribing tag #257 when the cap is 256 unsubscribes the least-recently-read tag first (assert the fake session sees an `UnsubscribeBulk`/equivalent for the evicted tag).
|
||||
|
||||
**Step 2 (implement):** replace `_subscribed` (set) with an LRU: `Dictionary<string, LinkedListNode<string>>` + `LinkedList<string>` under the existing `_gate` (already serialized — no new locking). Cap at 256 (named constant; comment the sizing rationale: one browse page of tags plus headroom). On read of an already-subscribed tag, move to front. On insert past cap, evict from the back and call the session's unsubscribe for the evicted batch. On `InvalidateSession`, clear both structures (existing behavior).
|
||||
|
||||
**Step 3:** dashboard filter → PASS. Docs. Commit: `perf(dashboard): LRU cap on the shared live-read session's advised set`
|
||||
|
||||
---
|
||||
|
||||
### Task 11: Alarm monitor — cached `CurrentAlarms` projection
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~3 min
|
||||
**Parallelizable with:** Tasks 1–10, 12, 13, 14
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Alarms/GatewayAlarmMonitor.cs` (~90–99 + every mutation site under `_sync`)
|
||||
- Test: extend `src/ZB.MOM.WW.MxGateway.Tests/Alarms/` monitor tests
|
||||
|
||||
**Why:** `CurrentAlarms` clones the full alarm set under the broadcast lock on every call.
|
||||
|
||||
**Step 1 (failing test):** two consecutive `CurrentAlarms` calls with no intervening transition return the same cached array instance; a transition invalidates it.
|
||||
|
||||
**Step 2 (implement):** add `private IReadOnlyList<ActiveAlarmSnapshot>? _currentAlarmsCache;` — `CurrentAlarms` builds it (still cloning, still under `_sync`) only when null; every mutation path that touches the alarm dictionary (`ApplyTransition`, reconcile apply, clear) nulls it under `_sync`. Callers already treat the result as read-only.
|
||||
|
||||
**Step 3:** alarms filter → PASS. Commit: `perf(alarms): memoize CurrentAlarms projection, invalidate on mutation`
|
||||
|
||||
---
|
||||
|
||||
### Task 12: Request-logging middleware — hoisted logger, bearer redaction fix
|
||||
|
||||
**Classification:** small (contains a security fix)
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** Tasks 1–11, 13, 14
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GatewayRequestLoggingMiddlewareExtensions.cs` (~29–38)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GatewayLogRedactor.cs` (~54–77)
|
||||
- Test: extend `src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/` redactor tests
|
||||
|
||||
**Why:** `CreateLogger` (factory lock + DI resolve) per request; and — the security half — `RedactClientIdentity` passes any bearer credential that doesn't contain `mxgw_` through **unredacted** into log scope, violating the "never log secrets" convention.
|
||||
|
||||
**Step 1 (failing test):** `RedactClientIdentity("Bearer eyJhbGciOi...")` (a non-mxgw token) returns a redacted form (e.g. `Bearer [redacted]`), never the raw token. Keep the existing mxgw-shaped redaction (`mxgw_<id>_***`) intact — those tests must still pass.
|
||||
|
||||
**Step 2 (implement):**
|
||||
- Redactor: any `authorization`-style value that is not recognized as an mxgw key redacts to a fixed `"[redacted]"` (preserve scheme word only). This is fail-closed.
|
||||
- Middleware: resolve the `ILogger` once outside the per-request lambda (category-keyed, not request-keyed) via the app's `ILoggerFactory` at `Use...` registration time; keep the scope construction as-is (it carries per-request fields the log pipeline consumes — do not conditionalize it on log level in this task; note as considered-and-skipped since scope consumers may be added at runtime).
|
||||
|
||||
**Step 3:** diagnostics filter → PASS. Commit: `fix(logging): fail-closed bearer redaction; hoist per-request logger creation`
|
||||
|
||||
---
|
||||
|
||||
### Task 13: Auth-path hygiene — span token parse, limiter partition keys
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** Tasks 1–12, 14
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcAuthorizationInterceptor.cs` (~153 `TryResolveKeyId`)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/CachingApiKeyVerifier.cs` (~229 `TryParseKeyId`)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs` (~244–269, ~398)
|
||||
- Test: existing auth tests under `src/ZB.MOM.WW.MxGateway.Tests/Security/` must stay green; add parse-equivalence cases
|
||||
|
||||
**Step 1 (failing test):** parse-equivalence table test: for a set of tokens (well-formed, missing `_`, empty, extra `_`), the new span parser returns exactly what `Split('_')` logic returned.
|
||||
|
||||
**Step 2 (implement):** replace `Split('_')` in both parsers with `IndexOf('_')` twice over a `ReadOnlySpan<char>`/string (no arrays, no substrings until the final key-id slice). In the limiter, compute the composite partition key once per RPC and pass it to both `Check` and `Reset` (or add an overload taking the precomputed key) instead of concatenating twice.
|
||||
|
||||
**Step 3:** security filter → PASS. Commit: `perf(auth): allocation-free token parsing; single partition-key build per RPC`
|
||||
|
||||
---
|
||||
|
||||
### Task 14: Bulk constraint loops, caches, and per-call hygiene
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Tasks 1–13 (NOT Task 6 if the mapper edit collides — it doesn't; different files)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs` (bulk loops ~466–troughs at 494/551/612/680; double session resolve ~104/126)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ConstraintEnforcer.cs` (~214–215 LINQ; expose `HasReadConstraints`/`HasWriteConstraints` if not present)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/GatewayApiKeyIdentityMapper.cs` (~39–42 cache cliff)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Sessions/SparseArrayExpander.cs` (~123–126 capacity hints)
|
||||
- Test: existing constraint/service tests under `src/ZB.MOM.WW.MxGateway.Tests/` + one new eviction test
|
||||
|
||||
**Step 1 (failing test):** constraint-blob cache: inserting entry `MaxCachedConstraintBlobs + 1` evicts the oldest instead of refusing to cache (FIFO like `GalaxyGlobMatcher` — copy its idiom).
|
||||
|
||||
**Step 2 (implement):**
|
||||
- Bulk loops: hoist a single `identity has no read/write constraints` check before each per-item loop → unconstrained keys take an O(1) fast path (no per-item async interface dispatch, no denial bookkeeping allocation).
|
||||
- Glob matching: replace the two `.Any(lambda)` calls with `for` loops over the glob lists.
|
||||
- Denied-path double clone: build the filtered command directly (new message, copy allowed entries in) instead of `command.Clone()` then clear-and-refill; `MapCommand`'s own clone stays (that one is the load-bearing no-aliasing copy).
|
||||
- Session double-resolve: add/`use` a `SessionManager` overload accepting the already-resolved `GatewaySession` (or have the service pass the session it resolved); keep the not-found exception behavior identical.
|
||||
- `SparseArrayExpander`: set `RepeatedField.Capacity = length` (per element type) before the fill loops.
|
||||
|
||||
**Step 3:** run `FullyQualifiedName~ConstraintEnforcer`, `~MxAccessGatewayService`, `~SparseArray` filters → PASS. Commit: `perf(grpc): O(1) unconstrained bulk fast path, direct filtered-command build, cache eviction, capacity hints`
|
||||
|
||||
---
|
||||
|
||||
### Task 15: Phase A gate — full gateway suite
|
||||
|
||||
**Classification:** trivial (verification only)
|
||||
**Estimated implement time:** ~5 min wall (suite runtime)
|
||||
**Parallelizable with:** none (runs after Tasks 1–14)
|
||||
|
||||
Run, in order:
|
||||
|
||||
```bash
|
||||
dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj
|
||||
```
|
||||
|
||||
Expected: 0 build errors, full suite green, clean process exit (0 surviving testhost). Fix anything red before Phase B. Commit only if fixes were needed.
|
||||
|
||||
---
|
||||
|
||||
# Phase B — Worker (.NET Framework 4.8; verified on windev in Task 24)
|
||||
|
||||
> Phase B implementers: you cannot compile. Be conservative — minimal diffs, match file idioms, net48 BCL only. Every task here lands as an unverified commit that Task 24 builds and tests remotely; keep commits clean so a failure bisects trivially.
|
||||
|
||||
### Task 16: Event drain loop — wake signal instead of 25 ms poll
|
||||
|
||||
**Classification:** high-risk (event path liveness)
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Tasks 18, 19, 21, 22, 23
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs` (add wake handle; `Enqueue` sets it)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs` (~18 `EventDrainInterval`, ~345–372 drain loop)
|
||||
- Modify: `docs/MxAccessWorkerInstanceDesign.md` (drain-loop paragraph ~381)
|
||||
- Test: extend `src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/` event-queue tests + `Ipc/` pipe-session tests (they run on windev)
|
||||
|
||||
**Why:** the drain loop polls at 25 ms with no wake from `Enqueue` — a 25 ms latency floor on every burst from idle, 40 wakeups/s per idle worker, and less burst absorption before the 10k queue faults the session.
|
||||
|
||||
**Implement:**
|
||||
- `MxAccessEventQueue`: add a `SemaphoreSlim _signal = new(0, 1)` (or an `AsyncAutoResetEvent`-shaped helper if the codebase has one — check first). `Enqueue` releases it (cap at 1, swallow `SemaphoreFullException`). Expose `Task WaitForEventsAsync(TimeSpan timeout, CancellationToken ct)`.
|
||||
- Drain loop: when a drain returns empty, `await queue.WaitForEventsAsync(EventDrainInterval, ct)` instead of `Task.Delay` — the 25 ms becomes a *fallback* ceiling, not the floor; a signaled wait returns immediately. Loop structure otherwise unchanged (fault handling, batch size).
|
||||
- Doc paragraph: drain is signal-driven with a 25 ms fallback tick.
|
||||
- Tests: enqueue-after-idle results in a drain without waiting for the fallback interval (windev-run; write it now).
|
||||
|
||||
Commit: `perf(worker): signal-driven event drain — removes the 25 ms latency floor and idle wakeups`
|
||||
|
||||
---
|
||||
|
||||
### Task 17: Event queue capacity — launcher-configurable
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Tasks 18, 19, 21, 22, 23 (NOT Task 16 — both edit `MxAccessEventQueue.cs`/`WorkerPipeSession.cs`; run after 16)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Configuration/WorkerOptions.cs` (+`EventQueueCapacity`, default 10000) and `GatewayOptionsValidator.cs` (≥1000, ≤1_000_000)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerProcessLauncher.cs` (new env var, mirror the `WorkerWriteCompletionWaitEnvironmentVariableName` pattern at ~25–29 and ~186–187 exactly)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Bootstrap/WorkerOptionsParser.cs` / `WorkerOptions.cs` / `EnvironmentVariableWorkerEnvironment.cs` (read it, following the write-completion variable's path)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs` (~52: pass capacity to `new MxAccessEventQueue(...)`)
|
||||
- Modify: `docs/GatewayConfiguration.md` (+`MxGateway:Worker:EventQueueCapacity`), `docs/MxAccessWorkerInstanceDesign.md` (capacity paragraph ~371)
|
||||
- Test: gateway side — validator test + launcher env-var test (these run locally); worker side — parser test (windev)
|
||||
|
||||
**Why:** the 10,000 default is headroom-critical (overflow faults the session) but not configurable without a rebuild.
|
||||
|
||||
**Implement:** copy the `WriteCompletionWaitMilliseconds` plumbing end to end under a new name (`MXGW_EVENT_QUEUE_CAPACITY` shaped like the existing variable's naming). Absent/invalid env value → default 10000 (never crash the worker on a bad value; log and default).
|
||||
|
||||
> **As-built note (1358332):** shipped as silent default without logging, matching the alarm-resolver precedent — no `ILogger` is reachable from the static resolve site without new plumbing; the silent fallback is disclosed in `GatewayConfiguration.md`. The Bootstrap parser files listed above were correctly NOT touched — the established env-var pattern reads `Environment.GetEnvironmentVariable` at the resolve site.
|
||||
|
||||
Note the gateway-side files here don't overlap Phase A tasks — safe after Task 15.
|
||||
|
||||
Commit: `perf(worker): launcher-configurable event queue capacity`
|
||||
|
||||
---
|
||||
|
||||
### Task 18: STA completion waits — message-driven, not sleep-polled
|
||||
|
||||
**Classification:** high-risk (STA/pump semantics)
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Tasks 16, 17, 19, 21, 22, 23
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessWriteCompletionCache.cs` (~97–118 wait loop)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessValueCache.cs` (~135–150 wait loop)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Sta/StaMessagePump.cs` (if it doesn't already expose a bounded "pump until signaled or timeout" primitive)
|
||||
- Modify: `docs/MxAccessWorkerInstanceDesign.md`
|
||||
- Test: extend `src/ZB.MOM.WW.MxGateway.Worker.Tests/Sta/` + `MxAccess/` cache tests (windev)
|
||||
|
||||
**Why:** both waits run `pumpStep(); ...; Thread.Sleep(5)` on the STA — during each 5 ms sleep no messages pump, so COM event dispatch stalls in 5 ms bites for up to 1.5 s (writes) / 1 s per tag (ReadBulk).
|
||||
|
||||
**Implement:**
|
||||
- Add a wake to both caches: the update path (`OnWriteComplete` recording a completion / `OnDataChange` recording a value) signals a Win32 auto-reset event (`AutoResetEvent` is fine — it wraps one).
|
||||
- Replace `Thread.Sleep(pollIntervalMs)` with a pump-integrated wait: `MsgWaitForMultipleObjectsEx(1, [waitHandle], remainingMs-capped-at-50, QS_ALLINPUT, MWMO_INPUTAVAILABLE)`; on `WAIT_OBJECT_0 + 1` (message arrived) run `pumpStep()` and re-check; on `WAIT_OBJECT_0` (signaled) re-check the entry immediately. The existing `StaMessagePump`/`StaRuntime` already use exactly this Win32 pattern (~`StaRuntime.cs:255–261`) — reuse/extract their P/Invoke declarations, do not duplicate.
|
||||
- **Semantics unchanged:** timeouts, deadline math, return values, and the unconfirmed-empty-statuses reply shape stay byte-identical. Only the *waiting mechanism* changes: latency to observe a completion drops from ≤5 ms granularity to immediate, and the pump keeps running throughout the wait.
|
||||
- **Do not** change the plain-`Write` completion-wait default in this task. The 1.5 s default is a documented OtOpcUa contract (`MxGateway:Worker:WriteCompletionWaitMilliseconds` is already configurable). Leave a doc note that operators with pure fire-and-forget write workloads can lower it.
|
||||
|
||||
Commit: `perf(worker): message-driven completion waits — the STA pumps continuously while waiting`
|
||||
|
||||
---
|
||||
|
||||
### Task 19: Handle registry — reverse index, cached views, O(1) removals
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Tasks 16, 17, 18, 21, 22, 23
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessHandleRegistry.cs` (`ItemHandles`/`ServerHandles`/`AdviceHandles` properties ~14–26; `RemoveAdviceHandles` ~137–148; `UnregisterServerHandle` ~46–65)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessSession.cs` (`TryGetCachedReadFor` ~988–1000)
|
||||
- Test: extend `src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/` registry tests (windev)
|
||||
|
||||
**Why:** the sorted list properties re-sort and copy the whole table on **every access**, `TryGetCachedReadFor` reads `ItemHandles` once per ReadBulk tag (O(tags × items·log items)), and advice/server removals do full LINQ scans (O(n²) bulk teardown).
|
||||
|
||||
**Implement:**
|
||||
- Reverse index: `Dictionary<long, Dictionary<string, int>>` server→(tagAddress→itemHandle) — or flat `Dictionary<(int,int-packed + tag)>` — maintained on register/unregister. `TryGetCachedReadFor` becomes two dictionary probes (the file's own comment already asks for this).
|
||||
- Cached materialization: memoize each sorted array with a version stamp bumped on any mutation; property returns the cached array when the version matches. Registry is STA-confined (verify: no locking in the file today ⇒ single-threaded by contract — state it in a comment), so no locking needed.
|
||||
- Removals: secondary index advice-by-item (`Dictionary<long, List<advice>>` keyed on the packed `(serverHandle, itemHandle)` the item table already uses) so `RemoveAdviceHandles`/`UnregisterServerHandle` stop scanning.
|
||||
|
||||
Commit: `perf(worker): reverse tag index + memoized views + indexed removals in the handle registry`
|
||||
|
||||
---
|
||||
|
||||
### Task 20: Event conversion — exact-format timestamps, compiled status accessors
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Tasks 16, 17, 18, 19, 21, 22, 23 (different files)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventMapper.cs` (~360–377 timestamp parse)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Conversion/MxStatusProxyConverter.cs` (~96–109 reflection reads)
|
||||
- Test: extend `src/ZB.MOM.WW.MxGateway.Worker.Tests/Conversion/` (windev) — these files have solid existing tests; add exact-format cases
|
||||
|
||||
**Implement:**
|
||||
- Timestamps: try `DateTime.TryParseExact` against a small cached array of the observed MXAccess formats (`M/d/yyyy h:mm:ss.fff tt` and its zero-padded/24 h siblings — derive the list from the existing tests' fixture strings) **first**, falling back to the existing two-stage `TryParse` chain so behavior never regresses on an unexpected locale. Order: exact formats → current-culture → invariant (today's chain).
|
||||
- Status fields: replace the per-read `field.GetValue` with delegates compiled once per field via `Expression.Lambda<Func<object, T>>` (net48-safe) cached alongside the existing `FieldInfo` cache. Same values out, no boxing per event.
|
||||
|
||||
Commit: `perf(worker): exact-format timestamp parse and compiled status-field accessors on the event path`
|
||||
|
||||
---
|
||||
|
||||
### Task 21: Event queue drain — size memoized at enqueue
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~4 min
|
||||
**Parallelizable with:** Tasks 18, 19, 20, 22, 23 (NOT 16/17 — same file; run after them)
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs` (~249–269 byte-budgeted `Drain`; enqueue path ~152–170)
|
||||
- Test: extend the windev event-queue tests: budget math unchanged for a mixed-size batch
|
||||
|
||||
**Why:** `Drain(maxEvents, maxTotalBytes)` calls `CalculateSize()` per event **inside** the queue lock the STA needs to enqueue — a large drain stalls COM callbacks.
|
||||
|
||||
**Implement:** compute `CalculateSize()` once at enqueue time (outside any lock — the caller owns the event exclusively there) and store it on the queue's node/wrapper alongside the event; `Drain` uses the memoized size. The WRK-21 never-strand-the-head guarantee is untouched (same comparisons, precomputed operand). Events are never mutated after enqueue (WRK-11 no-clone contract) so the memoized size cannot go stale — say so in a comment.
|
||||
|
||||
Commit: `perf(worker): memoize event frame size at enqueue; drain stops sizing under the STA's lock`
|
||||
|
||||
---
|
||||
|
||||
### Task 22: Worker frame writer/reader — pooled buffers
|
||||
|
||||
**Classification:** standard
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** Tasks 16, 17, 18, 19, 20, 21, 23
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWriter.cs` (~467 per-frame `new byte[]`)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameReader.cs` (~33 per-frame prefix buffer)
|
||||
- Test: windev `Ipc/` frame tests must stay green (they're thorough — rely on them)
|
||||
|
||||
**Why:** the worker side allocates a fresh frame buffer + prefix buffer per frame while the gateway side already pools (`ArrayPool`, GWC-30) — the fix was applied on one side only. `System.Buffers` is already referenced by the worker (its reader uses `ArrayPool.Shared`).
|
||||
|
||||
**Implement:** mirror the gateway codec: rent the frame buffer from `ArrayPool<byte>.Shared`, write prefix+payload into it, return in a `finally`; hoist the 4-byte prefix buffer to an instance field on the reader (single-reader by contract — copy the gateway reader's comment). Exact same wire bytes.
|
||||
|
||||
Commit: `perf(worker): pooled frame buffers — brings the net48 codec up to the gateway side's GWC-30 pattern`
|
||||
|
||||
---
|
||||
|
||||
### Task 23: Alarm consumer — cheap parse, truncation detection, configurable cadence
|
||||
|
||||
**Classification:** high-risk (alarm correctness)
|
||||
**Estimated implement time:** ~5 min (split 23a parse / 23b truncation+config if long)
|
||||
**Parallelizable with:** Tasks 16, 17, 19, 20, 21, 22
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/WnWrapAlarmConsumer.cs` (~402–437 parse; ~50 `DefaultMaxAlarmsPerFetch`; ~323–330 snapshot rebuild; `ComputeTransitions` absence rule ~356)
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs` (~22 hard-coded 500 ms)
|
||||
- Modify: `docs/GatewayConfiguration.md`, `docs/DesignDecisions.md` (alarm sections)
|
||||
- Test: extend windev `MxAccess/` alarm-consumer tests — the truncation test is the important one
|
||||
|
||||
**Implement (three independent sub-changes):**
|
||||
1. **Parse cost:** in the per-alarm extraction, replace the ~14 `SelectSingleNode(child)` XPath calls with one pass over `alarmNode.ChildNodes` switching on `Name` (same fields, same defaults for absent children). Keep `XmlDocument` (an `XmlReader` rewrite is a bigger change than the win justifies once XPath is gone). Reuse the snapshot dictionary across polls (clear-and-refill → swap two dictionaries) only if trivially safe; otherwise skip — the XPath removal is the payload.
|
||||
2. **Truncation cliff (correctness fix):** when the fetch returns exactly `maxAlarmsPerFetch` records, treat the snapshot as **truncated**: log a warning (rate-limited, identifiers only) and suppress the absence-implies-Clear inference in `ComputeTransitions` for that poll (present alarms still update; nothing is cleared on the evidence of a capped fetch). Add the test: 1024-record fetch + a known alarm missing from it → no Clear transition emitted, warning logged.
|
||||
3. **Cadence + cap configurable:** plumb `MxGateway:Alarms:PollIntervalMilliseconds` (default 500, min 100) and `MaxAlarmsPerFetch` (default 1024) through the existing env-var pattern (as in Task 17). Gateway-side option + validator + launcher env, worker-side parse.
|
||||
|
||||
Commit: `fix(alarms): truncation-safe transitions; perf: single-pass alarm parse; configurable poll cadence`
|
||||
|
||||
---
|
||||
|
||||
### Task 24: Phase B verification on windev (psbridge)
|
||||
|
||||
**Classification:** high-risk (this is the gate for every Phase B commit)
|
||||
**Estimated implement time:** ~10 min wall
|
||||
**Parallelizable with:** none (after all Phase B tasks)
|
||||
|
||||
**Steps:**
|
||||
1. Invoke the `psbridge` skill and follow it (it covers exec/push/deploy against the Windows box).
|
||||
2. Push/pull the branch to windev (whatever the skill's established flow is — the repo has a remote the Windows box shares; `git pull` the branch there).
|
||||
3. On windev, run in order and capture output:
|
||||
```powershell
|
||||
dotnet build src/ZB.MOM.WW.MxGateway.slnx
|
||||
dotnet build src/ZB.MOM.WW.MxGateway.Worker/ZB.MOM.WW.MxGateway.Worker.csproj -p:Platform=x86
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj
|
||||
```
|
||||
4. Any failure: fix on the Mac, commit, re-run the failed leg. Bisect by commit if the failure isn't obvious — Phase B commits are deliberately one-task-each.
|
||||
5. If psbridge is unreachable: STOP and report — Phase B remains "edited, unverified"; do not merge.
|
||||
|
||||
Live MXAccess smoke (`MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`, `WorkerLiveMxAccessSmokeTests`) if provider state is available on windev; otherwise record why skipped, per `docs/GatewayTesting.md`.
|
||||
|
||||
---
|
||||
|
||||
### Task 25: Wrap-up — docs sweep, umbrella index, review deltas
|
||||
|
||||
**Classification:** small
|
||||
**Estimated implement time:** ~5 min
|
||||
**Parallelizable with:** none (last)
|
||||
|
||||
**Files:**
|
||||
- Verify each task's doc edits landed (`gateway.md`, `docs/Sessions.md`, `docs/GatewayConfiguration.md`, `docs/GatewayDashboardDesign.md`, `docs/DesignDecisions.md`, `docs/MxAccessWorkerInstanceDesign.md`, `docs/WorkerFrameProtocol.md`)
|
||||
- Modify: `../scadaproj/CLAUDE.md` — **only if** a fact the umbrella index records changed (new `MxGateway:Worker:EventQueueCapacity` / alarm options are config, not indexed facts; expected outcome: no umbrella change needed — verify, don't assume)
|
||||
- Check: no `.proto` diffs (`git diff main -- '*.proto'` must be empty)
|
||||
|
||||
Commit anything found: `docs: remediation plan doc sweep`
|
||||
|
||||
---
|
||||
|
||||
## Explicitly deferred (decided, not forgotten)
|
||||
|
||||
| Finding | Why deferred |
|
||||
|---|---|
|
||||
| Value-cache triple clone per `OnDataChange` | Removing the defensive copies needs a GWC-07-style aliasing audit across cache consumers; risk outweighs the win until profiled. |
|
||||
| net48 pipe-read cancellation | Benign in practice (worker exits after shutdown); a correct fix means restructuring stream teardown for a path that only fires at exit. |
|
||||
| Control-frame completion coupled to event batch drain | Documented, bounded (≤128 frames) behavior of the two-class writer design; revisit only if heartbeat latency shows up in metrics. |
|
||||
| Blazor pages' loopback SignalR hop | Works correctly; in-process `WatchSnapshotsAsync` consumption is a dashboard refactor with payoff only at viewer counts the product doesn't target. |
|
||||
| Event-path triple async-iterator flattening | LOW-rated; touches the most invariant-dense code in the gateway for two `MoveNextAsync` hops per event. Reconsider after Tasks 3/4 land and if profiling still shows it. |
|
||||
| `SessionEventDistributor._subscribers` `ConcurrentDictionary` → plain `Dictionary` (Task 25 / Task 3 review) | Every mutation is already inside `_lifecycleLock`, so the concurrent type buys nothing. Behavior-neutral refactor with no measurable win, proposed after the windev gate was already green — not worth re-running the verification matrix for. Comment cleanups from the same review landed; this swap did not. |
|
||||
|
||||
## Execution notes for the orchestrator
|
||||
|
||||
- Branch: `git checkout -b perf/review-remediation` before Task 1.
|
||||
- Implementer subagents run on **Opus** per the user's instruction; reviewer chain per each task's Classification.
|
||||
- Parallel dispatch waves (no file overlap): **Wave 1:** 1, 3, 4, 5, 6, 8 · **Wave 2:** 2, 9, 10, 11, 12, 13, 14 · then 7 (after 2) · then 15 · **Wave 3 (Phase B):** 16, 18, 19, 20, 22, 23 · then 17, 21 (after 16) · then 24 · then 25. (Waves are a suggestion; the per-task `Parallelizable with` fields are the contract.)
|
||||
- Each implementer gets: its full task text, the ground rules block, and nothing else — the `Files:` block is the scope contract.
|
||||
@@ -0,0 +1,188 @@
|
||||
{
|
||||
"planPath": "docs/plans/2026-08-15-perf-review-remediation.md",
|
||||
"tasks": [
|
||||
{
|
||||
"id": 1,
|
||||
"subject": "Task 1: Named-pipe buffer sizes",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"subject": "Task 2: Metrics pull-gauge + Interlocked counters",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"subject": "Task 3: Distributor copy-on-write subscriber snapshot",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"subject": "Task 4: Dashboard event mirror viewer gating",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"subject": "Task 5: Snapshot pipeline idle gating + cached config",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"subject": "Task 6: Reply ownership transfer in MapCommandReply",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"subject": "Task 7: WorkerClient WaitAsync timeout + WorkerCancel",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
2
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"subject": "Task 8: Audit pipeline background writer + retention",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"subject": "Task 9: Parallel session teardown",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"subject": "Task 10: Dashboard live-data subscription cap",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"subject": "Task 11: Alarm monitor cached CurrentAlarms",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"subject": "Task 12: Logging middleware hoist + bearer redaction fix",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"subject": "Task 13: Auth-path span parsing + limiter keys",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"subject": "Task 14: Bulk constraint loops, caches, hygiene",
|
||||
"status": "completed"
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"subject": "Task 15: Phase A gate \u2014 full gateway suite",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
4,
|
||||
5,
|
||||
6,
|
||||
7,
|
||||
8,
|
||||
9,
|
||||
10,
|
||||
11,
|
||||
12,
|
||||
13,
|
||||
14
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"subject": "Task 16: Event drain wake signal",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
15
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"subject": "Task 17: Event queue capacity env plumbing",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
16
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 18,
|
||||
"subject": "Task 18: STA message-driven completion waits",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
15
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 19,
|
||||
"subject": "Task 19: Handle registry reverse index + O(1) removals",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
15
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 20,
|
||||
"subject": "Task 20: Event conversion TryParseExact + compiled accessors",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
15
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 21,
|
||||
"subject": "Task 21: Drain size memoized at enqueue",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
16,
|
||||
17
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 22,
|
||||
"subject": "Task 22: Worker frame writer/reader pooled buffers",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
15
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 23,
|
||||
"subject": "Task 23: Alarm consumer parse + truncation + cadence",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
15
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 24,
|
||||
"subject": "Task 24: Phase B verification on windev (psbridge)",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
16,
|
||||
17,
|
||||
18,
|
||||
19,
|
||||
20,
|
||||
21,
|
||||
22,
|
||||
23
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": 25,
|
||||
"subject": "Task 25: Wrap-up docs sweep + follow-ups",
|
||||
"status": "completed",
|
||||
"blockedBy": [
|
||||
24
|
||||
]
|
||||
}
|
||||
],
|
||||
"lastUpdated": "2026-08-15T18:55:00Z"
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
# Identifying A Deployed Build (Operator Runbook)
|
||||
|
||||
> **Written 2026-08-11 after a false alarm.** An investigation treated the windev production
|
||||
> binary as having no traceable provenance, on two pieces of evidence that both turned out to be
|
||||
> normal output of our own build and deploy procedure. The binary was fine. This runbook records
|
||||
> what those signals actually mean, so the next person spends minutes rather than a forensics pass.
|
||||
|
||||
## The version stamp is unreliable on Windows builds from 2026-07-09 to 2026-08-10
|
||||
|
||||
`src/Directory.Build.props` appends the git short SHA to `InformationalVersion` (`0.1.2+<sha>`) so a
|
||||
running binary can be mapped back to a commit. That stamping was introduced by `ec6f82b`
|
||||
(2026-07-09, TST-11) and was **broken on Windows for its first month**.
|
||||
|
||||
`$(MSBuildThisFileDirectory)` ends in a path separator. On Windows that trailing backslash escaped
|
||||
the closing quote of the `Exec` command, so `git rev-parse` never ran correctly; because the target
|
||||
runs with `ContinueOnError` and `ConsoleToMSBuild` (which mixes stderr into `ConsoleOutput`), git's
|
||||
failure text was stamped as the source revision. The observed form is:
|
||||
|
||||
```
|
||||
0.1.2+fatal: cannot change to ...
|
||||
```
|
||||
|
||||
The full string recovered from wonder's 2026-08-09 server binary (read 2026-08-12) shows the whole
|
||||
failure, including the mismatched `'` … `"` that caused it:
|
||||
|
||||
```
|
||||
0.1.2+fatal: cannot change to 'C:\build\mxgw-deploy\src" rev-parse --short HEAD': Invalid argument
|
||||
```
|
||||
|
||||
**Do not read the leading `0.1.2` as provenance.** It is the static base `<Version>` every build
|
||||
carries, not a truncated SHA. The hazard is a false positive rather than a blank: `0.1.2+fatal:…`
|
||||
reads like a version that succeeded and then picked up noise, when in fact there is no usable
|
||||
identity anywhere in the string. For a binary built in this window the commit is **not recoverable
|
||||
from the binary at all** — so finding nothing is the expected result, not evidence against a SHA
|
||||
established another way.
|
||||
|
||||
`0152180` (2026-08-10 05:49, merged in `c46e5bb`) fixed it two ways: the quoted path gained a
|
||||
trailing `.` so the separator can no longer escape the quote, and `SourceRevisionId` is now gated on
|
||||
a short-SHA shape so no future git failure text can become the revision either.
|
||||
|
||||
**What this means for an operator.** A git error string in the version of a binary built on Windows
|
||||
in that window is the *expected* result of our own build. It is non-diagnostic in **both**
|
||||
directions — it neither incriminates a build nor confirms one, so it should not be treated as
|
||||
evidence of anything. macOS builds in the same window stamp correctly, as do all builds after
|
||||
`0152180`.
|
||||
|
||||
## An absent `C:\build\mxgw-deploy` is expected
|
||||
|
||||
The deploy procedure builds from a **detached worktree** (`git worktree add C:\build\mxgw-deploy
|
||||
<sha>`) so the host's own checkout, which usually sits on a feature branch, is not disturbed. The
|
||||
worktree is removed once the publish is copied out. Finding that the directory a binary was built
|
||||
from no longer exists is the normal end state of a correct deploy, not a deleted trail.
|
||||
|
||||
## What does identify a build
|
||||
|
||||
In rough order of cost:
|
||||
|
||||
1. **Behaviour over the wire.** Works against a running service, needs no host access, and is the
|
||||
fastest discriminator for the worker. `53f69cd` correlates `OnWriteComplete` onto **plain**
|
||||
`Write`/`Write2` replies; `b948e69` did so only for `WriteSecured`/`WriteSecured2`. So a plain
|
||||
`Write` whose reply carries `statuses[0]` proves the worker is at or past `53f69cd`, and an empty
|
||||
`statuses` proves it is not. Keep it non-destructive by writing to a read-only tag — the refusal
|
||||
still exercises the path and returns `OPERATIONAL_ERROR` with detail `1007`.
|
||||
2. **PDB source hashes.** Slower, needs the deployed symbols, but independent of anything the build
|
||||
stamped. **Do not read this as *the* technique on its own.** What settled the 2026-08-11
|
||||
investigation was two independent derivations agreeing: a PDB source-hash match, and a
|
||||
contemporaneous deploy record written the same evening that named the same two commits. The
|
||||
convergence is the result's strength, not either method alone — a hash match tells you which
|
||||
sources a binary was built from, but not that the build was intentional or which host it went to.
|
||||
A reader with only one of the two available should weight it accordingly and look for a second
|
||||
line of evidence.
|
||||
3. **Deployment-side naming.** Since 2026-08-07 the server deploys to a dated directory
|
||||
(`Server-YYYYMMDD`) with the NSSM `Application`/`AppDirectory` repointed at it, and backup
|
||||
directories carry operator-chosen labels naming the work (for example
|
||||
`Worker.bak-20260809-planwrites`). Those conventions place a build in time and intent, and an
|
||||
accidental or off-book deploy tends not to follow them.
|
||||
|
||||
4. **The host's own backup directories, read as a chain.** Each `Server.bak.<timestamp>` holds the
|
||||
exe that deploy *replaced*, so a sweep of `VersionInfo` across them reconstructs the host's deploy
|
||||
history from the host itself, with no repo access and no deploy record. A backup stamped
|
||||
`20260811T060739` containing an exe written 2026-08-09 is the 08-09 build being displaced — the
|
||||
backup's timestamp dates the *next* deploy, not the build inside it. Reading a file's version is
|
||||
non-destructive, unlike opening a SQLite store in a backup directory, which mutates it. This is
|
||||
what established that wonder's `b948e69` and `0a9715d` were two deploys two days apart rather
|
||||
than two competing claims about one binary.
|
||||
|
||||
Note that **mixed Server and Worker SHAs are deliberate**, not drift: the two are swapped
|
||||
independently whenever the contracts are wire-identical, so a host legitimately runs one commit for
|
||||
the server and a later one for the worker.
|
||||
|
||||
## Recorded deploys
|
||||
|
||||
| Date | Host | Server | Worker |
|
||||
|---|---|---|---|
|
||||
| 2026-08-09 | windev (`10.100.0.48`) | `b948e69` (`Server-20260809`) | `53f69cd` |
|
||||
| 2026-08-09 | `wonder-app-vd03` | `b948e69` | `53f69cd` |
|
||||
| 2026-08-11 | `wonder-app-vd03` | `0a9715d` (this deploy wrote `Server.bak.20260811T060739`, holding the displaced 08-09 build) | *carried forward* |
|
||||
| 2026-08-12 | `wonder-app-vd03` | `55f2889` (this deploy wrote `Server.bak.20260812T040122`, holding `0a9715d`) | *carried forward* |
|
||||
|
||||
The two wonder rows after 08-09 are **server swaps**; their worker cells are carried forward from the
|
||||
08-09 entry rather than re-verified, so treat the worker SHA there as unconfirmed. Their server SHAs
|
||||
come from the backup-chain read described above (technique 4), except `55f2889`, which was read
|
||||
directly from the live exe's stamp — trustworthy because it postdates `0152180`.
|
||||
|
||||
`b948e69` is **confirmed by PDB source-hash match plus the contemporaneous record, never by a version
|
||||
stamp** — that build falls in the broken-stamp window and its stamp is structurally unavailable (see
|
||||
the first section). `0a9715d` is the first wonder build to stamp cleanly, since `0152180` landed
|
||||
before it.
|
||||
|
||||
The 2026-08-09 deploy was **two separate swaps**, which is why a single build time does not describe
|
||||
it: the 2026-08-11 investigation dated the server file write to 19:20:24 and the worker to 19:50:06,
|
||||
the latter two minutes after `53f69cd` merged at 19:48, with a matching service stop/start at
|
||||
19:50:29/34.
|
||||
|
||||
That reading is confirmable from artifacts still on disk, without trusting the narrative: windev
|
||||
carries **two** worker backup directories from that day (`Worker.bak-20260809` and
|
||||
`Worker.bak-20260809-planwrites`), and wonder carries `Worker.bak.20260809-planwrites`. Two backups
|
||||
because there were two worker operations. This is easy to misread as redundancy — it is the second
|
||||
swap's fingerprint.
|
||||
|
||||
## The 2026-08-09 deploy returned the worker to mainline
|
||||
|
||||
Worth stating because it went unrecorded at the time and later read as a mystery rather than as the
|
||||
improvement it was. Before that deploy, production ran worker `dd7ca163` (2026-05-22), which is
|
||||
contained **only** by `origin/test/client-e2e-coverage` and is not an ancestor of `main` — meaning
|
||||
the x86 worker in production could not be rebuilt from any mainline commit. `53f69cd` is on `main`,
|
||||
which closes that. Verified 2026-08-11 with `git merge-base --is-ancestor`.
|
||||
|
||||
## Related Documentation
|
||||
|
||||
- [Diagnostics](../Diagnostics.md)
|
||||
- [Gateway Configuration](../GatewayConfiguration.md)
|
||||
+60
-4
@@ -173,9 +173,15 @@ the worker's current state while the corresponding live transition may still be
|
||||
buffered in the monitor's lease, so both can broadcast and the two are
|
||||
indistinguishable on the feed. This applies to the acked-state delta and equally
|
||||
to the older Raise/Clear presence repair: nothing serializes a reconcile pass
|
||||
against the in-flight live stream. Alarm-feed consumers (`StreamAlarms` clients
|
||||
and the dashboard alarm hub) must apply transitions idempotently — treat one as
|
||||
"set this alarm to this state", never as an increment or a toggle.
|
||||
against the in-flight live stream. The monitor narrows that window with a
|
||||
best-effort dedup (NEXT-03): a buffered live transition whose worker timestamp
|
||||
and resulting state the cache already carries from a repair — or whose Clear
|
||||
matches a one-reconcile-generation tombstone keyed on the instance's original
|
||||
raise timestamp — is suppressed instead of re-broadcast. The dedup fires only on
|
||||
a positive marker match (unset timestamps never suppress), so the contract stays
|
||||
at-least-once: alarm-feed consumers (`StreamAlarms` clients and the dashboard
|
||||
alarm hub) must apply transitions idempotently — treat one as "set this alarm to
|
||||
this state", never as an increment or a toggle.
|
||||
|
||||
### Alarm providers and failover
|
||||
|
||||
@@ -299,9 +305,16 @@ Default transport: one bidirectional named pipe per worker.
|
||||
Pipe name:
|
||||
|
||||
```text
|
||||
mxaccess-gateway-{gatewayProcessId}-{sessionId}
|
||||
mxgw-{gatewayProcessId}-{sessionUid}
|
||||
```
|
||||
|
||||
`sessionUid` is the session id without its `session-` prefix (the raw guid hex).
|
||||
The name is deliberately short: on Unix-like hosts (the macOS/Linux test
|
||||
matrix), .NET named pipes are Unix domain sockets at
|
||||
`$TMPDIR/CoreFxPipe_{name}`, and macOS caps the socket path at 104 bytes while
|
||||
its default per-user `TMPDIR` already spends ~49 of them. The gateway PID keeps
|
||||
the name collision-free across gateway restarts.
|
||||
|
||||
Message framing:
|
||||
|
||||
```text
|
||||
@@ -649,6 +662,49 @@ The exact field names should be adjusted to match the actual interop struct,
|
||||
but the design principle is important: do not collapse status arrays into a
|
||||
single success flag.
|
||||
|
||||
### MxStatus Detail Vocabulary
|
||||
|
||||
`MxStatusProxy.detail` carries MXAccess's `MxStatusDetail` code verbatim. The
|
||||
vocabulary below is lifted from the installed toolkit's interop enum
|
||||
(`Interop.aaMxDataConsumer` → `MxStatusDetail`, confirmed against the
|
||||
`MxNativeCodec/MxStatus.cs` map in the MXAccess analysis project — see
|
||||
`docs/DesignDecisions.md` external sources). Consumers mapping statuses onto
|
||||
another protocol (e.g. OtOpcUa's OPC UA status mapping) should key on these
|
||||
codes rather than truncating or banding the raw value:
|
||||
|
||||
| Detail | Name | Detail | Name |
|
||||
|---|---|---|---|
|
||||
| 0 | `MX_S_Success` | 1003 | `MX_E_IndexOutOfRange` |
|
||||
| 1 | `MX_E_RequestTimedOut` | 1004 | `MX_E_DataOutOfRange` |
|
||||
| 2 | `MX_E_PlatformCommunicationError` | 1005 | `MX_E_IncorrectDataType` |
|
||||
| 3 | `MX_E_InvalidPlatformId` | 1006 | `MX_E_NotReadable` |
|
||||
| 4 | `MX_E_InvalidEngineId` | 1007 | `MX_E_NotWriteable` |
|
||||
| 5 | `MX_E_EngineCommunicationError` | 1008 | `MX_E_WriteAccessDenied` |
|
||||
| 6 | `MX_E_InvalidReference` | 1009 | `MX_E_UnknownError` |
|
||||
| 7 | `MX_E_NoGalaxyRepository` | 1010 | `MX_E_ObjectInitializing` |
|
||||
| 8 | `MX_E_InvalidObjectId` | 1011 | `MX_E_EngineInitializing` |
|
||||
| 9 | `MX_E_ObjectSignatureMismatch` | 1012 | `MX_E_SecuredWrite` |
|
||||
| 10 | `MX_E_AttributeSignatureMismatch` | 1013 | `MX_E_VerifiedWrite` |
|
||||
| 11 | `MX_E_ResolvingAttribute` | 1014 | `MX_E_NoAlarmAckPrivilege` |
|
||||
| 12 | `MX_E_ResolvingObject` | 1015 | `MX_E_AlarmAckedAlready` |
|
||||
| 13 | `MX_E_WrongDataType` | 1016 | `MX_E_UserNotHavingAccessRights` |
|
||||
| 14 | `MX_E_WrongNumberOfDimensions` | 1017 | `MX_E_VerifierNotHavingVerifyRights` |
|
||||
| 15 | `MX_E_InvalidIndex` | 8000 | `MX_E_AutomationObjectSpecificError` |
|
||||
| 16 | `MX_E_IndexOutOfOrder` | 1000 | `MX_E_InvalidPrimitiveId` |
|
||||
| 17 | `MX_E_DimensionDoesNotExist` | 1001 | `MX_E_InvalidAttributeId` |
|
||||
| 18 | `MX_E_ConversionNotSupported` | 1002 | `MX_E_InvalidPropertyId` |
|
||||
| 19 | `MX_E_UnableToConvertString` | 25 | `MX_E_GalaxyRepositoryBusy` |
|
||||
| 20 | `MX_E_Overflow` | 26 | `MX_E_EngineOverloaded` |
|
||||
| 21 | `MX_E_NmxVersionMismatch` | 23 | `MX_E_LmxVersionMismatch` |
|
||||
| 22 | `MX_E_NmxInvalidCommand` | 24 | `MX_E_LmxInvalidCommand` |
|
||||
|
||||
Codes observed live in write-completion correlation: `1007`
|
||||
(`MX_E_NotWriteable` — write to a read-only attribute) and `1008`
|
||||
(`MX_E_WriteAccessDenied` — e.g. a write through a plain-advised handle that
|
||||
lacks supervisory access). `1012`/`1013` mark attributes classified for
|
||||
secured/verified writes; `1016`/`1017` are the secured-write credential
|
||||
failures.
|
||||
|
||||
For command replies, return:
|
||||
|
||||
- protocol status,
|
||||
|
||||
@@ -53,6 +53,16 @@ first, then merge.
|
||||
- Unreachable-host red: point at a bogus port / stop sshd, confirm the job fails fast, not hangs.
|
||||
- Concurrency: push two branches back-to-back, confirm the second remote run waits on the lock.
|
||||
- Nightly: trigger the schedule path, confirm `live` runs and a forced failure opens an issue.
|
||||
**Done 2026-08-10** (Check 6). Verified two ways: (a) production — every red nightly since
|
||||
2026-07-17 has auto-filed an issue (#126–#139) authored by the `gitea-actions` bot, e.g. run 672
|
||||
→ issue #139, with the built-in token masked to `***` in the job log; (b) a forced-failure probe
|
||||
on the throwaway branch `test/tst25-check6-nightly-issue` (run 677 → issue #140, since closed and
|
||||
the branch deleted), which reproduced the job shape with `exit 1` in place of the live step and
|
||||
confirmed the `if: failure()` step fires, the token carries issue-write, and the payload is
|
||||
well-formed. The probe also caught the one defect: `${{ github.server_url }}` is the
|
||||
runner-internal `http://gitea:3000`, so the run link in the issue body was unreachable from a
|
||||
browser — the body now uses the `PUBLIC_SERVER_URL` job env instead (the API call still targets
|
||||
`github.server_url`, which is what the job container can resolve).
|
||||
- Confirm no key material appears in job logs.
|
||||
|
||||
## Degraded mode
|
||||
|
||||
@@ -26,7 +26,10 @@
|
||||
<Target Name="StampSourceRevision"
|
||||
BeforeTargets="GetAssemblyVersion;GenerateAssemblyInfo"
|
||||
Condition="'$(SourceRevisionId)' == ''">
|
||||
<Exec Command="git -C "$(MSBuildThisFileDirectory)" rev-parse --short HEAD"
|
||||
<!-- The trailing "." is load-bearing: $(MSBuildThisFileDirectory) ends in a path
|
||||
separator, and on Windows that trailing backslash escapes the closing quote,
|
||||
mangling the command so git's stderr got stamped as the revision (NEXT-09). -->
|
||||
<Exec Command="git -C "$(MSBuildThisFileDirectory)." rev-parse --short HEAD"
|
||||
ConsoleToMSBuild="true"
|
||||
StandardOutputImportance="Low"
|
||||
ContinueOnError="true"
|
||||
@@ -34,7 +37,9 @@
|
||||
<Output TaskParameter="ConsoleOutput" PropertyName="_StampedGitSha" />
|
||||
</Exec>
|
||||
<PropertyGroup>
|
||||
<SourceRevisionId Condition="'$(_StampedGitSha)' != ''">$(_StampedGitSha.Trim())</SourceRevisionId>
|
||||
<!-- Accept only something that looks like a git short SHA; Exec's ConsoleOutput
|
||||
mixes in stderr, so any git failure text must never become the revision. -->
|
||||
<SourceRevisionId Condition="$([System.Text.RegularExpressions.Regex]::IsMatch('$(_StampedGitSha.Trim())', '^[0-9a-f]{7,40}$'))">$(_StampedGitSha.Trim())</SourceRevisionId>
|
||||
</PropertyGroup>
|
||||
</Target>
|
||||
|
||||
|
||||
+2
-2
@@ -22,8 +22,8 @@
|
||||
(IntegrationTests-028).
|
||||
-->
|
||||
<ItemGroup>
|
||||
<PackageReference Include="ZB.MOM.WW.Auth.Abstractions" Version="0.1.5" />
|
||||
<PackageReference Include="ZB.MOM.WW.Auth.Ldap" Version="0.1.5" />
|
||||
<PackageReference Include="ZB.MOM.WW.Auth.Abstractions" Version="0.2.1" />
|
||||
<PackageReference Include="ZB.MOM.WW.Auth.Ldap" Version="0.2.1" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="10.0.7" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.7" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -34,6 +34,21 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
private readonly Dictionary<string, ActiveAlarmSnapshot> _alarms = new(StringComparer.Ordinal);
|
||||
private readonly List<Subscriber> _subscribers = [];
|
||||
|
||||
// Memoized CurrentAlarms projection, guarded by _sync: the cloned, read-only view of _alarms
|
||||
// handed to the dashboard and the QueryActiveAlarms RPC. Cloning the whole set per read held
|
||||
// _sync — the broadcast lock — for the length of the copy, so a polled dashboard stalled every
|
||||
// ApplyTransition/Broadcast behind it. Null means "not built for the current generation":
|
||||
// every path that writes _alarms must null this under _sync, or readers keep a stale set.
|
||||
private ActiveAlarmSnapshot[]? _currentAlarmsProjection;
|
||||
|
||||
// NEXT-03 dedup tombstones, guarded by _sync: alarm instances whose Clear was synthesized by
|
||||
// the most recent reconcile pass, keyed by reference with the instance's original raise
|
||||
// timestamp as the identity marker. A buffered live Clear for the same instance is a duplicate
|
||||
// of the repair and is suppressed. One generation deep: each reconcile pass replaces the map,
|
||||
// so a tombstone lives at least one reconcile interval — far longer than the lease buffer the
|
||||
// duplicate would be sitting in — and the map stays bounded by the feed's churn per interval.
|
||||
private readonly Dictionary<string, Timestamp> _clearedByReconcile = new(StringComparer.Ordinal);
|
||||
|
||||
// Current provider status (mode + degraded + reason + since), guarded by _sync.
|
||||
// Initialized to the alarm-manager, not-degraded baseline so a late joiner sees
|
||||
// a sensible status even before any OnAlarmProviderModeChanged event arrives.
|
||||
@@ -85,7 +100,12 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
{
|
||||
lock (_sync)
|
||||
{
|
||||
return _alarms.Values.Select(alarm => alarm.Clone()).ToArray();
|
||||
// Same clone semantics as an uncached read — callers still get instances no
|
||||
// mutation can leak back into the cache — but built once per alarm-set
|
||||
// generation instead of once per caller.
|
||||
return _currentAlarmsProjection ??= _alarms.Values
|
||||
.Select(alarm => alarm.Clone())
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -413,17 +433,66 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
{
|
||||
if (transition.TransitionKind == AlarmTransitionKind.Clear)
|
||||
{
|
||||
_alarms.Remove(reference);
|
||||
bool wasKnown = _alarms.Remove(reference);
|
||||
if (wasKnown)
|
||||
{
|
||||
_currentAlarmsProjection = null;
|
||||
}
|
||||
|
||||
if (!wasKnown && IsDuplicateOfReconcileClear(reference, transition))
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_alarms[reference] = SnapshotFromTransition(transition);
|
||||
ActiveAlarmSnapshot snapshot = SnapshotFromTransition(transition);
|
||||
bool duplicate = _alarms.TryGetValue(reference, out ActiveAlarmSnapshot? existing)
|
||||
&& IsDuplicateOfCachedState(existing, snapshot);
|
||||
_alarms[reference] = snapshot;
|
||||
_currentAlarmsProjection = null;
|
||||
if (duplicate)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
Broadcast(new AlarmFeedMessage { Transition = transition }, reference);
|
||||
}
|
||||
}
|
||||
|
||||
// NEXT-03: best-effort dedup of the reconcile/live race. A reconcile that already synthesized
|
||||
// this transition as a feed repair left the cache carrying the worker's transition timestamp
|
||||
// and resulting state — both derived from the same worker-side value the live transition
|
||||
// carries — so an exact (timestamp, state) match means this live transition's outcome has
|
||||
// already been broadcast. Suppress only on a positive match: an unset timestamp on either
|
||||
// side keeps today's at-least-once behavior.
|
||||
private static bool IsDuplicateOfCachedState(ActiveAlarmSnapshot existing, ActiveAlarmSnapshot incoming)
|
||||
{
|
||||
return existing.LastTransitionTimestamp is not null
|
||||
&& incoming.LastTransitionTimestamp is not null
|
||||
&& existing.LastTransitionTimestamp.Equals(incoming.LastTransitionTimestamp)
|
||||
&& existing.CurrentState == incoming.CurrentState;
|
||||
}
|
||||
|
||||
// NEXT-03, the Clear leg. A reconcile Clear repair removes the cache entry before the buffered
|
||||
// live Clear drains, so there is no cached state to compare against; the tombstone recorded by
|
||||
// ApplyReconcile identifies the cleared instance by its original raise timestamp instead. The
|
||||
// match consumes the tombstone, so a genuinely new raise/clear cycle (which carries a newer
|
||||
// original raise timestamp) is never swallowed. Caller holds _sync.
|
||||
private bool IsDuplicateOfReconcileClear(string reference, OnAlarmTransitionEvent transition)
|
||||
{
|
||||
if (transition.OriginalRaiseTimestamp is not null
|
||||
&& _clearedByReconcile.TryGetValue(reference, out Timestamp? clearedInstance)
|
||||
&& clearedInstance.Equals(transition.OriginalRaiseTimestamp))
|
||||
{
|
||||
_clearedByReconcile.Remove(reference);
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// Handles the worker's provider-mode-change event: updates the stored provider
|
||||
// status, broadcasts it to every subscriber (provider status is global, not
|
||||
// alarm-scoped), records the switch metric, and forces a cache reconcile so the
|
||||
@@ -533,11 +602,14 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
//
|
||||
// Delivery semantics: feed repair transitions are AT-LEAST-ONCE, not exactly-once. A reconcile
|
||||
// reads the worker's current state while the corresponding live transition may still be
|
||||
// buffered in the alarm lease's channel; both then broadcast, and the two are indistinguishable
|
||||
// on the feed. This is inherent to the reconcile design and pre-dates the acked-state delta
|
||||
// (the Raise/Clear repair has always had it), since nothing serializes a reconcile against the
|
||||
// in-flight live stream. Consumers must therefore treat alarm state idempotently — apply a
|
||||
// transition as "set the alarm to this state", never as an increment or a toggle.
|
||||
// buffered in the alarm lease's channel; both would then broadcast, and the two are
|
||||
// indistinguishable on the feed, since nothing serializes a reconcile against the in-flight
|
||||
// live stream. ApplyTransition narrows that window with a best-effort dedup (NEXT-03): a live
|
||||
// transition whose worker timestamp and resulting state the cache already carries — or whose
|
||||
// Clear matches a tombstone recorded below — was already broadcast as a repair and is
|
||||
// suppressed. The dedup fires only on a positive marker match, so the contract stays
|
||||
// at-least-once: consumers must still treat alarm state idempotently — apply a transition as
|
||||
// "set the alarm to this state", never as an increment or a toggle.
|
||||
private void ApplyReconcile(IEnumerable<ActiveAlarmSnapshot> snapshots)
|
||||
{
|
||||
Dictionary<string, ActiveAlarmSnapshot> next = new(StringComparer.Ordinal);
|
||||
@@ -551,10 +623,19 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
|
||||
lock (_sync)
|
||||
{
|
||||
// Previous-generation tombstones have outlived the buffered live transitions they
|
||||
// guard against (one full reconcile interval); start this pass's generation fresh.
|
||||
_clearedByReconcile.Clear();
|
||||
|
||||
foreach (KeyValuePair<string, ActiveAlarmSnapshot> existing in _alarms)
|
||||
{
|
||||
if (!next.ContainsKey(existing.Key))
|
||||
{
|
||||
if (existing.Value.OriginalRaiseTimestamp is not null)
|
||||
{
|
||||
_clearedByReconcile[existing.Key] = existing.Value.OriginalRaiseTimestamp;
|
||||
}
|
||||
|
||||
Broadcast(
|
||||
new AlarmFeedMessage { Transition = TransitionFromSnapshot(existing.Value, AlarmTransitionKind.Clear) },
|
||||
existing.Key);
|
||||
@@ -587,6 +668,8 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
{
|
||||
_alarms[incoming.Key] = incoming.Value;
|
||||
}
|
||||
|
||||
_currentAlarmsProjection = null;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -633,6 +716,7 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
lock (_sync)
|
||||
{
|
||||
_alarms.Clear();
|
||||
_currentAlarmsProjection = null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -46,6 +46,33 @@ public sealed class AlarmsOptions
|
||||
/// </summary>
|
||||
public int ReconcileIntervalSeconds { get; init; } = 30;
|
||||
|
||||
/// <summary>
|
||||
/// Cadence at which the worker's STA polls the AVEVA alarm consumer
|
||||
/// (<c>GetXmlCurrentAlarms2</c>) for the current active-alarm snapshot.
|
||||
/// Default 500 ms; must be between 100 ms and 3,600,000 ms (one hour).
|
||||
/// Every poll is a COM call plus an XML parse on the STA that also
|
||||
/// serves reads and writes, so driving it below 100 ms starves the
|
||||
/// command path; above an hour the cadence stops being a cadence and
|
||||
/// silently disables alarm polling. Conveyed to the worker through the
|
||||
/// <c>MXGATEWAY_ALARM_POLL_INTERVAL_MS</c> environment variable.
|
||||
/// </summary>
|
||||
public int PollIntervalMilliseconds { get; init; } = 500;
|
||||
|
||||
/// <summary>
|
||||
/// Cap the worker passes to <c>GetXmlCurrentAlarms2</c>'s
|
||||
/// <c>maxAlmCnt</c> argument. Default 1024; must be between 64 and
|
||||
/// 65,536 — the worker is a 32-bit process that materializes each
|
||||
/// fetch as one BSTR plus a full XmlDocument, so an unbounded cap
|
||||
/// faults the STA rather than merely slowing it. A fetch that comes
|
||||
/// back holding exactly this many records is treated as truncated: the
|
||||
/// worker keeps the alarms the capped fetch could not mention in its
|
||||
/// snapshot rather than letting their absence read as a clear. Raise it
|
||||
/// on galaxies whose steady-state active-alarm count approaches the
|
||||
/// cap. Conveyed to the worker through the
|
||||
/// <c>MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH</c> environment variable.
|
||||
/// </summary>
|
||||
public int MaxAlarmsPerFetch { get; init; } = 1024;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration for the alarm-manager ↔ subtag fallback mechanism:
|
||||
/// operating mode, failure-detection thresholds, discovery, and subtag
|
||||
|
||||
@@ -11,4 +11,5 @@ public sealed record EffectiveLdapConfiguration(
|
||||
string ServiceAccountPassword,
|
||||
string UserNameAttribute,
|
||||
string DisplayNameAttribute,
|
||||
string GroupAttribute);
|
||||
string GroupAttribute,
|
||||
IReadOnlyList<string> FallbackServers);
|
||||
|
||||
@@ -13,6 +13,33 @@ namespace ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
/// </summary>
|
||||
public sealed class GalaxyRepositoryOptionsValidator : OptionsValidatorBase<GalaxyRepositoryOptions>
|
||||
{
|
||||
// See GatewayOptionsValidator for why this is nullable and what null means.
|
||||
private readonly string? _contentRootPath;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="GalaxyRepositoryOptionsValidator"/> class for
|
||||
/// the dependency-injection path, taking the content root from the host environment.
|
||||
/// </summary>
|
||||
/// <param name="environment">The host environment.</param>
|
||||
public GalaxyRepositoryOptionsValidator(IHostEnvironment environment)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(environment);
|
||||
_contentRootPath = environment.ContentRootPath;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="GalaxyRepositoryOptionsValidator"/> class for
|
||||
/// unit tests and non-DI callers.
|
||||
/// </summary>
|
||||
/// <param name="contentRootPath">
|
||||
/// Content root to test the snapshot path against; <see langword="null"/> leaves the
|
||||
/// content-root rule inactive.
|
||||
/// </param>
|
||||
internal GalaxyRepositoryOptionsValidator(string? contentRootPath = null)
|
||||
{
|
||||
_contentRootPath = contentRootPath;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Validate(ValidationBuilder builder, GalaxyRepositoryOptions options)
|
||||
{
|
||||
@@ -37,5 +64,10 @@ public sealed class GalaxyRepositoryOptionsValidator : OptionsValidatorBase<Gala
|
||||
options.SnapshotCachePath,
|
||||
"MxGateway:Galaxy:SnapshotCachePath must be an absolute (rooted) path so the Galaxy snapshot never lands in the launch working directory.",
|
||||
builder);
|
||||
GatewayConfigPathRules.AddIfUnderContentRoot(
|
||||
options.SnapshotCachePath,
|
||||
_contentRootPath,
|
||||
$"MxGateway:Galaxy:SnapshotCachePath must not be inside the application directory ({_contentRootPath}). The upgrade procedure renames that directory, so the cached snapshot is discarded on every deploy and the gateway starts cold.",
|
||||
builder);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,25 +53,102 @@ internal static class GatewayConfigPathRules
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_ = Path.GetFullPath(value);
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
builder.Add(message);
|
||||
}
|
||||
catch (NotSupportedException)
|
||||
{
|
||||
builder.Add(message);
|
||||
}
|
||||
catch (PathTooLongException)
|
||||
{
|
||||
builder.Add(message);
|
||||
}
|
||||
catch (IOException)
|
||||
if (!TryGetFullPath(value, out _))
|
||||
{
|
||||
builder.Add(message);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Fails validation when <paramref name="value"/> resolves to a location inside
|
||||
/// <paramref name="contentRoot"/> — the directory the application runs from.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Rooted is not the same as safe, and this is the rule that closes the gap.</b>
|
||||
/// <see cref="AddIfNotRooted"/> stops a store drifting with the working directory, but an
|
||||
/// absolute path <em>inside the app directory</em> passes it cleanly — and that is what failed
|
||||
/// in production on 2026-08-09. The upgrade procedure renames the app directory to
|
||||
/// <c>Server.bak.*</c> and unpacks a new one; a store living there is renamed away with it, the
|
||||
/// process then creates a fresh empty one at the same path, and nothing reports an error. All
|
||||
/// API keys were lost and no gRPC client could authenticate for two days. The deploy itself was
|
||||
/// executed correctly — the binaries were the point of the rename, and the store was collateral.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The same shape catches the dev-side symptom: a store under the content root lands in the
|
||||
/// source tree, which is how <c>mxgateway-secrets.db</c> once tripped the repository's
|
||||
/// tree-hygiene test.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Comparison is case-insensitive only on Windows. On a case-insensitive macOS volume this can
|
||||
/// miss a violation that differs only in case, which is a missed warning in dev; assuming
|
||||
/// case-insensitivity on Linux would instead reject a legitimate path, and a false startup
|
||||
/// abort is the worse failure.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="value">The configured path value.</param>
|
||||
/// <param name="contentRoot">The application content root to test against.</param>
|
||||
/// <param name="message">The failure message to record when the value is under the content root.</param>
|
||||
/// <param name="builder">The validation builder accumulating failures.</param>
|
||||
public static void AddIfUnderContentRoot(
|
||||
string? value,
|
||||
string? contentRoot,
|
||||
string message,
|
||||
ValidationBuilder builder)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value) || string.IsNullOrWhiteSpace(contentRoot))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// A malformed path is AddIfInvalidPath's message to report; staying silent here keeps one
|
||||
// bad value from producing two failures that say different things about the same mistake.
|
||||
if (!TryGetFullPath(value, out string fullValue) || !TryGetFullPath(contentRoot, out string fullRoot))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
fullRoot = fullRoot.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
|
||||
|
||||
StringComparison comparison = OperatingSystem.IsWindows()
|
||||
? StringComparison.OrdinalIgnoreCase
|
||||
: StringComparison.Ordinal;
|
||||
|
||||
// The separator is load-bearing: a bare prefix test would also match a sibling directory
|
||||
// whose name merely starts with the root's ("/srv/app" against "/srv/app-data").
|
||||
if (string.Equals(fullValue, fullRoot, comparison)
|
||||
|| fullValue.StartsWith(fullRoot + Path.DirectorySeparatorChar, comparison))
|
||||
{
|
||||
builder.Add(message);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryGetFullPath(string value, out string fullPath)
|
||||
{
|
||||
try
|
||||
{
|
||||
fullPath = Path.GetFullPath(value);
|
||||
return true;
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
fullPath = string.Empty;
|
||||
return false;
|
||||
}
|
||||
catch (NotSupportedException)
|
||||
{
|
||||
fullPath = string.Empty;
|
||||
return false;
|
||||
}
|
||||
catch (PathTooLongException)
|
||||
{
|
||||
fullPath = string.Empty;
|
||||
return false;
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
fullPath = string.Empty;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +30,8 @@ public sealed class GatewayConfigurationProvider(IOptions<GatewayOptions> option
|
||||
ServiceAccountPassword: RedactedValue,
|
||||
UserNameAttribute: value.Ldap.UserNameAttribute,
|
||||
DisplayNameAttribute: value.Ldap.DisplayNameAttribute,
|
||||
GroupAttribute: value.Ldap.GroupAttribute),
|
||||
GroupAttribute: value.Ldap.GroupAttribute,
|
||||
FallbackServers: value.Ldap.FallbackServers),
|
||||
Worker: new EffectiveWorkerConfiguration(
|
||||
ExecutablePath: value.Worker.ExecutablePath,
|
||||
WorkingDirectory: value.Worker.WorkingDirectory,
|
||||
|
||||
@@ -10,20 +10,33 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
||||
private const int MinimumMaxMessageBytes = 1024;
|
||||
private const int MaximumMaxMessageBytes = 256 * 1024 * 1024;
|
||||
|
||||
// Bounds on the worker's outbound event-queue capacity. The floor keeps enough headroom that a
|
||||
// normal subscription burst cannot overflow the queue (an overflow faults the whole session);
|
||||
// the ceiling keeps a mistyped value from committing the x86 worker to an unbounded backlog.
|
||||
private const int MinimumWorkerEventQueueCapacity = 1000;
|
||||
private const int MaximumWorkerEventQueueCapacity = 1_000_000;
|
||||
|
||||
// Whether the host is running in the Production environment. Drives the production-only
|
||||
// hard-stops (dashboard login disabled, plaintext LDAP transport) that must abort startup
|
||||
// rather than merely warn. Non-production hosts keep the permissive dev posture.
|
||||
private readonly bool _isProduction;
|
||||
|
||||
// The application content root. Store paths must not live under it — see
|
||||
// GatewayConfigPathRules.AddIfUnderContentRoot. Null for non-DI callers that supply no
|
||||
// environment, which skips the rule rather than inventing a root to test against.
|
||||
private readonly string? _contentRootPath;
|
||||
|
||||
/// <summary>
|
||||
/// Initializes a new instance of the <see cref="GatewayOptionsValidator"/> class for the
|
||||
/// dependency-injection path, deriving the production posture from the host environment.
|
||||
/// dependency-injection path, deriving the production posture and content root from the host
|
||||
/// environment.
|
||||
/// </summary>
|
||||
/// <param name="environment">The host environment.</param>
|
||||
public GatewayOptionsValidator(IHostEnvironment environment)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(environment);
|
||||
_isProduction = environment.IsProduction();
|
||||
_contentRootPath = environment.ContentRootPath;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -32,15 +45,20 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
||||
/// hard-stops do not fire; pass <see langword="true"/> to exercise them.
|
||||
/// </summary>
|
||||
/// <param name="isProduction">Whether to treat the host as running in Production.</param>
|
||||
internal GatewayOptionsValidator(bool isProduction = false)
|
||||
/// <param name="contentRootPath">
|
||||
/// Content root to test store paths against; <see langword="null"/> leaves the content-root
|
||||
/// rule inactive, which is what a caller with no real host wants.
|
||||
/// </param>
|
||||
internal GatewayOptionsValidator(bool isProduction = false, string? contentRootPath = null)
|
||||
{
|
||||
_isProduction = isProduction;
|
||||
_contentRootPath = contentRootPath;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void Validate(ValidationBuilder builder, GatewayOptions options)
|
||||
{
|
||||
ValidateAuthentication(options.Authentication, builder);
|
||||
ValidateAuthentication(options.Authentication, _contentRootPath, builder);
|
||||
ValidateLdap(options.Ldap, builder, _isProduction);
|
||||
ValidateWorker(options.Worker, builder);
|
||||
ValidateSessions(options.Sessions, builder);
|
||||
@@ -88,6 +106,13 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
||||
"MxGateway:Security:ApiKeyFailureTrackedPeers must be greater than zero.",
|
||||
builder);
|
||||
|
||||
// Retention must be at least one day: 0 would sweep the audit table on every pass, which
|
||||
// is a way to silently disable auditing rather than an expression of intent.
|
||||
AddIfNotPositive(
|
||||
options.AuditRetentionDays,
|
||||
"MxGateway:Security:AuditRetentionDays must be greater than zero (at least one day of audit history is retained).",
|
||||
builder);
|
||||
|
||||
// The two-layer limiter knobs (SEC-31) accept 0 as "disable this layer": a zero aggregate
|
||||
// limit turns off cross-peer counting, and a zero probe interval restores absolute blocking.
|
||||
// Negatives express no intent.
|
||||
@@ -101,7 +126,10 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
||||
builder);
|
||||
}
|
||||
|
||||
private static void ValidateAuthentication(AuthenticationOptions options, ValidationBuilder builder)
|
||||
private static void ValidateAuthentication(
|
||||
AuthenticationOptions options,
|
||||
string? contentRootPath,
|
||||
ValidationBuilder builder)
|
||||
{
|
||||
if (!Enum.IsDefined(options.Mode))
|
||||
{
|
||||
@@ -123,6 +151,11 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
||||
options.SqlitePath,
|
||||
"MxGateway:Authentication:SqlitePath must be an absolute (rooted) path so the credential store never lands in the launch working directory.",
|
||||
builder);
|
||||
AddIfUnderContentRoot(
|
||||
options.SqlitePath,
|
||||
contentRootPath,
|
||||
$"MxGateway:Authentication:SqlitePath must not be inside the application directory ({contentRootPath}). The upgrade procedure renames that directory, which abandons the credential store and silently starts an empty one — every API key is lost and no client can authenticate.",
|
||||
builder);
|
||||
AddIfBlank(
|
||||
options.PepperSecretName,
|
||||
"MxGateway:Authentication:PepperSecretName is required when API-key authentication is enabled.",
|
||||
@@ -248,6 +281,12 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
||||
"MxGateway:Worker:HeartbeatGraceSeconds must be greater than or equal to HeartbeatIntervalSeconds.");
|
||||
}
|
||||
|
||||
if (options.EventQueueCapacity is < MinimumWorkerEventQueueCapacity or > MaximumWorkerEventQueueCapacity)
|
||||
{
|
||||
builder.Add(
|
||||
$"MxGateway:Worker:EventQueueCapacity must be between {MinimumWorkerEventQueueCapacity} and {MaximumWorkerEventQueueCapacity}.");
|
||||
}
|
||||
|
||||
if (options.MaxMessageBytes is < MinimumMaxMessageBytes or > MaximumMaxMessageBytes)
|
||||
{
|
||||
builder.Add(
|
||||
@@ -387,8 +426,38 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
||||
|
||||
private static readonly string[] ValidAlarmFallbackModes = ["Auto", "ForceAlarmManager", "ForceSubtag"];
|
||||
|
||||
private const int MinimumAlarmPollIntervalMilliseconds = 100;
|
||||
|
||||
// One hour. Above this the cadence stops being a cadence: int.MaxValue
|
||||
// milliseconds is ~24 days, which silently disables alarm polling instead
|
||||
// of reporting the misconfiguration.
|
||||
private const int MaximumAlarmPollIntervalMilliseconds = 3_600_000;
|
||||
|
||||
private const int MinimumMaxAlarmsPerFetch = 64;
|
||||
|
||||
// The worker is a 32-bit process and materializes each fetch as one BSTR
|
||||
// plus a full XmlDocument over it, so an unbounded cap is an out-of-memory
|
||||
// fault on the STA rather than a slow poll.
|
||||
private const int MaximumMaxAlarmsPerFetch = 65_536;
|
||||
|
||||
private static void ValidateAlarms(AlarmsOptions options, ValidationBuilder builder)
|
||||
{
|
||||
// Validated regardless of Enabled: both values are stamped onto every
|
||||
// worker launch environment, so a bad value is a misconfiguration even
|
||||
// before the central monitor is switched on.
|
||||
if (options.PollIntervalMilliseconds is < MinimumAlarmPollIntervalMilliseconds
|
||||
or > MaximumAlarmPollIntervalMilliseconds)
|
||||
{
|
||||
builder.Add(
|
||||
$"MxGateway:Alarms:PollIntervalMilliseconds must be between {MinimumAlarmPollIntervalMilliseconds} and {MaximumAlarmPollIntervalMilliseconds}.");
|
||||
}
|
||||
|
||||
if (options.MaxAlarmsPerFetch is < MinimumMaxAlarmsPerFetch or > MaximumMaxAlarmsPerFetch)
|
||||
{
|
||||
builder.Add(
|
||||
$"MxGateway:Alarms:MaxAlarmsPerFetch must be between {MinimumMaxAlarmsPerFetch} and {MaximumMaxAlarmsPerFetch}.");
|
||||
}
|
||||
|
||||
if (!options.Enabled)
|
||||
{
|
||||
return;
|
||||
@@ -555,4 +624,14 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
||||
|
||||
private static void AddIfInvalidPath(string? value, string message, ValidationBuilder builder)
|
||||
=> GatewayConfigPathRules.AddIfInvalidPath(value, message, builder);
|
||||
|
||||
// Rooted is not the same as safe: an absolute path inside the app directory passes
|
||||
// AddIfNotRooted and is still renamed away by the upgrade procedure. See
|
||||
// GatewayConfigPathRules.AddIfUnderContentRoot.
|
||||
private static void AddIfUnderContentRoot(
|
||||
string? value,
|
||||
string? contentRoot,
|
||||
string message,
|
||||
ValidationBuilder builder)
|
||||
=> GatewayConfigPathRules.AddIfUnderContentRoot(value, contentRoot, message, builder);
|
||||
}
|
||||
|
||||
@@ -68,4 +68,19 @@ public sealed class LdapOptions
|
||||
|
||||
/// <summary>Gets the LDAP attribute name for group membership.</summary>
|
||||
public string GroupAttribute { get; init; } = "memberOf";
|
||||
|
||||
/// <summary>
|
||||
/// Gets the ordered fallback LDAP endpoints (<c>"host"</c> or <c>"host:port"</c>) the shared
|
||||
/// provider walks when the primary fails with a system-side error. Empty (the default) leaves
|
||||
/// single-endpoint behaviour unchanged. Mirrors
|
||||
/// <see cref="ZB.MOM.WW.Auth.Abstractions.Ldap.LdapOptions.FallbackServers"/>, added in
|
||||
/// ZB.MOM.WW.Auth 0.2.0.
|
||||
/// <para>
|
||||
/// Carried here only so the effective-config display does not hide a configured backup DC —
|
||||
/// nothing on the gateway side reads it. Entry syntax is validated at boot by the shared
|
||||
/// <c>LdapOptionsValidator</c>, which owns the (internal) parser; re-validating here would
|
||||
/// mean a second, drifting copy of that grammar.
|
||||
/// </para>
|
||||
/// </summary>
|
||||
public IReadOnlyList<string> FallbackServers { get; init; } = [];
|
||||
}
|
||||
|
||||
@@ -88,4 +88,13 @@ public sealed class SecurityOptions
|
||||
/// ceiling of twice this value. Default is 4096.
|
||||
/// </summary>
|
||||
public int ApiKeyFailureTrackedPeers { get; init; } = 4096;
|
||||
|
||||
/// <summary>
|
||||
/// Gets how many days of canonical audit history the gateway keeps. The audit drain sweeps
|
||||
/// <c>audit_event</c> once at startup and hourly thereafter, deleting rows older than this
|
||||
/// window; without it the table grows without bound in the same SQLite file the
|
||||
/// authentication hot path reads. Must be greater than zero — audit retention cannot be
|
||||
/// disabled by configuration, only widened. Default is 90 days.
|
||||
/// </summary>
|
||||
public int AuditRetentionDays { get; init; } = 90;
|
||||
}
|
||||
|
||||
@@ -33,6 +33,18 @@ public sealed class WorkerOptions
|
||||
/// </summary>
|
||||
public int WriteCompletionWaitMilliseconds { get; init; } = 1500;
|
||||
|
||||
/// <summary>
|
||||
/// Capacity of the worker's outbound MXAccess event queue, in events.
|
||||
/// Default 10,000; must be between 1,000 and 1,000,000. This is
|
||||
/// headroom, not a throttle: the queue has no drop policy, so a burst
|
||||
/// that fills it faults the session with a <c>QueueOverflow</c> worker
|
||||
/// fault. Raise it for sessions whose subscription set can outrun the
|
||||
/// drain loop (large advise sets, slow event consumers). Conveyed to
|
||||
/// the worker through the <c>MXGATEWAY_EVENT_QUEUE_CAPACITY</c>
|
||||
/// environment variable.
|
||||
/// </summary>
|
||||
public int EventQueueCapacity { get; init; } = 10000;
|
||||
|
||||
/// <summary>The maximum time in seconds for graceful shutdown.</summary>
|
||||
public int ShutdownTimeoutSeconds { get; init; } = 10;
|
||||
|
||||
|
||||
@@ -36,6 +36,32 @@ public static class DashboardDisplay
|
||||
return string.IsNullOrWhiteSpace(value) ? "-" : value;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Formats a nullable text value for display, shortened to a maximum length.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// For table cells bound to text the gateway does not control — fault messages, COM
|
||||
/// exception text, SQL errors. One multi-line exception otherwise makes a single row
|
||||
/// several times taller than its neighbours. Call sites keep the full text reachable
|
||||
/// on the element's <c>title</c> and on the row's detail page.
|
||||
/// </remarks>
|
||||
/// <param name="value">The text to format.</param>
|
||||
/// <param name="maxLength">Maximum characters to render before the ellipsis.</param>
|
||||
/// <returns>Formatted text, ellipsized when longer than <paramref name="maxLength"/>, or "-" if null or empty.</returns>
|
||||
public static string Abbreviate(string? value, int maxLength = 80)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return "-";
|
||||
}
|
||||
|
||||
// Length-checked, never a bare range slice: a value shorter than maxLength
|
||||
// would throw and take the whole page render down with it.
|
||||
return value.Length <= maxLength
|
||||
? value
|
||||
: string.Concat(value.AsSpan(0, maxLength).TrimEnd(), "…");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Formats a long count value for display with thousands separator.
|
||||
/// </summary>
|
||||
|
||||
@@ -1,80 +1,118 @@
|
||||
using Microsoft.AspNetCore.Components;
|
||||
using Microsoft.AspNetCore.SignalR.Client;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Components;
|
||||
|
||||
/// <summary>
|
||||
/// Base class for Blazor dashboard pages that watch gateway metrics
|
||||
/// snapshots. The previous implementation polled
|
||||
/// <see cref="IDashboardSnapshotService.WatchSnapshotsAsync"/> directly; we
|
||||
/// now subscribe to <see cref="DashboardSnapshotHub"/> so updates are
|
||||
/// pushed and disconnects survive reconnects via SignalR's
|
||||
/// auto-reconnect.
|
||||
/// Base class for Blazor dashboard pages that watch gateway metrics snapshots.
|
||||
/// Pages subscribe to the in-process <see cref="IDashboardSnapshotFeed"/>, which
|
||||
/// multicasts a single <see cref="IDashboardSnapshotService.WatchSnapshotsAsync"/>
|
||||
/// enumeration to every circuit. An earlier implementation had each page open its
|
||||
/// own SignalR connection to <c>/hubs/snapshot</c> — a loopback WebSocket back into
|
||||
/// this same process, per page. The snapshot hub and its publisher remain for
|
||||
/// external (non-circuit) clients; server-rendered pages no longer use them.
|
||||
/// </summary>
|
||||
public abstract class DashboardPageBase : ComponentBase, IAsyncDisposable
|
||||
{
|
||||
private HubConnection? _hub;
|
||||
/// <summary>
|
||||
/// Upper bound on waiting for the watch loop while disposing. The loop marshals
|
||||
/// renders through the renderer's dispatcher and disposal can run on that same
|
||||
/// dispatcher, so the wait is bounded rather than unconditional.
|
||||
/// </summary>
|
||||
private static readonly TimeSpan WatchDrainTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>Snapshot service used to seed the initial render before the hub connects.</summary>
|
||||
private readonly CancellationTokenSource _watchCancellation = new();
|
||||
private Task? _watchTask;
|
||||
|
||||
/// <summary>Snapshot service used to seed the initial render before the first feed update.</summary>
|
||||
[Inject]
|
||||
protected IDashboardSnapshotService SnapshotService { get; set; } = null!;
|
||||
|
||||
/// <summary>Factory that builds the SignalR connection (mints the hub bearer token).</summary>
|
||||
/// <summary>Shared in-process snapshot feed this page renders from.</summary>
|
||||
[Inject]
|
||||
protected DashboardHubConnectionFactory HubFactory { get; set; } = null!;
|
||||
protected IDashboardSnapshotFeed SnapshotFeed { get; set; } = null!;
|
||||
|
||||
/// <summary>Logger used to report a snapshot subscription that ended or would not drain.</summary>
|
||||
[Inject]
|
||||
protected ILogger<DashboardPageBase>? Logger { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// The most recent gateway metric snapshot. Synchronously seeded from
|
||||
/// <see cref="IDashboardSnapshotService.GetSnapshot"/> for the very
|
||||
/// first render, then refreshed by hub push.
|
||||
/// <see cref="IDashboardSnapshotService.GetSnapshot"/> for the very first
|
||||
/// render, then refreshed from the feed.
|
||||
/// </summary>
|
||||
protected DashboardSnapshot? Snapshot { get; private set; }
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async Task OnInitializedAsync()
|
||||
protected override Task OnInitializedAsync()
|
||||
{
|
||||
Snapshot = SnapshotService.GetSnapshot();
|
||||
await ConnectHubAsync().ConfigureAwait(false);
|
||||
|
||||
// Deliberately not awaited: the watch loop runs for the lifetime of the page
|
||||
// and is cancelled and drained by DisposeAsync.
|
||||
_watchTask = WatchSnapshotsAsync(_watchCancellation.Token);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>Disposes the SignalR hub connection created for this page, tolerating disposal-time errors.</summary>
|
||||
/// <summary>Cancels the snapshot subscription created for this page, tolerating disposal-time errors.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_hub is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _hub.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Disposal-time errors are best-effort.
|
||||
}
|
||||
}
|
||||
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
private async Task ConnectHubAsync()
|
||||
{
|
||||
_hub = HubFactory.Create("/hubs/snapshot");
|
||||
_hub.On<DashboardSnapshot>(DashboardSnapshotHub.SnapshotMessage, async snapshot =>
|
||||
{
|
||||
Snapshot = snapshot;
|
||||
await InvokeAsync(StateHasChanged).ConfigureAwait(false);
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
await _hub.StartAsync().ConfigureAwait(false);
|
||||
await _watchCancellation.CancelAsync().ConfigureAwait(false);
|
||||
|
||||
if (_watchTask is not null)
|
||||
{
|
||||
await _watchTask.WaitAsync(WatchDrainTimeout).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
// Accepted limitation: the abandoned loop still holds its feed subscription, so
|
||||
// the feed's idle gate stays open until it does unwind. There is no way to force
|
||||
// a detach — the loop is parked on a dispatcher that is not draining — so the
|
||||
// warning is the operator's only signal that a circuit teardown wedged.
|
||||
Logger?.LogWarning(
|
||||
"Dashboard page {Page} did not release its snapshot subscription within {Timeout}; "
|
||||
+ "the shared snapshot feed stays active until it unwinds.",
|
||||
GetType().Name,
|
||||
WatchDrainTimeout);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Hub is best-effort; the initial GetSnapshot() seed remains
|
||||
// valid and the snapshot service keeps populating its cache for
|
||||
// the next reconnect cycle.
|
||||
// Other disposal-time errors are best-effort.
|
||||
}
|
||||
|
||||
_watchCancellation.Dispose();
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
private async Task WatchSnapshotsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await foreach (DashboardSnapshot snapshot in SnapshotFeed
|
||||
.WatchAsync(cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
Snapshot = snapshot;
|
||||
await InvokeAsync(StateHasChanged).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// The page is going away.
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
// The feed is best-effort: the last rendered snapshot stays on screen and the
|
||||
// snapshot service keeps serving GetSnapshot() for the next page load. Logged
|
||||
// once here, on the way out of the loop — never per snapshot.
|
||||
Logger?.LogWarning(
|
||||
error,
|
||||
"Live snapshot updates ended for dashboard page {Page}; it keeps the last rendered snapshot.",
|
||||
GetType().Name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
@inherits LayoutComponentBase
|
||||
@using ZB.MOM.WW.Secrets.Ui
|
||||
|
||||
@* Thin layout: delegates the side-rail chassis (hamburger, brand, responsive
|
||||
collapse) to the shared ZB.MOM.WW.Theme <ThemeShell>. The nav is reproduced
|
||||
@@ -19,7 +20,20 @@
|
||||
</NavRailSection>
|
||||
<NavRailSection Title="Admin" Key="admin">
|
||||
<NavRailItem Href="/apikeys" Text="API Keys" />
|
||||
<NavRailItem Href="/admin/secrets" Text="Secrets" />
|
||||
@* Gated on the SAME policy the mounted /admin/secrets page enforces, not on a role
|
||||
literal, so nav visibility cannot drift from page access. In this host the two are
|
||||
equivalent — GatewayOptionsValidator constrains Dashboard:GroupToRole values to
|
||||
Administrator or Viewer, so the shared library's other manage-granting roles
|
||||
(secrets-manager, secrets-reveal) are unreachable here — but the policy form stays
|
||||
correct if that ever relaxes. Deliberately NOT applied to the API Keys item above:
|
||||
that page renders read-only for Viewers, so hiding its link would remove legitimate
|
||||
read access, whereas the secrets page denies a Viewer outright and its link would be
|
||||
a dead end. *@
|
||||
<AuthorizeView Policy="@SecretsAuthorization.ManagePolicy">
|
||||
<Authorized>
|
||||
<NavRailItem Href="/admin/secrets" Text="Secrets" />
|
||||
</Authorized>
|
||||
</AuthorizeView>
|
||||
<NavRailItem Href="/settings" Text="Settings" />
|
||||
</NavRailSection>
|
||||
</Nav>
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
@page "/alarms"
|
||||
@implements IAsyncDisposable
|
||||
@using Microsoft.AspNetCore.SignalR.Client
|
||||
@using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs
|
||||
@using ZB.MOM.WW.MxGateway.Server.Alarms
|
||||
@inject IDashboardLiveDataService LiveData
|
||||
@inject IOptions<GatewayOptions> GatewayOptions
|
||||
@inject DashboardHubConnectionFactory HubFactory
|
||||
@inject IGatewayAlarmService AlarmService
|
||||
|
||||
<PageTitle>Dashboard Alarms</PageTitle>
|
||||
|
||||
@@ -173,13 +172,13 @@
|
||||
private Task? _pollTask;
|
||||
|
||||
private DashboardAlarmProviderStatus _providerStatus = DashboardAlarmProviderStatus.Healthy;
|
||||
private HubConnection? _alarmsHub;
|
||||
private Task? _providerStatusTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void OnInitialized()
|
||||
{
|
||||
_pollTask = PollLoopAsync();
|
||||
_ = AttachAlarmsHubAsync();
|
||||
_providerStatusTask = ProviderStatusLoopAsync();
|
||||
}
|
||||
|
||||
private string? ProviderStatusTitle()
|
||||
@@ -189,26 +188,51 @@
|
||||
: null;
|
||||
}
|
||||
|
||||
private async Task AttachAlarmsHubAsync()
|
||||
// The badge tracks the central monitor directly rather than looping back through
|
||||
// /hubs/alarms: the alarm service is an in-process multi-subscriber fan-out, so a
|
||||
// server-rendered page needs no SignalR client, no loopback socket and no auth token.
|
||||
// Alarm rows still come from the 3-second poll below — this loop only feeds the badge.
|
||||
private async Task ProviderStatusLoopAsync()
|
||||
{
|
||||
_alarmsHub = HubFactory.Create("/hubs/alarms");
|
||||
_alarmsHub.On<AlarmFeedMessage>(AlarmsHub.AlarmMessage, async message =>
|
||||
while (!_cts.IsCancellationRequested)
|
||||
{
|
||||
if (message.PayloadCase == AlarmFeedMessage.PayloadOneofCase.ProviderStatus)
|
||||
try
|
||||
{
|
||||
_providerStatus = DashboardAlarmProviderStatus.FromFeed(message);
|
||||
await InvokeAsync(StateHasChanged).ConfigureAwait(false);
|
||||
}
|
||||
});
|
||||
await foreach (AlarmFeedMessage message in AlarmService
|
||||
.StreamAsync(alarmFilterPrefix: null, _cts.Token)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
if (message.PayloadCase != AlarmFeedMessage.PayloadOneofCase.ProviderStatus)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await _alarmsHub.StartAsync(_cts.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// The badge is best-effort; it stays at the healthy default until
|
||||
// the hub reconnects and delivers a fresh provider-status message.
|
||||
await InvokeAsync(() =>
|
||||
{
|
||||
_providerStatus = DashboardAlarmProviderStatus.FromFeed(message);
|
||||
StateHasChanged();
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// The monitor completes a subscriber's stream when it falls behind, and
|
||||
// again when the monitor restarts. Both are recoverable by resubscribing;
|
||||
// the badge holds its last value in the meantime.
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), _cts.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -312,18 +336,6 @@
|
||||
{
|
||||
await _cts.CancelAsync();
|
||||
|
||||
if (_alarmsHub is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _alarmsHub.DisposeAsync();
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Disposal-time errors are best-effort.
|
||||
}
|
||||
}
|
||||
|
||||
if (_pollTask is not null)
|
||||
{
|
||||
try
|
||||
@@ -335,6 +347,17 @@
|
||||
}
|
||||
}
|
||||
|
||||
if (_providerStatusTask is not null)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _providerStatusTask;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
_cts.Dispose();
|
||||
GC.SuppressFinalize(this);
|
||||
}
|
||||
|
||||
@@ -133,8 +133,10 @@ else
|
||||
</div>
|
||||
|
||||
<div class="mt-3">
|
||||
<button type="submit" class="btn btn-success btn-sm me-1" disabled="@IsBusy">Save</button>
|
||||
<button type="button" class="btn btn-outline-secondary btn-sm" disabled="@IsBusy" @onclick="CloseCreateDialog">Cancel</button>
|
||||
<div class="btn-group btn-group-sm" role="group" aria-label="Create API key actions">
|
||||
<button type="submit" class="btn btn-success" disabled="@IsBusy">Save</button>
|
||||
<button type="button" class="btn btn-outline-secondary" disabled="@IsBusy" @onclick="CloseCreateDialog">Cancel</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -44,7 +44,8 @@ else
|
||||
</div>
|
||||
@if (!string.IsNullOrWhiteSpace(Snapshot.Galaxy.LastError))
|
||||
{
|
||||
<div class="empty-state mt-2">@Snapshot.Galaxy.LastError</div>
|
||||
@* Overview stays compact; the Galaxy page renders the error in full. *@
|
||||
<div class="empty-state mt-2" title="@Snapshot.Galaxy.LastError">@DashboardDisplay.Abbreviate(Snapshot.Galaxy.LastError, 160)</div>
|
||||
}
|
||||
</section>
|
||||
|
||||
|
||||
@@ -131,11 +131,6 @@ else
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="text-secondary small mt-2">
|
||||
Browse data is served by the <code>galaxy_repository.v1.GalaxyRepository</code> gRPC
|
||||
service. Clients call <code>DiscoverHierarchy</code> for the full tree and
|
||||
<code>GetLastDeployTime</code> to detect redeployments.
|
||||
</div>
|
||||
</section>
|
||||
}
|
||||
|
||||
|
||||
+112
-43
@@ -1,11 +1,11 @@
|
||||
@page "/sessions/{SessionId}"
|
||||
@inherits DashboardPageBase
|
||||
@implements IAsyncDisposable
|
||||
@using Microsoft.AspNetCore.SignalR.Client
|
||||
@using ZB.MOM.WW.MxGateway.Contracts.Proto
|
||||
@using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs
|
||||
@inject AuthenticationStateProvider AuthenticationStateProvider
|
||||
@inject IDashboardSessionAdminService SessionAdminService
|
||||
@inject IDashboardSessionEventSubscriber EventSubscriber
|
||||
|
||||
<PageTitle>Dashboard Session</PageTitle>
|
||||
|
||||
@@ -157,7 +157,18 @@ else
|
||||
private DashboardSessionSummary? CurrentSession => Snapshot?.Sessions.FirstOrDefault(session =>
|
||||
string.Equals(session.SessionId, SessionId, StringComparison.Ordinal));
|
||||
|
||||
private HubConnection? _eventsHub;
|
||||
// Upper bound on waiting for the event pump while detaching, mirroring
|
||||
// DashboardPageBase's snapshot-watch drain: the pump marshals renders through the
|
||||
// renderer's dispatcher and a detach can run on that same dispatcher, so the wait
|
||||
// is bounded rather than unconditional.
|
||||
private static readonly TimeSpan EventPumpDrainTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
// Written only on the renderer's dispatcher (the lifecycle methods below), and read
|
||||
// on it from inside the pump's dispatched callback — that pairing is what makes the
|
||||
// stale-batch guard in PumpEventsAsync reliable.
|
||||
private IDashboardEventSubscription? _eventSubscription;
|
||||
private CancellationTokenSource? _eventPumpCancellation;
|
||||
private Task? _eventPumpTask;
|
||||
private bool _eventsConnected;
|
||||
private string? _subscribedSessionId;
|
||||
private readonly LinkedList<MxEvent> _recentEvents = new();
|
||||
@@ -183,8 +194,11 @@ else
|
||||
{
|
||||
if (!string.Equals(_subscribedSessionId, SessionId, StringComparison.Ordinal))
|
||||
{
|
||||
await DetachEventsHubAsync().ConfigureAwait(false);
|
||||
await AttachEventsHubAsync().ConfigureAwait(false);
|
||||
// Deliberately no ConfigureAwait(false): the resumption must stay on the
|
||||
// renderer's dispatcher so the new subscription is published to
|
||||
// _eventSubscription from the same thread the pump's guard reads it on.
|
||||
await DetachEventsAsync();
|
||||
AttachEvents();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -261,68 +275,123 @@ else
|
||||
string ConfirmButtonClass,
|
||||
Func<System.Security.Claims.ClaimsPrincipal, Task<DashboardSessionAdminResult>> Action);
|
||||
|
||||
private async Task AttachEventsHubAsync()
|
||||
// The dashboard runs in the same process as the event mirror, so this page reads
|
||||
// the session's mirrored events straight from it. It used to open a loopback
|
||||
// SignalR connection to /hubs/events — mint a hub token, negotiate, hold a
|
||||
// WebSocket, serialize every event — to reach data already sitting in memory.
|
||||
// IDashboardSessionEventSubscriber resolves to the same singleton that serves
|
||||
// IDashboardEventBroadcaster, and the subscription registers with
|
||||
// EventsHubViewerRegistry, so the "nobody is watching" gate keeps working for
|
||||
// both audiences.
|
||||
// ACL posture is unchanged from the hub path: any dashboard Viewer may watch
|
||||
// any session (SEC-25 tracks the per-session ACL for both seams).
|
||||
private void AttachEvents()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(SessionId))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_eventsHub = HubFactory.Create("/hubs/events");
|
||||
_eventsHub.On<MxEvent>(EventsHub.EventMessage, async mxEvent =>
|
||||
{
|
||||
_recentEvents.AddFirst(mxEvent);
|
||||
while (_recentEvents.Count > MaxRecentEvents)
|
||||
{
|
||||
_recentEvents.RemoveLast();
|
||||
}
|
||||
_eventSubscription = EventSubscriber.Subscribe(SessionId);
|
||||
_eventPumpCancellation = new CancellationTokenSource();
|
||||
_eventsConnected = true;
|
||||
_subscribedSessionId = SessionId;
|
||||
|
||||
await InvokeAsync(StateHasChanged).ConfigureAwait(false);
|
||||
});
|
||||
|
||||
_eventsHub.Closed += _ =>
|
||||
{
|
||||
_eventsConnected = false;
|
||||
return InvokeAsync(StateHasChanged);
|
||||
};
|
||||
_eventsHub.Reconnected += _ =>
|
||||
{
|
||||
_eventsConnected = true;
|
||||
return InvokeAsync(StateHasChanged);
|
||||
};
|
||||
// Deliberately not awaited: the pump runs for as long as the page watches this
|
||||
// session and is cancelled and drained by DetachEventsAsync.
|
||||
_eventPumpTask = PumpEventsAsync(_eventSubscription, _eventPumpCancellation.Token);
|
||||
}
|
||||
|
||||
private async Task PumpEventsAsync(IDashboardEventSubscription subscription, CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await _eventsHub.StartAsync().ConfigureAwait(false);
|
||||
await _eventsHub.SendAsync("SubscribeSession", SessionId).ConfigureAwait(false);
|
||||
_eventsConnected = true;
|
||||
_subscribedSessionId = SessionId;
|
||||
while (await subscription.Reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
// Drain what is queued and render once: a burst costs one render pass,
|
||||
// not one per event. Reading past the display cap would be wasted work,
|
||||
// and anything left queued is picked up on the next pass.
|
||||
List<MxEvent> batch = new();
|
||||
while (batch.Count < MaxRecentEvents && subscription.Reader.TryRead(out MxEvent? mxEvent))
|
||||
{
|
||||
batch.Add(mxEvent);
|
||||
}
|
||||
|
||||
if (batch.Count == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
await InvokeAsync(() =>
|
||||
{
|
||||
// The batch was read before this callback was dispatched, and a
|
||||
// session switch can land in between. Rendering it then would show
|
||||
// the previous session's events under the new session's heading, so
|
||||
// a batch whose subscription is no longer the live one is dropped.
|
||||
// Safe as an unsynchronized read: _eventSubscription is written on
|
||||
// this same dispatcher.
|
||||
if (!ReferenceEquals(_eventSubscription, subscription))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (MxEvent mxEvent in batch)
|
||||
{
|
||||
_recentEvents.AddFirst(mxEvent);
|
||||
}
|
||||
|
||||
while (_recentEvents.Count > MaxRecentEvents)
|
||||
{
|
||||
_recentEvents.RemoveLast();
|
||||
}
|
||||
|
||||
StateHasChanged();
|
||||
}).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
_eventsConnected = false;
|
||||
// The page navigated to another session or was disposed.
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// Either the renderer went away mid-dispatch, or the drain below timed out
|
||||
// and disposed the cancellation source this loop is still reading.
|
||||
}
|
||||
}
|
||||
|
||||
private async Task DetachEventsHubAsync()
|
||||
private async Task DetachEventsAsync()
|
||||
{
|
||||
HubConnection? hub = _eventsHub;
|
||||
_eventsHub = null;
|
||||
IDashboardEventSubscription? subscription = _eventSubscription;
|
||||
CancellationTokenSource? cancellation = _eventPumpCancellation;
|
||||
Task? pump = _eventPumpTask;
|
||||
_eventSubscription = null;
|
||||
_eventPumpCancellation = null;
|
||||
_eventPumpTask = null;
|
||||
_eventsConnected = false;
|
||||
_subscribedSessionId = null;
|
||||
_recentEvents.Clear();
|
||||
|
||||
if (hub is not null)
|
||||
// Cancel and drop the subscription before draining. Disposing it releases the
|
||||
// viewer registration — the whole point of the gate — and completes the channel,
|
||||
// so the pump has an exit even if cancellation is missed.
|
||||
cancellation?.Cancel();
|
||||
subscription?.Dispose();
|
||||
|
||||
try
|
||||
{
|
||||
try
|
||||
if (pump is not null)
|
||||
{
|
||||
await hub.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Disposal-time errors are best-effort.
|
||||
await pump.WaitAsync(EventPumpDrainTimeout);
|
||||
}
|
||||
}
|
||||
catch
|
||||
{
|
||||
// Detach-time errors (including a drain timeout) are best-effort.
|
||||
}
|
||||
|
||||
// Disposed after the drain so the pump is no longer reading the token.
|
||||
cancellation?.Dispose();
|
||||
}
|
||||
|
||||
private static string EventStatusLabel(MxEvent evt)
|
||||
@@ -334,7 +403,7 @@ else
|
||||
|
||||
public new async ValueTask DisposeAsync()
|
||||
{
|
||||
await DetachEventsHubAsync().ConfigureAwait(false);
|
||||
await DetachEventsAsync();
|
||||
await base.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,7 +83,8 @@ else
|
||||
<td>@DashboardDisplay.DateTime(session.OpenedAt)</td>
|
||||
<td>@DashboardDisplay.DateTime(session.LastClientActivityAt)</td>
|
||||
<td>@DashboardDisplay.DateTime(session.LastWorkerHeartbeatAt)</td>
|
||||
<td>@DashboardDisplay.Text(session.LastFault)</td>
|
||||
@* Full text stays reachable on the tooltip and on the session detail page. *@
|
||||
<td title="@session.LastFault">@DashboardDisplay.Abbreviate(session.LastFault)</td>
|
||||
@if (CanManage)
|
||||
{
|
||||
<td>
|
||||
|
||||
@@ -26,6 +26,21 @@ else
|
||||
<tr><th scope="row">Run migrations</th><td>@Snapshot.Configuration.Authentication.RunMigrationsOnStartup</td></tr>
|
||||
<tr><th scope="row">LDAP enabled</th><td>@Snapshot.Configuration.Ldap.Enabled</td></tr>
|
||||
<tr><th scope="row">LDAP server</th><td>@Snapshot.Configuration.Ldap.Server:@Snapshot.Configuration.Ldap.Port</td></tr>
|
||||
<tr>
|
||||
<th scope="row">LDAP fallback servers</th>
|
||||
@* Rendered even when empty: "none" is the operationally interesting answer
|
||||
on a host someone believes has a backup DC configured. *@
|
||||
<td>
|
||||
@if (Snapshot.Configuration.Ldap.FallbackServers.Count == 0)
|
||||
{
|
||||
<span class="text-muted">none</span>
|
||||
}
|
||||
else
|
||||
{
|
||||
<code>@string.Join(", ", Snapshot.Configuration.Ldap.FallbackServers)</code>
|
||||
}
|
||||
</td>
|
||||
</tr>
|
||||
<tr><th scope="row">LDAP transport</th><td>@Snapshot.Configuration.Ldap.Transport</td></tr>
|
||||
<tr><th scope="row">LDAP search base</th><td><code>@Snapshot.Configuration.Ldap.SearchBase</code></td></tr>
|
||||
<tr><th scope="row">LDAP service account</th><td><code>@Snapshot.Configuration.Ldap.ServiceAccountDn</code></td></tr>
|
||||
|
||||
@@ -67,7 +67,8 @@ else
|
||||
<td><StatusBadge Text="@worker.State.ToString()" /></td>
|
||||
<td><NavLink href="@($"sessions/{Uri.EscapeDataString(worker.SessionId)}")"><code>@worker.SessionId</code></NavLink></td>
|
||||
<td>@DashboardDisplay.DateTime(worker.LastHeartbeatAt)</td>
|
||||
<td>@DashboardDisplay.Text(worker.LastFault)</td>
|
||||
@* Full text stays reachable on the tooltip and on the session detail page. *@
|
||||
<td title="@worker.LastFault">@DashboardDisplay.Abbreviate(worker.LastFault)</td>
|
||||
@if (CanManage)
|
||||
{
|
||||
<td>
|
||||
|
||||
@@ -46,9 +46,11 @@
|
||||
}
|
||||
else if (Node.LoadState == BrowseLoadState.Error)
|
||||
{
|
||||
<div class="tree-load-status text-danger">
|
||||
@* Abbreviated: sibling tree rows are nowrap inside a fixed-height
|
||||
scroller, so a full COM/SQL error would stretch the whole pane. *@
|
||||
<div class="tree-load-status text-danger" title="@Node.LoadError">
|
||||
<span class="tree-toggle tree-toggle-empty"></span>
|
||||
<span>Failed to load: @Node.LoadError</span>
|
||||
<span>Failed to load: @DashboardDisplay.Abbreviate(Node.LoadError, 60)</span>
|
||||
</div>
|
||||
}
|
||||
|
||||
|
||||
@@ -14,16 +14,18 @@
|
||||
<p class="mb-0">@Message</p>
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<button type="button" class="btn btn-outline-secondary"
|
||||
disabled="@IsBusy"
|
||||
@onclick="OnCancel">
|
||||
Cancel
|
||||
</button>
|
||||
<button type="button" class="btn @ConfirmButtonClass"
|
||||
disabled="@IsBusy"
|
||||
@onclick="OnConfirm">
|
||||
@ConfirmLabel
|
||||
</button>
|
||||
<div class="btn-group" role="group" aria-label="Confirm or cancel">
|
||||
<button type="button" class="btn btn-outline-secondary"
|
||||
disabled="@IsBusy"
|
||||
@onclick="OnCancel">
|
||||
Cancel
|
||||
</button>
|
||||
<button type="button" class="btn @ConfirmButtonClass"
|
||||
disabled="@IsBusy"
|
||||
@onclick="OnConfirm">
|
||||
@ConfirmLabel
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -25,7 +25,7 @@ else
|
||||
<td><code>@DashboardDisplay.Text(fault.SessionId)</code></td>
|
||||
<td>@(fault.WorkerProcessId?.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? "-")</td>
|
||||
<td><StatusBadge Text="@fault.State" /></td>
|
||||
<td>@fault.Message</td>
|
||||
<td title="@fault.Message">@DashboardDisplay.Abbreviate(fault.Message)</td>
|
||||
</tr>
|
||||
}
|
||||
</tbody>
|
||||
|
||||
@@ -15,13 +15,36 @@ public sealed class DashboardLiveDataService : IDashboardLiveDataService, IAsync
|
||||
{
|
||||
private const string BackendName = "Galaxy";
|
||||
private const string ClientName = "mxgateway-dashboard";
|
||||
|
||||
// One browse page of tags plus headroom. Bounds the standing advise load the
|
||||
// single dashboard worker carries — and the event churn that advise set feeds —
|
||||
// however much of a galaxy an operator browses through in one sitting.
|
||||
//
|
||||
// The bound is per-read, not absolute: a read may never evict a tag it is itself
|
||||
// about to return, so a single read of more distinct tags than the cap leaves the
|
||||
// set that large. The invariant EvictForAsync actually maintains is
|
||||
//
|
||||
// |advise set| after a read <= max(MaxSubscribedTags, distinct tags in that read)
|
||||
//
|
||||
// and any overshoot is squeezed back out by the next read that subscribes a tag
|
||||
// (see EvictForAsync). A browse page requests far fewer tags than the cap, so in
|
||||
// practice the set settles at MaxSubscribedTags.
|
||||
private const int MaxSubscribedTags = 256;
|
||||
|
||||
private static readonly TimeSpan ReadTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
private readonly ISessionManager _sessionManager;
|
||||
private readonly IGatewayAlarmService _alarmService;
|
||||
private readonly ILogger<DashboardLiveDataService> _logger;
|
||||
private readonly SemaphoreSlim _gate = new(1, 1);
|
||||
private readonly HashSet<string> _subscribed = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
// Least-recently-read-last advise set: the list holds every currently advised
|
||||
// tag ordered most- to least-recently read, the dictionary indexes into it.
|
||||
// Both are only ever touched under _gate, which already serialises all viewers.
|
||||
private readonly Dictionary<string, LinkedListNode<SubscribedTag>> _subscribed =
|
||||
new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
private readonly LinkedList<SubscribedTag> _recency = new();
|
||||
|
||||
private GatewaySession? _session;
|
||||
private int _serverHandle;
|
||||
@@ -58,15 +81,15 @@ public sealed class DashboardLiveDataService : IDashboardLiveDataService, IAsync
|
||||
(GatewaySession session, int serverHandle) = await EnsureReadyAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
string[] toSubscribe = tagAddresses.Where(tag => !_subscribed.Contains(tag)).ToArray();
|
||||
string[] toSubscribe = TouchAndCollectNewTags(tagAddresses, out int justReadCount);
|
||||
if (toSubscribe.Length > 0)
|
||||
{
|
||||
await session.SubscribeBulkAsync(serverHandle, toSubscribe, cancellationToken)
|
||||
await EvictForAsync(session, serverHandle, toSubscribe.Length, justReadCount, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
foreach (string tag in toSubscribe)
|
||||
{
|
||||
_subscribed.Add(tag);
|
||||
}
|
||||
IReadOnlyList<SubscribeResult> subscribeResults = await session
|
||||
.SubscribeBulkAsync(serverHandle, toSubscribe, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
TrackSubscribed(toSubscribe, subscribeResults);
|
||||
}
|
||||
|
||||
IReadOnlyList<BulkReadResult> results = await session
|
||||
@@ -107,6 +130,148 @@ public sealed class DashboardLiveDataService : IDashboardLiveDataService, IAsync
|
||||
return Task.FromResult(new DashboardAlarmQueryResult(alarms, error, _alarmService.WorkerProcessId));
|
||||
}
|
||||
|
||||
// Promotes every already-advised tag in this read to the front of the recency
|
||||
// list and returns the tags that still need subscribing (distinct, in request
|
||||
// order). `justReadCount` is how many distinct tags of this read were already
|
||||
// advised — they now occupy the front of the list and must never be evicted to
|
||||
// make room for the same read's new tags. Callers must hold _gate.
|
||||
//
|
||||
// Every tag of one read is equally recently read; the recency list needs a total
|
||||
// order anyway, so the whole service uses one tie-break: later in the request wins.
|
||||
// Promoting in request order gives that here, and TrackSubscribed inserts new tags
|
||||
// the same way.
|
||||
private string[] TouchAndCollectNewTags(IReadOnlyCollection<string> tagAddresses, out int justReadCount)
|
||||
{
|
||||
int touched = 0;
|
||||
List<string> toSubscribe = [];
|
||||
HashSet<string> seen = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (string tag in tagAddresses)
|
||||
{
|
||||
if (_subscribed.TryGetValue(tag, out LinkedListNode<SubscribedTag>? node))
|
||||
{
|
||||
if (!ReferenceEquals(node, _recency.First))
|
||||
{
|
||||
_recency.Remove(node);
|
||||
_recency.AddFirst(node);
|
||||
}
|
||||
|
||||
if (seen.Add(tag))
|
||||
{
|
||||
touched++;
|
||||
}
|
||||
}
|
||||
else if (seen.Add(tag))
|
||||
{
|
||||
toSubscribe.Add(tag);
|
||||
}
|
||||
}
|
||||
|
||||
justReadCount = touched;
|
||||
return [.. toSubscribe];
|
||||
}
|
||||
|
||||
// Drops least-recently-read tags off the back of the advise set until the
|
||||
// incoming tags fit under MaxSubscribedTags, unadvising them on the worker in
|
||||
// one batch. A failed unadvise must not fail the read: the tags are dropped
|
||||
// from tracking regardless, and the session-invalidation path already handles
|
||||
// gateway/worker drift. Callers must hold _gate.
|
||||
//
|
||||
// Eviction stops at the tags this read just touched (`justReadCount`), so a read
|
||||
// whose own distinct tags outnumber the cap ends over it — see MaxSubscribedTags
|
||||
// for the exact invariant. That overshoot is not sticky: the next read that
|
||||
// subscribes anything computes `overflow` against the oversized set and evicts the
|
||||
// whole excess in one pass (a 300-tag set plus one new tag evicts 45 and lands
|
||||
// back at the cap). A read that subscribes nothing new evicts nothing, but it also
|
||||
// cannot grow the set.
|
||||
//
|
||||
// Cancellation mid-eviction follows this file's policy: OperationCanceledException
|
||||
// is deliberately not caught here or in ReadAsync, so it propagates with the tags
|
||||
// already dropped from tracking — the same end state as a failed unadvise.
|
||||
private async Task EvictForAsync(
|
||||
GatewaySession session,
|
||||
int serverHandle,
|
||||
int incomingCount,
|
||||
int justReadCount,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
int overflow = _subscribed.Count + incomingCount - MaxSubscribedTags;
|
||||
int evictable = _subscribed.Count - justReadCount;
|
||||
int evictCount = Math.Min(overflow, evictable);
|
||||
if (evictCount <= 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
List<int> evictedHandles = new(evictCount);
|
||||
for (int i = 0; i < evictCount && _recency.Last is { } oldest; i++)
|
||||
{
|
||||
_recency.RemoveLast();
|
||||
_subscribed.Remove(oldest.Value.TagAddress);
|
||||
if (oldest.Value.ItemHandle != 0)
|
||||
{
|
||||
evictedHandles.Add(oldest.Value.ItemHandle);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug(
|
||||
"Dashboard advise set hit its cap of {Cap}; evicted {EvictedCount} least-recently-read tags.",
|
||||
MaxSubscribedTags,
|
||||
evictCount);
|
||||
|
||||
if (evictedHandles.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await session.UnsubscribeBulkAsync(serverHandle, evictedHandles, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
_logger.LogDebug(
|
||||
exception,
|
||||
"Unadvising {EvictedCount} evicted dashboard tags failed; they stay dropped from tracking.",
|
||||
evictedHandles.Count);
|
||||
}
|
||||
}
|
||||
|
||||
// Records the freshly advised tags as the most recently read, keeping each
|
||||
// tag's item handle so eviction can unadvise it. Tags the worker failed to
|
||||
// advise are still tracked (matching the pre-cap behaviour of not retrying
|
||||
// them on every read) but carry no handle, so eviction just forgets them.
|
||||
// Callers must hold _gate.
|
||||
private void TrackSubscribed(IReadOnlyList<string> tagAddresses, IReadOnlyList<SubscribeResult> results)
|
||||
{
|
||||
Dictionary<string, int> handles = new(results.Count, StringComparer.OrdinalIgnoreCase);
|
||||
foreach (SubscribeResult result in results)
|
||||
{
|
||||
if (result.WasSuccessful && !string.IsNullOrEmpty(result.TagAddress))
|
||||
{
|
||||
handles[result.TagAddress] = result.ItemHandle;
|
||||
}
|
||||
}
|
||||
|
||||
// Request order, so the read's last tag ends up most recent — the same
|
||||
// tie-break TouchAndCollectNewTags applies to the tags it promotes.
|
||||
foreach (string tag in tagAddresses)
|
||||
{
|
||||
handles.TryGetValue(tag, out int itemHandle);
|
||||
_subscribed[tag] = _recency.AddFirst(new SubscribedTag(tag, itemHandle));
|
||||
}
|
||||
}
|
||||
|
||||
// Forgets the whole advise set without unadvising: every call site is one where
|
||||
// the backing session (and with it every item handle) is already gone.
|
||||
// Callers must hold _gate.
|
||||
private void ClearSubscriptions()
|
||||
{
|
||||
_subscribed.Clear();
|
||||
_recency.Clear();
|
||||
}
|
||||
|
||||
// Returns a Ready session + its Register server handle, opening a fresh
|
||||
// session when none exists or the current one is no longer usable. Callers
|
||||
// must hold _gate.
|
||||
@@ -132,7 +297,7 @@ public sealed class DashboardLiveDataService : IDashboardLiveDataService, IAsync
|
||||
await CloseQuietlyAsync(existing.SessionId).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
_subscribed.Clear();
|
||||
ClearSubscriptions();
|
||||
_session = null;
|
||||
|
||||
GatewaySession session = await _sessionManager.OpenSessionAsync(
|
||||
@@ -178,7 +343,7 @@ public sealed class DashboardLiveDataService : IDashboardLiveDataService, IAsync
|
||||
{
|
||||
_session = null;
|
||||
_serverHandle = 0;
|
||||
_subscribed.Clear();
|
||||
ClearSubscriptions();
|
||||
}
|
||||
|
||||
private async Task CloseQuietlyAsync(string sessionId)
|
||||
@@ -212,4 +377,8 @@ public sealed class DashboardLiveDataService : IDashboardLiveDataService, IAsync
|
||||
|
||||
_gate.Dispose();
|
||||
}
|
||||
|
||||
// One entry of the advise set. ItemHandle is the handle the worker bound for
|
||||
// the tag, or 0 when the subscribe failed and there is nothing to unadvise.
|
||||
private readonly record struct SubscribedTag(string TagAddress, int ItemHandle);
|
||||
}
|
||||
|
||||
@@ -38,6 +38,7 @@ public static class DashboardServiceCollectionExtensions
|
||||
services.AddZbLdapAuth(configuration, "MxGateway:Ldap");
|
||||
|
||||
services.AddSingleton<IDashboardSnapshotService, DashboardSnapshotService>();
|
||||
services.AddSingleton<IDashboardSnapshotFeed, DashboardSnapshotFeed>();
|
||||
services.AddSingleton<IDashboardLiveDataService, DashboardLiveDataService>();
|
||||
services.AddSingleton<IDashboardAuthenticator, DashboardAuthenticator>();
|
||||
services.AddSingleton<IGroupRoleMapper<string>, DashboardGroupRoleMapper>();
|
||||
@@ -47,7 +48,21 @@ public static class DashboardServiceCollectionExtensions
|
||||
services.AddSingleton<HubTokenService>();
|
||||
services.AddScoped<Hubs.DashboardHubConnectionFactory>();
|
||||
services.AddScoped<IDashboardBrowseService, DashboardBrowseService>();
|
||||
services.AddSingleton<Hubs.IDashboardEventBroadcaster, Hubs.DashboardEventBroadcaster>();
|
||||
// Singleton: EventsHub instances are transient (one per hub invocation), so the
|
||||
// subscriber bookkeeping they share with the broadcaster must outlive them.
|
||||
services.AddSingleton<Hubs.EventsHubViewerRegistry>();
|
||||
|
||||
// One instance behind two interfaces, registered concretely and forwarded: the
|
||||
// publish side (IDashboardEventBroadcaster, driven by the session pipeline) and
|
||||
// the in-process subscribe side (IDashboardSessionEventSubscriber, used by the
|
||||
// session-details page) share subscriber bookkeeping, so resolving them to two
|
||||
// instances would leave the page subscribed to a mirror nobody publishes to.
|
||||
services.AddSingleton<Hubs.DashboardEventBroadcaster>();
|
||||
services.AddSingleton<Hubs.IDashboardEventBroadcaster>(
|
||||
static provider => provider.GetRequiredService<Hubs.DashboardEventBroadcaster>());
|
||||
services.AddSingleton<Hubs.IDashboardSessionEventSubscriber>(
|
||||
static provider => provider.GetRequiredService<Hubs.DashboardEventBroadcaster>());
|
||||
services.AddSingleton<Hubs.DashboardSnapshotHubConnectionCounter>();
|
||||
services.AddHostedService<Hubs.DashboardSnapshotPublisher>();
|
||||
services.AddHostedService<Hubs.AlarmsHubPublisher>();
|
||||
services.AddHttpContextAccessor();
|
||||
|
||||
@@ -0,0 +1,372 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Channels;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
|
||||
/// <summary>
|
||||
/// Fans one <see cref="IDashboardSnapshotService.WatchSnapshotsAsync"/> enumeration out to
|
||||
/// every dashboard circuit. The underlying watch is not multicast — each enumeration owns a
|
||||
/// timer and builds its own snapshot per tick — so subscribing per page would multiply the
|
||||
/// snapshot cost by the number of open pages.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The pump is idle-gated: it starts when the first subscriber arrives and is cancelled and
|
||||
/// awaited when the last subscriber <em>of the live generation</em> leaves, so an unwatched
|
||||
/// gateway runs no timer and builds no snapshots. Successive pumps are chained through
|
||||
/// <c>_pumpTask</c>, so a rapid
|
||||
/// unsubscribe/resubscribe restarts a fresh pump without ever running two enumerations at
|
||||
/// once.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Every subscriber is tagged with the pump generation it joined under, and a dying pump only
|
||||
/// ever detaches its own generation. A pump ends its generation the instant its source fails
|
||||
/// or completes — before the (possibly slow) enumerator disposal — so a subscriber arriving
|
||||
/// while a pump unwinds starts a fresh generation instead of silently attaching to a dead
|
||||
/// pump that is about to detach everybody and leave nobody watching.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class DashboardSnapshotFeed : IDashboardSnapshotFeed
|
||||
{
|
||||
/// <summary>Generation value meaning "no pump is accepting subscribers".</summary>
|
||||
private const long NoGeneration = 0;
|
||||
|
||||
private readonly IDashboardSnapshotService _snapshotService;
|
||||
private readonly ILogger<DashboardSnapshotFeed> _logger;
|
||||
private readonly object _gate = new();
|
||||
private readonly List<Subscription> _subscribers = [];
|
||||
|
||||
/// <summary>
|
||||
/// The most recent pump, completed while idle. A starting pump awaits its predecessor
|
||||
/// before enumerating, which is what guarantees a single live enumeration.
|
||||
/// </summary>
|
||||
private Task _pumpTask = Task.CompletedTask;
|
||||
|
||||
/// <summary>Cancellation for the live pump; null when no generation is accepting subscribers.</summary>
|
||||
private CancellationTokenSource? _pumpCancellation;
|
||||
|
||||
/// <summary>The generation new subscribers join, or <see cref="NoGeneration"/> when no pump is live.</summary>
|
||||
private long _generation = NoGeneration;
|
||||
|
||||
/// <summary>Last generation handed out; only ever incremented under <c>_gate</c>.</summary>
|
||||
private long _lastGeneration = NoGeneration;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="DashboardSnapshotFeed"/> class.</summary>
|
||||
/// <param name="snapshotService">Snapshot source to multicast.</param>
|
||||
/// <param name="logger">Optional logger for pump faults.</param>
|
||||
public DashboardSnapshotFeed(
|
||||
IDashboardSnapshotService snapshotService,
|
||||
ILogger<DashboardSnapshotFeed>? logger = null)
|
||||
{
|
||||
_snapshotService = snapshotService ?? throw new ArgumentNullException(nameof(snapshotService));
|
||||
_logger = logger ?? NullLogger<DashboardSnapshotFeed>.Instance;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<DashboardSnapshot> WatchAsync(
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
// Capacity 1 + DropOldest: a viewer only ever wants the latest snapshot, so a
|
||||
// circuit that renders slowly neither buffers without bound nor blocks the pump
|
||||
// (TryWrite always succeeds) — it just skips the snapshots it was too slow for.
|
||||
Channel<DashboardSnapshot> channel = Channel.CreateBounded<DashboardSnapshot>(
|
||||
new BoundedChannelOptions(1)
|
||||
{
|
||||
FullMode = BoundedChannelFullMode.DropOldest,
|
||||
SingleReader = true,
|
||||
SingleWriter = false,
|
||||
});
|
||||
|
||||
Subscription subscription = Subscribe(channel);
|
||||
try
|
||||
{
|
||||
await foreach (DashboardSnapshot snapshot in channel.Reader
|
||||
.ReadAllAsync(cancellationToken)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
yield return snapshot;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
// Untokened on purpose: teardown must run to completion even when this
|
||||
// subscriber is unwinding because its own token fired.
|
||||
await UnsubscribeAsync(subscription).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private Subscription Subscribe(Channel<DashboardSnapshot> channel)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
// A live generation is joined; otherwise this subscriber starts one. Keying on
|
||||
// "is a generation live" rather than "is this the first subscriber" is what makes
|
||||
// a subscriber arriving while a pump unwinds start a fresh pump for itself.
|
||||
long generation = _pumpCancellation is null ? StartPumpLocked() : _generation;
|
||||
Subscription subscription = new(channel, generation);
|
||||
_subscribers.Add(subscription);
|
||||
return subscription;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task UnsubscribeAsync(Subscription subscription)
|
||||
{
|
||||
CancellationTokenSource? cancellation;
|
||||
Task pump;
|
||||
lock (_gate)
|
||||
{
|
||||
if (!_subscribers.Remove(subscription))
|
||||
{
|
||||
// The pump already detached this subscription (it completed or faulted).
|
||||
return;
|
||||
}
|
||||
|
||||
if (subscription.Generation != _generation)
|
||||
{
|
||||
// This viewer belonged to a generation that has already ended. The live
|
||||
// pump — if there is one — serves other viewers and must not be cancelled
|
||||
// on their behalf; the dying pump is stopping under its own steam.
|
||||
return;
|
||||
}
|
||||
|
||||
if (HasSubscribersLocked(_generation))
|
||||
{
|
||||
// Other viewers are still watching the live generation. Counting the whole
|
||||
// list here would be wrong: subscribers of an ending generation linger in it
|
||||
// until that pump's Reset runs, and they must not hold the idle gate open.
|
||||
return;
|
||||
}
|
||||
|
||||
cancellation = _pumpCancellation;
|
||||
_pumpCancellation = null;
|
||||
_generation = NoGeneration;
|
||||
pump = _pumpTask;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
cancellation?.Cancel();
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
// The pump ended on its own and disposed its cancellation source first.
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await pump.ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// A pump fault has already been reported to the subscribers it had; the
|
||||
// unsubscribing caller is only waiting for the enumeration to stop.
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Reports whether any subscriber is still being served by a generation.</summary>
|
||||
/// <param name="generation">The generation to look for.</param>
|
||||
/// <returns>True when at least one subscriber carries that generation.</returns>
|
||||
private bool HasSubscribersLocked(long generation)
|
||||
{
|
||||
foreach (Subscription subscriber in _subscribers)
|
||||
{
|
||||
if (subscriber.Generation == generation)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Starts a pump generation. Must be called while holding <c>_gate</c>; the caller adds
|
||||
/// the subscribers that belong to the returned generation.
|
||||
/// </summary>
|
||||
/// <returns>The new generation identifier.</returns>
|
||||
private long StartPumpLocked()
|
||||
{
|
||||
long generation = ++_lastGeneration;
|
||||
CancellationTokenSource cancellation = new();
|
||||
Task previous = _pumpTask;
|
||||
_generation = generation;
|
||||
_pumpCancellation = cancellation;
|
||||
|
||||
// Task.Run, not a direct call: an async iterator runs synchronously up to its
|
||||
// first suspension, and the first pull of the underlying watch can read the API
|
||||
// key table. That must not run on the subscribing circuit's thread, let alone
|
||||
// while this lock is held.
|
||||
_pumpTask = Task.Run(() => PumpAsync(generation, previous, cancellation, cancellation.Token));
|
||||
return generation;
|
||||
}
|
||||
|
||||
private async Task PumpAsync(
|
||||
long generation,
|
||||
Task previous,
|
||||
CancellationTokenSource cancellation,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Never overlap with the enumeration this pump replaces.
|
||||
await previous.ConfigureAwait(ConfigureAwaitOptions.SuppressThrowing);
|
||||
|
||||
// Enumerated by hand rather than with await foreach so the generation can be
|
||||
// ended the moment the source fails or completes — await foreach would run the
|
||||
// enumerator's disposal first, and a subscriber arriving during that disposal
|
||||
// would join a generation that is already doomed.
|
||||
IAsyncEnumerator<DashboardSnapshot> snapshots = _snapshotService
|
||||
.WatchSnapshotsAsync(cancellationToken)
|
||||
.GetAsyncEnumerator(cancellationToken);
|
||||
try
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
bool moved;
|
||||
try
|
||||
{
|
||||
moved = await snapshots.MoveNextAsync().ConfigureAwait(false);
|
||||
}
|
||||
catch
|
||||
{
|
||||
EndGeneration(generation);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (!moved)
|
||||
{
|
||||
EndGeneration(generation);
|
||||
break;
|
||||
}
|
||||
|
||||
Broadcast(generation, snapshots.Current);
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
await snapshots.DisposeAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// The source completed on its own; hand the completion to this generation's
|
||||
// subscribers and re-arm so the next one starts a fresh enumeration.
|
||||
Reset(generation, error: null);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// The production DashboardSnapshotService swallows cancellation and yield-breaks,
|
||||
// so normal teardown exits through the fall-through above (with an ownership-checked
|
||||
// Reset that finds no subscribers); an implementation that propagates the token
|
||||
// instead exits here. Both shapes end the generation exactly once.
|
||||
EndGeneration(generation);
|
||||
}
|
||||
catch (Exception error)
|
||||
{
|
||||
_logger.LogWarning(error, "Dashboard snapshot feed stopped; the next subscriber restarts it.");
|
||||
Reset(generation, error);
|
||||
}
|
||||
finally
|
||||
{
|
||||
cancellation.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private void Broadcast(long generation, DashboardSnapshot snapshot)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
foreach (Subscription subscriber in _subscribers)
|
||||
{
|
||||
if (subscriber.Generation != generation)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// Bounded/DropOldest: always accepted unless the channel is completed.
|
||||
subscriber.Channel.Writer.TryWrite(snapshot);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops handing <paramref name="generation"/> to new subscribers. Called the instant a
|
||||
/// pump's source fails or completes, before its enumerator is disposed.
|
||||
/// </summary>
|
||||
/// <param name="generation">The generation that has ended.</param>
|
||||
private void EndGeneration(long generation)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
EndGenerationLocked(generation);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Clears the live-pump state if <paramref name="generation"/> still owns it.</summary>
|
||||
/// <param name="generation">The generation that has ended.</param>
|
||||
private void EndGenerationLocked(long generation)
|
||||
{
|
||||
if (_generation != generation)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_generation = NoGeneration;
|
||||
_pumpCancellation = null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Detaches the subscribers of a finished generation and re-arms the feed. Subscribers of
|
||||
/// any other generation are left alone — they belong to a pump that is still running (or
|
||||
/// about to), so a dying pump must not take them down with it.
|
||||
/// </summary>
|
||||
/// <param name="generation">The generation whose subscribers are being detached.</param>
|
||||
/// <param name="error">Failure to surface, or null when the source completed cleanly.</param>
|
||||
private void Reset(long generation, Exception? error)
|
||||
{
|
||||
List<Channel<DashboardSnapshot>> detached = [];
|
||||
lock (_gate)
|
||||
{
|
||||
for (int index = _subscribers.Count - 1; index >= 0; index--)
|
||||
{
|
||||
if (_subscribers[index].Generation != generation)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
detached.Add(_subscribers[index].Channel);
|
||||
_subscribers.RemoveAt(index);
|
||||
}
|
||||
|
||||
EndGenerationLocked(generation);
|
||||
|
||||
if (_subscribers.Count > 0 && _pumpCancellation is null)
|
||||
{
|
||||
// Belt and braces: subscribers left with no live pump would be frozen for
|
||||
// good, because only a subscriber that finds no generation starts one.
|
||||
long restarted = StartPumpLocked();
|
||||
foreach (Subscription subscriber in _subscribers)
|
||||
{
|
||||
subscriber.Generation = restarted;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (Channel<DashboardSnapshot> channel in detached)
|
||||
{
|
||||
channel.Writer.TryComplete(error);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>One viewer's delivery channel plus the pump generation serving it.</summary>
|
||||
/// <param name="channel">Delivery channel for this viewer.</param>
|
||||
/// <param name="generation">Pump generation this viewer joined under.</param>
|
||||
private sealed class Subscription(Channel<DashboardSnapshot> channel, long generation)
|
||||
{
|
||||
/// <summary>Gets the viewer's delivery channel.</summary>
|
||||
public Channel<DashboardSnapshot> Channel { get; } = channel;
|
||||
|
||||
/// <summary>Gets or sets the pump generation currently serving this viewer.</summary>
|
||||
public long Generation { get; set; } = generation;
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,17 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
{
|
||||
private const string HealthyStatus = "Healthy";
|
||||
|
||||
/// <summary>
|
||||
/// Minimum spacing between API key list reads. The list is a SQLite query whose
|
||||
/// content only changes when an operator creates, rotates, or revokes a key, so
|
||||
/// refreshing it on every ~1s snapshot tick buys nothing; the dashboard still sees
|
||||
/// a key change within this interval.
|
||||
/// </summary>
|
||||
private static readonly TimeSpan ApiKeySummaryRefreshInterval = TimeSpan.FromSeconds(15);
|
||||
|
||||
/// <summary>Sentinel for "the API key summaries have never been refreshed".</summary>
|
||||
private const long NeverRefreshedTicks = long.MinValue;
|
||||
|
||||
private readonly ISessionRegistry _sessionRegistry;
|
||||
private readonly GatewayMetrics _metrics;
|
||||
private readonly IGatewayConfigurationProvider _configurationProvider;
|
||||
@@ -30,6 +41,13 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
private readonly ILogger<DashboardSnapshotService> _logger;
|
||||
private readonly SemaphoreSlim _apiKeySummaryRefreshGate = new(1, 1);
|
||||
private IReadOnlyList<DashboardApiKeySummary> _apiKeySummaries = Array.Empty<DashboardApiKeySummary>();
|
||||
private long _apiKeySummariesRefreshedAtTicks = NeverRefreshedTicks;
|
||||
// The effective configuration is built from IOptions<GatewayOptions> and is startup-static:
|
||||
// the gateway binds options once at boot and never reloads them, so this projection cannot
|
||||
// change for the process lifetime. Build it once instead of re-projecting the whole option
|
||||
// tree on every snapshot tick. A racing first build is harmless — the projection is pure,
|
||||
// so either winner stores equivalent content.
|
||||
private EffectiveGatewayConfiguration? _effectiveConfiguration;
|
||||
// Memoizes ONLY the O(N) template/category breakdown against the cache sequence. The shared
|
||||
// library bumps Sequence only on a heavy refresh that replaces the object set, so an unchanged
|
||||
// sequence means the breakdown is unchanged and can be reused — keeping the ~1s snapshot tick
|
||||
@@ -100,10 +118,23 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
Metrics: CreateMetricSummaries(metricsSnapshot),
|
||||
Faults: CreateFaultSummaries(sessions, generatedAt),
|
||||
ApiKeys: Volatile.Read(ref _apiKeySummaries),
|
||||
Configuration: _configurationProvider.GetEffectiveConfiguration(),
|
||||
Configuration: ResolveEffectiveConfiguration(),
|
||||
Galaxy: ResolveGalaxySummary());
|
||||
}
|
||||
|
||||
private EffectiveGatewayConfiguration ResolveEffectiveConfiguration()
|
||||
{
|
||||
EffectiveGatewayConfiguration? cached = Volatile.Read(ref _effectiveConfiguration);
|
||||
if (cached is not null)
|
||||
{
|
||||
return cached;
|
||||
}
|
||||
|
||||
EffectiveGatewayConfiguration configuration = _configurationProvider.GetEffectiveConfiguration();
|
||||
Volatile.Write(ref _effectiveConfiguration, configuration);
|
||||
return configuration;
|
||||
}
|
||||
|
||||
private DashboardGalaxySummary ResolveGalaxySummary()
|
||||
{
|
||||
GalaxyHierarchyCacheEntry entry = _galaxyHierarchyCache.Current;
|
||||
@@ -255,6 +286,20 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
|
||||
private async Task RefreshApiKeySummariesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
DateTimeOffset now = _timeProvider.GetUtcNow();
|
||||
long lastRefreshedAtTicks = Interlocked.Read(ref _apiKeySummariesRefreshedAtTicks);
|
||||
if (lastRefreshedAtTicks != NeverRefreshedTicks
|
||||
&& now.UtcTicks - lastRefreshedAtTicks < ApiKeySummaryRefreshInterval.Ticks)
|
||||
{
|
||||
// Inside the refresh window: reuse the cached summaries rather than
|
||||
// re-reading the API key table on this tick. Only a *successful* refresh
|
||||
// moves the timestamp, so a failed read is retried on the next tick.
|
||||
// This check is deliberately outside the refresh gate, so it races
|
||||
// benignly: if two callers both read a stale timestamp, the zero-timeout
|
||||
// gate below admits one and the other returns without touching the store.
|
||||
return;
|
||||
}
|
||||
|
||||
if (!await _apiKeySummaryRefreshGate.WaitAsync(0, cancellationToken).ConfigureAwait(false))
|
||||
{
|
||||
return;
|
||||
@@ -278,6 +323,7 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
.ToArray();
|
||||
|
||||
Volatile.Write(ref _apiKeySummaries, summaries);
|
||||
Interlocked.Exchange(ref _apiKeySummariesRefreshedAtTicks, now.UtcTicks);
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Threading.Channels;
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
@@ -6,11 +7,13 @@ using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
|
||||
/// <summary>
|
||||
/// Broadcasts MxEvents to <see cref="EventsHub"/> clients subscribed to the
|
||||
/// session's group. Fire-and-forget: we hand the send to the hub context
|
||||
/// and return immediately so the source gRPC stream is never blocked.
|
||||
/// Errors are logged once and dropped — keeping the SignalR mirror best-effort
|
||||
/// preserves the gRPC contract that exists today.
|
||||
/// Broadcasts MxEvents to the two dashboard audiences for a session: remote
|
||||
/// <see cref="EventsHub"/> clients subscribed to the session's group, and
|
||||
/// in-process subscribers opened through
|
||||
/// <see cref="IDashboardSessionEventSubscriber.Subscribe"/>. Fire-and-forget: we
|
||||
/// hand the send to the hub context and return immediately so the source gRPC
|
||||
/// stream is never blocked. Errors are logged once and dropped — keeping the
|
||||
/// SignalR mirror best-effort preserves the gRPC contract that exists today.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// When <c>MxGateway:Dashboard:ShowTagValues</c> is false (the default), tag
|
||||
@@ -21,13 +24,49 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
/// the mirror independently of the still-outstanding per-session hub ACL
|
||||
/// (see <see cref="EventsHub"/>).
|
||||
/// </remarks>
|
||||
/// <param name="hubContext">Hub context used to send to the session's group.</param>
|
||||
/// <param name="viewerRegistry">
|
||||
/// Live-subscriber registry consulted before any per-event work is done. Both
|
||||
/// audiences register here — hub connections by their SignalR connection id,
|
||||
/// in-process subscriptions by a synthetic one — so the gate stays a single
|
||||
/// source of truth.
|
||||
/// </param>
|
||||
/// <param name="options">Gateway options supplying <c>Dashboard:ShowTagValues</c>.</param>
|
||||
/// <param name="logger">Logger for best-effort mirror failures.</param>
|
||||
public sealed class DashboardEventBroadcaster(
|
||||
IHubContext<EventsHub> hubContext,
|
||||
EventsHubViewerRegistry viewerRegistry,
|
||||
IOptions<GatewayOptions> options,
|
||||
ILogger<DashboardEventBroadcaster> logger) : IDashboardEventBroadcaster
|
||||
ILogger<DashboardEventBroadcaster> logger) : IDashboardEventBroadcaster, IDashboardSessionEventSubscriber
|
||||
{
|
||||
/// <summary>
|
||||
/// Queue depth per in-process subscriber. The consumer is a Blazor page
|
||||
/// rendering the newest handful of events, so a burst it cannot keep up with
|
||||
/// is dropped oldest-first rather than allowed to grow — same best-effort
|
||||
/// contract the SignalR mirror already has.
|
||||
/// </summary>
|
||||
private const int InProcessQueueCapacity = 256;
|
||||
|
||||
private readonly bool _showTagValues = options.Value.Dashboard.ShowTagValues;
|
||||
|
||||
private readonly object _syncRoot = new();
|
||||
|
||||
/// <summary>
|
||||
/// In-process subscribers per session. Values are treated as immutable once
|
||||
/// stored: a subscribe or dispose swaps in a new array under
|
||||
/// <see cref="_syncRoot"/>, so <see cref="Publish"/> can grab the reference
|
||||
/// and write to it after releasing the lock.
|
||||
/// </summary>
|
||||
private readonly Dictionary<string, InProcessSubscription[]> _inProcessSubscribers =
|
||||
new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>
|
||||
/// Total live in-process subscribers, read without the lock so the common
|
||||
/// case — nobody has a session-details page open — never contends on it.
|
||||
/// Written only under <see cref="_syncRoot"/>.
|
||||
/// </summary>
|
||||
private int _inProcessSubscriberCount;
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Publish(string sessionId, MxEvent mxEvent)
|
||||
{
|
||||
@@ -36,8 +75,22 @@ public sealed class DashboardEventBroadcaster(
|
||||
return;
|
||||
}
|
||||
|
||||
// Every session's dashboard-mirror subscriber calls Publish for every event,
|
||||
// whether or not a browser is on that session's page. Without this gate the
|
||||
// steady state — no dashboard viewer at all — still paid a deep protobuf
|
||||
// clone (redaction is on by default) plus a send to an empty SignalR group
|
||||
// per event. Bail before both.
|
||||
if (!viewerRegistry.HasViewers(sessionId))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
MxEvent outbound = _showTagValues ? mxEvent : RedactValues(mxEvent);
|
||||
|
||||
// In-process delivery first: it is synchronous, cannot throw, and must not be
|
||||
// skipped by the early return the hub send's guard clause takes.
|
||||
DeliverInProcess(sessionId, outbound);
|
||||
|
||||
// Wrap the Task acquisition in a try/catch so a hypothetical synchronous throw
|
||||
// from SendAsync (e.g. an implementation that throws before returning the Task)
|
||||
// cannot escape Publish. The interface contract is never-throw; fire-and-forget.
|
||||
@@ -68,6 +121,117 @@ public sealed class DashboardEventBroadcaster(
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IDashboardEventSubscription Subscribe(string sessionId)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(sessionId);
|
||||
|
||||
// A synthetic connection id keeps the registry's per-connection bookkeeping
|
||||
// usable for a subscriber that has no SignalR connection behind it. The
|
||||
// "inproc-" prefix cannot collide with a SignalR connection id and makes the
|
||||
// origin obvious in a debugger.
|
||||
string connectionId = "inproc-" + Guid.NewGuid().ToString("N");
|
||||
InProcessSubscription subscription = new(this, sessionId, connectionId, InProcessQueueCapacity);
|
||||
|
||||
// Register before the subscriber becomes a delivery target, exactly as
|
||||
// EventsHub.SubscribeSession registers before joining the group: the reverse
|
||||
// order would leave a window in which this subscriber is a delivery target but
|
||||
// Publish's gate still reports the session unwatched, silently dropping events
|
||||
// it should receive. The cost of this order is at worst a redaction clone that
|
||||
// reaches nobody for the width of the window.
|
||||
viewerRegistry.AddViewer(connectionId, sessionId);
|
||||
|
||||
lock (_syncRoot)
|
||||
{
|
||||
_inProcessSubscribers[sessionId] =
|
||||
_inProcessSubscribers.TryGetValue(sessionId, out InProcessSubscription[]? existing)
|
||||
? [.. existing, subscription]
|
||||
: [subscription];
|
||||
|
||||
Volatile.Write(ref _inProcessSubscriberCount, _inProcessSubscriberCount + 1);
|
||||
}
|
||||
|
||||
return subscription;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Hands the already-redacted event to every in-process subscriber of the
|
||||
/// session. Writes are non-blocking and lossy by construction, so this never
|
||||
/// stalls the caller's event pipeline.
|
||||
/// </summary>
|
||||
/// <param name="sessionId">Session the event belongs to.</param>
|
||||
/// <param name="outbound">The event as the dashboard should see it.</param>
|
||||
private void DeliverInProcess(string sessionId, MxEvent outbound)
|
||||
{
|
||||
// The gate above admits hub-only viewers too, so check for in-process
|
||||
// subscribers before touching the lock at all.
|
||||
if (Volatile.Read(ref _inProcessSubscriberCount) == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
InProcessSubscription[] subscribers;
|
||||
lock (_syncRoot)
|
||||
{
|
||||
if (!_inProcessSubscribers.TryGetValue(sessionId, out InProcessSubscription[]? found))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
subscribers = found;
|
||||
}
|
||||
|
||||
// The array is never mutated in place, so the writes happen outside the lock.
|
||||
foreach (InProcessSubscription subscriber in subscribers)
|
||||
{
|
||||
subscriber.TryWrite(outbound);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes a disposed subscription from the delivery map and releases its
|
||||
/// viewer registration. Called at most once per subscription.
|
||||
/// </summary>
|
||||
/// <param name="subscription">The subscription being disposed.</param>
|
||||
private void Unsubscribe(InProcessSubscription subscription)
|
||||
{
|
||||
// Drop the delivery target first and deregister after, mirroring
|
||||
// EventsHub.UnsubscribeSession: the mirror stays enabled for the brief overlap
|
||||
// rather than dropping events still owed to the session's other subscribers.
|
||||
lock (_syncRoot)
|
||||
{
|
||||
if (_inProcessSubscribers.TryGetValue(subscription.SessionId, out InProcessSubscription[]? existing))
|
||||
{
|
||||
InProcessSubscription[] remaining =
|
||||
[.. existing.Where(candidate => !ReferenceEquals(candidate, subscription))];
|
||||
|
||||
// Equal lengths mean it was never in this bucket, so the counter it
|
||||
// would decrement is not its own to release.
|
||||
if (remaining.Length != existing.Length)
|
||||
{
|
||||
if (remaining.Length == 0)
|
||||
{
|
||||
// Drop the key so the map does not grow one entry per session ever viewed.
|
||||
_inProcessSubscribers.Remove(subscription.SessionId);
|
||||
}
|
||||
else
|
||||
{
|
||||
_inProcessSubscribers[subscription.SessionId] = remaining;
|
||||
}
|
||||
|
||||
Volatile.Write(ref _inProcessSubscriberCount, _inProcessSubscriberCount - 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
viewerRegistry.RemoveViewer(subscription.ConnectionId, subscription.SessionId);
|
||||
|
||||
// The synthetic connection id is used once and never reconnects, so nothing
|
||||
// else will ever call ReleaseConnection for it; without this the registry
|
||||
// would retain an empty per-connection entry per subscription ever opened.
|
||||
viewerRegistry.ReleaseConnection(subscription.ConnectionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Produces a deep clone of <paramref name="source"/> with every tag-value
|
||||
/// field cleared, leaving tag reference, quality, status, and timestamps
|
||||
@@ -90,4 +254,75 @@ public sealed class DashboardEventBroadcaster(
|
||||
|
||||
return redacted;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One in-process subscriber's feed: a bounded, drop-oldest channel plus the
|
||||
/// registry bookkeeping that keeps <see cref="Publish"/>'s viewer gate honest
|
||||
/// while the feed is live.
|
||||
/// </summary>
|
||||
private sealed class InProcessSubscription : IDashboardEventSubscription
|
||||
{
|
||||
private readonly DashboardEventBroadcaster _owner;
|
||||
|
||||
private readonly Channel<MxEvent> _channel;
|
||||
|
||||
private int _disposed;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="InProcessSubscription"/> class.</summary>
|
||||
/// <param name="owner">Broadcaster to deregister from on disposal.</param>
|
||||
/// <param name="sessionId">Session whose events this subscription carries.</param>
|
||||
/// <param name="connectionId">Synthetic connection id registered with the viewer registry.</param>
|
||||
/// <param name="capacity">Queue depth before the oldest queued event is dropped.</param>
|
||||
internal InProcessSubscription(
|
||||
DashboardEventBroadcaster owner,
|
||||
string sessionId,
|
||||
string connectionId,
|
||||
int capacity)
|
||||
{
|
||||
_owner = owner;
|
||||
SessionId = sessionId;
|
||||
ConnectionId = connectionId;
|
||||
_channel = Channel.CreateBounded<MxEvent>(new BoundedChannelOptions(capacity)
|
||||
{
|
||||
// DropOldest, not Wait: a write must never block the gRPC event
|
||||
// pipeline that calls Publish, and the newest events are the ones a
|
||||
// live view wants.
|
||||
FullMode = BoundedChannelFullMode.DropOldest,
|
||||
SingleReader = true,
|
||||
SingleWriter = false,
|
||||
});
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public ChannelReader<MxEvent> Reader => _channel.Reader;
|
||||
|
||||
/// <summary>Gets the session this subscription is watching.</summary>
|
||||
internal string SessionId { get; }
|
||||
|
||||
/// <summary>Gets the synthetic connection id held in the viewer registry.</summary>
|
||||
internal string ConnectionId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Queues an event for the subscriber, dropping the oldest queued event when
|
||||
/// the reader has fallen behind. Never blocks and never throws.
|
||||
/// </summary>
|
||||
/// <param name="mxEvent">The event to queue.</param>
|
||||
internal void TryWrite(MxEvent mxEvent) => _channel.Writer.TryWrite(mxEvent);
|
||||
|
||||
/// <summary>
|
||||
/// Deregisters the subscription and completes its channel so a reader's
|
||||
/// loop ends. Idempotent — a second call does nothing, so it can never
|
||||
/// release a viewer count that a sibling subscription owns.
|
||||
/// </summary>
|
||||
public void Dispose()
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) != 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_owner.Unsubscribe(this);
|
||||
_channel.Writer.TryComplete();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,8 +9,15 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
/// immediately via <see cref="OnConnectedAsync"/>; subsequent refreshes are
|
||||
/// broadcast by <see cref="DashboardSnapshotPublisher"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Connections are counted into <see cref="DashboardSnapshotHubConnectionCounter"/>
|
||||
/// so <see cref="DashboardSnapshotPublisher"/> can stop building and broadcasting
|
||||
/// snapshots while nobody is watching.
|
||||
/// </remarks>
|
||||
[Authorize(Policy = DashboardAuthenticationDefaults.HubClientsPolicy)]
|
||||
public sealed class DashboardSnapshotHub(IDashboardSnapshotService snapshotService) : Hub
|
||||
public sealed class DashboardSnapshotHub(
|
||||
IDashboardSnapshotService snapshotService,
|
||||
DashboardSnapshotHubConnectionCounter connectionCounter) : Hub
|
||||
{
|
||||
/// <summary>Method name used to push snapshot updates to clients.</summary>
|
||||
public const string SnapshotMessage = "SnapshotUpdated";
|
||||
@@ -18,7 +25,17 @@ public sealed class DashboardSnapshotHub(IDashboardSnapshotService snapshotServi
|
||||
/// <inheritdoc />
|
||||
public override async Task OnConnectedAsync()
|
||||
{
|
||||
// Count the viewer before seeding it, so the publisher resumes its tick
|
||||
// no later than the first snapshot this connection renders.
|
||||
connectionCounter.Increment();
|
||||
await Clients.Caller.SendAsync(SnapshotMessage, snapshotService.GetSnapshot()).ConfigureAwait(false);
|
||||
await base.OnConnectedAsync().ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override async Task OnDisconnectedAsync(Exception? exception)
|
||||
{
|
||||
connectionCounter.Decrement();
|
||||
await base.OnDisconnectedAsync(exception).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
|
||||
/// <summary>
|
||||
/// Process-wide count of live <see cref="DashboardSnapshotHub"/> connections.
|
||||
/// Registered as a singleton and read by <see cref="DashboardSnapshotPublisher"/>
|
||||
/// to idle-gate the snapshot tick: with no dashboard connected there is nothing
|
||||
/// to broadcast to, so no snapshot is built.
|
||||
/// </summary>
|
||||
public sealed class DashboardSnapshotHubConnectionCounter
|
||||
{
|
||||
private int _count;
|
||||
|
||||
/// <summary>Gets the number of live snapshot hub connections.</summary>
|
||||
public int Count => Volatile.Read(ref _count);
|
||||
|
||||
/// <summary>Records a new snapshot hub connection.</summary>
|
||||
/// <returns>The connection count after the increment.</returns>
|
||||
public int Increment()
|
||||
{
|
||||
return Interlocked.Increment(ref _count);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records a snapshot hub disconnection, clamped at zero: SignalR can invoke
|
||||
/// <c>OnDisconnectedAsync</c> for a connection whose <c>OnConnectedAsync</c>
|
||||
/// faulted, and a negative count would idle-gate the publisher while viewers
|
||||
/// are still attached.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The clamp is applied inside the compare-and-swap rather than as a repair
|
||||
/// afterwards. Decrementing first and then correcting a negative result races:
|
||||
/// two unmatched decrements from zero would both plan a repair, a real
|
||||
/// connection could increment in between, and the stale repair would then
|
||||
/// overwrite that live connection's increment — freezing a real viewer's
|
||||
/// dashboard behind the idle gate. Reading, clamping, and publishing as one
|
||||
/// atomic step means a lost race simply retries against the fresh value.
|
||||
/// </remarks>
|
||||
/// <returns>The connection count after the decrement.</returns>
|
||||
public int Decrement()
|
||||
{
|
||||
int current;
|
||||
int next;
|
||||
do
|
||||
{
|
||||
current = Volatile.Read(ref _count);
|
||||
next = current > 0 ? current - 1 : 0;
|
||||
}
|
||||
while (Interlocked.CompareExchange(ref _count, next, current) != current);
|
||||
|
||||
return next;
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,7 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
/// gateway process; clients listen via the hub.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <see cref="ExecuteAsync"/> wraps the snapshot subscription in
|
||||
/// a reconnect loop with a configurable retry delay (5s by default,
|
||||
/// mirroring <see cref="AlarmsHubPublisher"/>). A transient failure inside
|
||||
@@ -16,44 +17,67 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
/// one-time logger-init failure or a transient SQL error from the Galaxy
|
||||
/// summary projection — would otherwise end the BackgroundService with no
|
||||
/// reconnect, taking the dashboard offline until process restart.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The loop is idle-gated on <see cref="DashboardSnapshotHubConnectionCounter"/>.
|
||||
/// Each snapshot costs a session-registry snapshot and sort, a metrics snapshot
|
||||
/// that copies dictionaries under the global metrics lock, and (periodically) a
|
||||
/// SQLite read of the API key table — work with no consumer when no dashboard is
|
||||
/// connected. While the count is zero the publisher does not advance the snapshot
|
||||
/// enumerator at all, so the producing iterator stays suspended and builds nothing.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class DashboardSnapshotPublisher : BackgroundService
|
||||
{
|
||||
private static readonly TimeSpan DefaultReconnectDelay = TimeSpan.FromSeconds(5);
|
||||
private static readonly TimeSpan DefaultIdlePollInterval = TimeSpan.FromSeconds(1);
|
||||
|
||||
private readonly IDashboardSnapshotService _snapshotService;
|
||||
private readonly IHubContext<DashboardSnapshotHub> _hubContext;
|
||||
private readonly DashboardSnapshotHubConnectionCounter _connectionCounter;
|
||||
private readonly ILogger<DashboardSnapshotPublisher> _logger;
|
||||
private readonly TimeSpan _reconnectDelay;
|
||||
private readonly TimeSpan _idlePollInterval;
|
||||
|
||||
/// <summary>Initializes a new instance of the DashboardSnapshotPublisher class.</summary>
|
||||
/// <param name="snapshotService">The snapshot service to subscribe to.</param>
|
||||
/// <param name="hubContext">The SignalR hub context for broadcasting.</param>
|
||||
/// <param name="connectionCounter">Live snapshot hub connection count used to idle-gate the tick.</param>
|
||||
/// <param name="logger">The logger instance.</param>
|
||||
public DashboardSnapshotPublisher(
|
||||
IDashboardSnapshotService snapshotService,
|
||||
IHubContext<DashboardSnapshotHub> hubContext,
|
||||
DashboardSnapshotHubConnectionCounter connectionCounter,
|
||||
ILogger<DashboardSnapshotPublisher> logger)
|
||||
: this(snapshotService, hubContext, logger, DefaultReconnectDelay)
|
||||
: this(snapshotService, hubContext, connectionCounter, logger, DefaultReconnectDelay, DefaultIdlePollInterval)
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>Initializes a new instance of the DashboardSnapshotPublisher class with custom reconnect delay.</summary>
|
||||
/// <remarks>Internal hook for testing: tests inject a very short reconnect delay so assertions don't wait full 5s.</remarks>
|
||||
/// <summary>Initializes a new instance of the DashboardSnapshotPublisher class with custom cadences.</summary>
|
||||
/// <remarks>
|
||||
/// Internal hook for testing: tests inject a very short reconnect delay so assertions
|
||||
/// don't wait the full 5s, and a short idle poll so the resume-from-idle path is fast.
|
||||
/// </remarks>
|
||||
/// <param name="snapshotService">The snapshot service to subscribe to.</param>
|
||||
/// <param name="hubContext">The SignalR hub context for broadcasting.</param>
|
||||
/// <param name="connectionCounter">Live snapshot hub connection count used to idle-gate the tick.</param>
|
||||
/// <param name="logger">The logger instance.</param>
|
||||
/// <param name="reconnectDelay">The delay before reconnecting after a subscription failure.</param>
|
||||
/// <param name="idlePollInterval">How often the idle publisher re-checks for a connected viewer.</param>
|
||||
internal DashboardSnapshotPublisher(
|
||||
IDashboardSnapshotService snapshotService,
|
||||
IHubContext<DashboardSnapshotHub> hubContext,
|
||||
DashboardSnapshotHubConnectionCounter connectionCounter,
|
||||
ILogger<DashboardSnapshotPublisher> logger,
|
||||
TimeSpan reconnectDelay)
|
||||
TimeSpan reconnectDelay,
|
||||
TimeSpan idlePollInterval)
|
||||
{
|
||||
_snapshotService = snapshotService;
|
||||
_hubContext = hubContext;
|
||||
_connectionCounter = connectionCounter;
|
||||
_logger = logger;
|
||||
_reconnectDelay = reconnectDelay;
|
||||
_idlePollInterval = idlePollInterval;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
@@ -66,15 +90,31 @@ public sealed class DashboardSnapshotPublisher : BackgroundService
|
||||
{
|
||||
try
|
||||
{
|
||||
await foreach (DashboardSnapshot snapshot in _snapshotService
|
||||
// Enumerated by hand rather than with await foreach: the snapshot is
|
||||
// built inside the producer's MoveNextAsync, so not calling MoveNextAsync
|
||||
// is what makes the idle gate skip the build and not just the broadcast.
|
||||
await using IAsyncEnumerator<DashboardSnapshot> snapshots = _snapshotService
|
||||
.WatchSnapshotsAsync(stoppingToken)
|
||||
.ConfigureAwait(false))
|
||||
.GetAsyncEnumerator(stoppingToken);
|
||||
|
||||
while (!stoppingToken.IsCancellationRequested)
|
||||
{
|
||||
if (stoppingToken.IsCancellationRequested)
|
||||
if (_connectionCounter.Count == 0)
|
||||
{
|
||||
// Nobody is watching: leave the producer suspended and re-check
|
||||
// shortly. The first viewer to connect resumes the tick, and is
|
||||
// seeded directly by the hub's OnConnectedAsync meanwhile.
|
||||
await Task.Delay(_idlePollInterval, stoppingToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!await snapshots.MoveNextAsync().ConfigureAwait(false))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
DashboardSnapshot snapshot = snapshots.Current;
|
||||
|
||||
try
|
||||
{
|
||||
await _hubContext.Clients
|
||||
|
||||
@@ -9,8 +9,14 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
/// session; <see cref="DashboardEventBroadcaster"/> sends messages to
|
||||
/// <c>session:{id}</c> as events arrive from the live gRPC stream.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Group membership is mirrored into <see cref="EventsHubViewerRegistry"/>
|
||||
/// because SignalR does not expose it, and the broadcaster consults the
|
||||
/// registry to skip all mirror work for sessions nobody is watching.
|
||||
/// </remarks>
|
||||
/// <param name="viewerRegistry">Registry tracking which sessions have live subscribers.</param>
|
||||
[Authorize(Policy = DashboardAuthenticationDefaults.HubClientsPolicy)]
|
||||
public sealed class EventsHub : Hub
|
||||
public sealed class EventsHub(EventsHubViewerRegistry viewerRegistry) : Hub
|
||||
{
|
||||
/// <summary>Method name used to push individual <c>MxEvent</c> values to clients.</summary>
|
||||
public const string EventMessage = "MxEvent";
|
||||
@@ -55,19 +61,43 @@ public sealed class EventsHub : Hub
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
// Register before joining the group: the reverse order would leave a window
|
||||
// in which this connection is a group member but the broadcaster's gate still
|
||||
// reports the session unwatched, silently dropping events it should receive.
|
||||
viewerRegistry.AddViewer(Context.ConnectionId, sessionId);
|
||||
|
||||
return Groups.AddToGroupAsync(Context.ConnectionId, GroupName(sessionId));
|
||||
}
|
||||
|
||||
/// <summary>Unsubscribes the calling SignalR connection from the per-session events group.</summary>
|
||||
/// <param name="sessionId">Session id to unsubscribe the caller from.</param>
|
||||
/// <returns>A task representing the unsubscription operation.</returns>
|
||||
public Task UnsubscribeSession(string sessionId)
|
||||
public async Task UnsubscribeSession(string sessionId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sessionId))
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
return;
|
||||
}
|
||||
|
||||
return Groups.RemoveFromGroupAsync(Context.ConnectionId, GroupName(sessionId));
|
||||
// Leave the group first, deregister after — the mirror stays enabled for the
|
||||
// brief overlap rather than dropping events still owed to other subscribers.
|
||||
await Groups.RemoveFromGroupAsync(Context.ConnectionId, GroupName(sessionId)).ConfigureAwait(false);
|
||||
|
||||
viewerRegistry.RemoveViewer(Context.ConnectionId, sessionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Releases every session subscription the dropped connection held. A browser
|
||||
/// tab that closes never calls <see cref="UnsubscribeSession"/>, so without
|
||||
/// this the session would look watched forever and the mirror would keep
|
||||
/// cloning and sending events to an empty group.
|
||||
/// </summary>
|
||||
/// <param name="exception">The exception that terminated the connection, if any.</param>
|
||||
/// <returns>A task representing the disconnect handling.</returns>
|
||||
public override Task OnDisconnectedAsync(Exception? exception)
|
||||
{
|
||||
viewerRegistry.ReleaseConnection(Context.ConnectionId);
|
||||
|
||||
return base.OnDisconnectedAsync(exception);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
using System.Collections.Concurrent;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
|
||||
/// <summary>
|
||||
/// Tracks which sessions currently have at least one live <see cref="EventsHub"/>
|
||||
/// subscriber, so <see cref="DashboardEventBroadcaster"/> can skip the redaction
|
||||
/// clone and the group send for sessions nobody is watching.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// SignalR does not expose group membership, so the hub mirrors its own
|
||||
/// <c>AddToGroup</c>/<c>RemoveFromGroup</c> calls here. In the steady state no
|
||||
/// browser is on a session-details page, yet every session's dashboard-mirror
|
||||
/// subscriber still called <c>Publish</c> for every event — a deep protobuf
|
||||
/// clone (values are redacted by default) plus a send to an empty group, per
|
||||
/// event, thrown away. This registry is the cheap gate in front of that work.
|
||||
/// <para>
|
||||
/// Per-connection subscriptions are tracked as well, because a browser tab that
|
||||
/// simply goes away never calls <c>UnsubscribeSession</c>; the hub's
|
||||
/// <c>OnDisconnectedAsync</c> releases everything the connection held.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class EventsHubViewerRegistry
|
||||
{
|
||||
private readonly ConcurrentDictionary<string, int> _viewersBySession = new(StringComparer.Ordinal);
|
||||
|
||||
private readonly ConcurrentDictionary<string, ConcurrentDictionary<string, byte>> _sessionsByConnection =
|
||||
new(StringComparer.Ordinal);
|
||||
|
||||
/// <summary>
|
||||
/// Records that <paramref name="connectionId"/> is watching
|
||||
/// <paramref name="sessionId"/>. Repeat calls for the same pair are
|
||||
/// idempotent, so one <see cref="RemoveViewer"/> always clears them.
|
||||
/// </summary>
|
||||
/// <param name="connectionId">SignalR connection id of the subscriber.</param>
|
||||
/// <param name="sessionId">Session id being watched.</param>
|
||||
public void AddViewer(string connectionId, string sessionId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(connectionId) || string.IsNullOrWhiteSpace(sessionId))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
ConcurrentDictionary<string, byte> sessions = _sessionsByConnection.GetOrAdd(
|
||||
connectionId,
|
||||
static _ => new ConcurrentDictionary<string, byte>(StringComparer.Ordinal));
|
||||
|
||||
// The per-connection set is the source of truth for the count: only a
|
||||
// subscription that was genuinely new increments the session's viewers.
|
||||
if (!sessions.TryAdd(sessionId, 0))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_viewersBySession.AddOrUpdate(sessionId, 1, static (_, count) => count + 1);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records that <paramref name="connectionId"/> stopped watching
|
||||
/// <paramref name="sessionId"/>. A removal with no matching
|
||||
/// <see cref="AddViewer"/> is a no-op, so the count cannot go negative.
|
||||
/// </summary>
|
||||
/// <param name="connectionId">SignalR connection id of the subscriber.</param>
|
||||
/// <param name="sessionId">Session id no longer being watched.</param>
|
||||
public void RemoveViewer(string connectionId, string sessionId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(connectionId) || string.IsNullOrWhiteSpace(sessionId))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!_sessionsByConnection.TryGetValue(connectionId, out ConcurrentDictionary<string, byte>? sessions)
|
||||
|| !sessions.TryRemove(sessionId, out _))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
ReleaseSession(sessionId);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Releases every subscription held by <paramref name="connectionId"/>.
|
||||
/// Called from the hub's disconnect callback, which is the only reliable
|
||||
/// signal for a browser tab that closed without unsubscribing.
|
||||
/// </summary>
|
||||
/// <param name="connectionId">SignalR connection id that dropped.</param>
|
||||
public void ReleaseConnection(string connectionId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(connectionId))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
// Detaching the set is safe against a SubscribeSession that arrives after the disconnect
|
||||
// only because SignalR dispatches a connection's hub invocations sequentially by default
|
||||
// (MaximumParallelInvocationsPerClient = 1): OnDisconnectedAsync cannot overlap an
|
||||
// AddViewer for the same connection, so no late add can re-create the entry and leak a
|
||||
// count that nothing will ever release. Raising that option would break this.
|
||||
if (!_sessionsByConnection.TryRemove(connectionId, out ConcurrentDictionary<string, byte>? sessions))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (string sessionId in sessions.Keys)
|
||||
{
|
||||
// TryRemove, not a bare enumeration: a concurrent RemoveViewer on the
|
||||
// same detached set must not let the session be decremented twice.
|
||||
if (sessions.TryRemove(sessionId, out _))
|
||||
{
|
||||
ReleaseSession(sessionId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Gets a value indicating whether any hub connection is watching the session.</summary>
|
||||
/// <param name="sessionId">Session id to test.</param>
|
||||
/// <returns><see langword="true"/> when at least one connection is subscribed.</returns>
|
||||
public bool HasViewers(string sessionId) =>
|
||||
!string.IsNullOrEmpty(sessionId)
|
||||
&& _viewersBySession.TryGetValue(sessionId, out int count)
|
||||
&& count > 0;
|
||||
|
||||
/// <summary>
|
||||
/// Decrements the session's viewer count, dropping the entry entirely at
|
||||
/// zero so the dictionary does not grow one key per session ever viewed.
|
||||
/// The compare-and-swap loop keeps the decrement correct against a
|
||||
/// concurrent <see cref="AddViewer"/> on the same session.
|
||||
/// </summary>
|
||||
/// <param name="sessionId">Session id whose count is released.</param>
|
||||
private void ReleaseSession(string sessionId)
|
||||
{
|
||||
while (true)
|
||||
{
|
||||
if (!_viewersBySession.TryGetValue(sessionId, out int count))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (count <= 1)
|
||||
{
|
||||
if (_viewersBySession.TryRemove(new KeyValuePair<string, int>(sessionId, count)))
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
else if (_viewersBySession.TryUpdate(sessionId, count - 1, count))
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
using System.Threading.Channels;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
|
||||
/// <summary>
|
||||
/// A live in-process feed of one session's dashboard-mirrored MxEvents, handed
|
||||
/// out by <see cref="IDashboardSessionEventSubscriber.Subscribe"/>. Server-side
|
||||
/// Blazor components read it directly instead of looping back through
|
||||
/// <see cref="EventsHub"/> over a loopback SignalR connection.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Events are delivered exactly as a hub client would see them — the same
|
||||
/// redacted clone the group send carries, so <c>MxGateway:Dashboard:ShowTagValues</c>
|
||||
/// governs both paths identically. The feed is a bounded, lossy queue: a
|
||||
/// consumer that falls behind loses the oldest queued events, matching the
|
||||
/// best-effort contract the SignalR mirror already has. Disposing the
|
||||
/// subscription deregisters it, which is what lets the broadcaster go back to
|
||||
/// skipping all mirror work for a session nobody is watching — so callers must
|
||||
/// dispose. Dispose is idempotent.
|
||||
/// </remarks>
|
||||
public interface IDashboardEventSubscription : IDisposable
|
||||
{
|
||||
/// <summary>Gets the reader delivering this session's mirrored events.</summary>
|
||||
ChannelReader<MxEvent> Reader { get; }
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs;
|
||||
|
||||
/// <summary>
|
||||
/// In-process subscription seam on the dashboard event mirror. Implemented by
|
||||
/// <see cref="DashboardEventBroadcaster"/> alongside
|
||||
/// <see cref="IDashboardEventBroadcaster"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The interactive-server dashboard runs in the same process as the broadcaster,
|
||||
/// so a session-details page has no reason to open a loopback SignalR connection
|
||||
/// back to <see cref="EventsHub"/> — mint a hub token, negotiate, hold a
|
||||
/// WebSocket, and serialize every event — just to read events the broadcaster
|
||||
/// already holds. It subscribes here instead. The registry gate stays honest
|
||||
/// either way: an in-process subscription registers a synthetic connection id
|
||||
/// with <see cref="EventsHubViewerRegistry"/> exactly as the hub registers a real
|
||||
/// one, so <see cref="IDashboardEventBroadcaster.Publish"/> keeps skipping the
|
||||
/// redaction clone for sessions nobody is watching.
|
||||
/// <para>
|
||||
/// It is a separate interface rather than a member of
|
||||
/// <see cref="IDashboardEventBroadcaster"/> because publishing and consuming are
|
||||
/// different roles: the session pipeline only ever publishes, and its test
|
||||
/// doubles should not have to implement a subscription feed.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public interface IDashboardSessionEventSubscriber
|
||||
{
|
||||
/// <summary>Opens an in-process feed of the session's mirrored events.</summary>
|
||||
/// <param name="sessionId">Session id whose events the caller wants.</param>
|
||||
/// <returns>
|
||||
/// The subscription. Dispose it to stop the feed and release the viewer
|
||||
/// registration that keeps the mirror enabled for this session.
|
||||
/// </returns>
|
||||
IDashboardEventSubscription Subscribe(string sessionId);
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
|
||||
/// <summary>
|
||||
/// In-process multicast over <see cref="IDashboardSnapshotService.WatchSnapshotsAsync"/>.
|
||||
/// One enumeration of the underlying watch is fanned out to every subscriber, so N
|
||||
/// dashboard circuits cost one snapshot build per tick instead of N — and while nobody
|
||||
/// subscribes, nothing runs at all.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// There is no authentication or authorization gate here: the feed is reached only from
|
||||
/// Blazor dashboard components, whose endpoints already require
|
||||
/// <see cref="DashboardAuthenticationDefaults.ViewerPolicy"/>, so every caller is a circuit
|
||||
/// authorized as Viewer. Remote (non-circuit) consumers still go through
|
||||
/// <c>/hubs/snapshot</c>, which applies the hub authorization policy itself.
|
||||
/// </remarks>
|
||||
public interface IDashboardSnapshotFeed
|
||||
{
|
||||
/// <summary>
|
||||
/// Watches the shared snapshot stream. Each caller gets the snapshots produced while
|
||||
/// it is subscribed; a caller that reads slowly sees only the newest snapshot rather
|
||||
/// than a backlog, and never delays the other subscribers.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Token that ends this caller's subscription.</param>
|
||||
/// <returns>An asynchronous stream of dashboard snapshots.</returns>
|
||||
IAsyncEnumerable<DashboardSnapshot> WatchAsync(CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -15,6 +15,28 @@ public static class GatewayLogRedactor
|
||||
"WriteSecured2"
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Authorization schemes whose name may survive redaction. Anything outside this list is
|
||||
/// dropped whole: an unrecognized leading word is as likely to be credential material as it
|
||||
/// is to be a scheme, so it is not worth the leak.
|
||||
/// </summary>
|
||||
private static readonly string[] KnownAuthorizationSchemes =
|
||||
[
|
||||
"Bearer",
|
||||
"Basic",
|
||||
"Digest",
|
||||
"Negotiate",
|
||||
"NTLM",
|
||||
"ApiKey",
|
||||
"Token",
|
||||
];
|
||||
|
||||
/// <summary>Prefix identifying a gateway-issued API key.</summary>
|
||||
private const string GatewayKeyPrefix = "mxgw_";
|
||||
|
||||
/// <summary>Upper bound on a key id kept in the clear; a longer run is treated as secret material.</summary>
|
||||
private const int MaxKeyIdLength = 64;
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether a command method bears credentials.
|
||||
/// </summary>
|
||||
@@ -27,44 +49,24 @@ public static class GatewayLogRedactor
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Redacts the API key secret portion of a Bearer authorization header.
|
||||
/// Redacts the credential portion of an authorization header value.
|
||||
/// </summary>
|
||||
/// <param name="authorizationHeader">The authorization header value to redact.</param>
|
||||
/// <returns>The header with the secret portion redacted, or the original value when it is null, blank, or not a Bearer header.</returns>
|
||||
/// <returns>The header with the credential redacted, or the original value when it is null or blank.</returns>
|
||||
public static string? RedactApiKey(string? authorizationHeader)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(authorizationHeader))
|
||||
{
|
||||
return authorizationHeader;
|
||||
}
|
||||
|
||||
const string bearerPrefix = "Bearer ";
|
||||
if (!authorizationHeader.StartsWith(bearerPrefix, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return RedactedValue;
|
||||
}
|
||||
|
||||
string token = authorizationHeader[bearerPrefix.Length..].Trim();
|
||||
|
||||
if (!token.StartsWith("mxgw_", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return $"{bearerPrefix}{RedactedValue}";
|
||||
}
|
||||
|
||||
string[] tokenParts = token.Split('_', 3, StringSplitOptions.RemoveEmptyEntries);
|
||||
if (tokenParts.Length < 2)
|
||||
{
|
||||
return $"{bearerPrefix}mxgw_{RedactedValue}";
|
||||
}
|
||||
|
||||
return $"{bearerPrefix}mxgw_{tokenParts[1]}_{RedactedValue}";
|
||||
return RedactClientIdentity(authorizationHeader);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Redacts the client identity if it contains an API key.
|
||||
/// Redacts the credential carried by a client identity. Redaction fails closed: only a
|
||||
/// gateway-issued API key keeps its <c>mxgw_<key-id>_</c> shape (so operators can tell keys
|
||||
/// apart in logs), and only a recognized scheme keeps its name. Every other value — a foreign
|
||||
/// bearer token, a scheme-less string, junk — is replaced whole, because nothing that reaches
|
||||
/// this method is known to be safe to log.
|
||||
/// </summary>
|
||||
/// <param name="clientIdentity">The client identity string to redact.</param>
|
||||
/// <returns>The redacted client identity, or the original value when it contains no API key.</returns>
|
||||
/// <returns>The redacted client identity, or the original value when it is null or blank.</returns>
|
||||
public static string? RedactClientIdentity(string? clientIdentity)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(clientIdentity))
|
||||
@@ -72,9 +74,61 @@ public static class GatewayLogRedactor
|
||||
return clientIdentity;
|
||||
}
|
||||
|
||||
return clientIdentity.Contains("mxgw_", StringComparison.OrdinalIgnoreCase)
|
||||
? RedactApiKey(clientIdentity)
|
||||
: clientIdentity;
|
||||
ReadOnlySpan<char> value = clientIdentity.AsSpan().Trim();
|
||||
int separatorIndex = value.IndexOf(' ');
|
||||
|
||||
if (separatorIndex < 0)
|
||||
{
|
||||
// A single token carries no scheme, so the token itself is the credential.
|
||||
return RedactedValue;
|
||||
}
|
||||
|
||||
ReadOnlySpan<char> scheme = value[..separatorIndex];
|
||||
ReadOnlySpan<char> credential = value[(separatorIndex + 1)..].Trim();
|
||||
|
||||
if (credential.IsEmpty || !IsKnownAuthorizationScheme(scheme))
|
||||
{
|
||||
return RedactedValue;
|
||||
}
|
||||
|
||||
return credential.StartsWith(GatewayKeyPrefix, StringComparison.OrdinalIgnoreCase)
|
||||
? $"{scheme} {GatewayKeyPrefix}{RedactKeyId(credential)}"
|
||||
: $"{scheme} {RedactedValue}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Renders the trailing portion of a gateway API key: the key id when the key is well formed,
|
||||
/// otherwise nothing but the placeholder.
|
||||
/// </summary>
|
||||
/// <param name="credential">The credential, known to start with the gateway key prefix.</param>
|
||||
/// <returns>The <c><key-id>_[redacted]</c> tail, or just the placeholder.</returns>
|
||||
private static string RedactKeyId(ReadOnlySpan<char> credential)
|
||||
{
|
||||
ReadOnlySpan<char> remainder = credential[GatewayKeyPrefix.Length..];
|
||||
int secretIndex = remainder.IndexOf('_');
|
||||
|
||||
// No separator means no secret boundary to trust, so the whole remainder is treated as secret.
|
||||
return secretIndex is <= 0 or > MaxKeyIdLength
|
||||
? RedactedValue
|
||||
: $"{remainder[..secretIndex]}_{RedactedValue}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether a leading word is a recognized authorization scheme.
|
||||
/// </summary>
|
||||
/// <param name="scheme">The candidate scheme word.</param>
|
||||
/// <returns><see langword="true"/> when the word may survive redaction; otherwise <see langword="false"/>.</returns>
|
||||
private static bool IsKnownAuthorizationScheme(ReadOnlySpan<char> scheme)
|
||||
{
|
||||
foreach (string knownScheme in KnownAuthorizationSchemes)
|
||||
{
|
||||
if (scheme.Equals(knownScheme, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
+8
-4
@@ -24,12 +24,16 @@ public static class GatewayRequestLoggingMiddlewareExtensions
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(app);
|
||||
|
||||
// Resolved once at registration: the logger is keyed by category, not by request, so the
|
||||
// per-request DI resolve and logger-factory lock bought nothing.
|
||||
ILogger logger = app.ApplicationServices
|
||||
.GetRequiredService<ILoggerFactory>()
|
||||
.CreateLogger("MxGateway.Request");
|
||||
|
||||
return app.Use(async (context, next) =>
|
||||
{
|
||||
ILogger logger = context.RequestServices
|
||||
.GetRequiredService<ILoggerFactory>()
|
||||
.CreateLogger("MxGateway.Request");
|
||||
|
||||
// Scope construction is deliberately unconditional: gating it on IsEnabled would drop
|
||||
// scope state for providers (and scope consumers) registered after startup.
|
||||
using IDisposable? scope = logger.BeginGatewayScope(new GatewayLogScope(
|
||||
SessionId: ReadHeader(context, SessionIdHeaderName),
|
||||
WorkerProcessId: ReadInt32Header(context, WorkerProcessIdHeaderName),
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
using Microsoft.Extensions.Diagnostics.HealthChecks;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Diagnostics;
|
||||
|
||||
/// <summary>
|
||||
/// Reports how many MXAccess sessions are healthy. Each session is one worker process holding one
|
||||
/// MXAccess COM instance — a live connection into a Galaxy — so this is the "how many Galaxy
|
||||
/// connections are healthy" probe, expressed in the vocabulary the code actually uses.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Zero sessions is healthy, deliberately.</b> The gateway is a server: it opens a session when
|
||||
/// a client asks and holds none otherwise, so idle-with-no-clients is the normal steady state, not
|
||||
/// a fault. A count-based rule ("unhealthy below N") would sit red forever on a host nothing dials
|
||||
/// yet, and a probe that is permanently red is one people learn to ignore — which costs more than
|
||||
/// having no probe. The status here is therefore false only when a session exists and its worker
|
||||
/// has actually failed.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// This is tagged <c>active</c> rather than <c>ready</c> for the same reason. Readiness gates
|
||||
/// whether the process should receive traffic, and a gateway with no sessions is legitimately ready
|
||||
/// to serve — unlike the auth store, which every call depends on (see
|
||||
/// <see cref="AuthStoreHealthCheck"/>). Failing readiness on session state would take a working
|
||||
/// gateway out of rotation for a condition its own clients cause.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class SessionHealthCheck : IHealthCheck
|
||||
{
|
||||
private readonly ISessionRegistry _sessionRegistry;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="SessionHealthCheck"/> class.</summary>
|
||||
/// <param name="sessionRegistry">Registry holding the live sessions.</param>
|
||||
public SessionHealthCheck(ISessionRegistry sessionRegistry) =>
|
||||
_sessionRegistry = sessionRegistry ?? throw new ArgumentNullException(nameof(sessionRegistry));
|
||||
|
||||
/// <summary>Buckets the live sessions by state and grades the result.</summary>
|
||||
/// <param name="context">The health check context.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <returns>
|
||||
/// Healthy when nothing is faulted (including when no sessions are open), Degraded when some
|
||||
/// sessions are faulted but others are still usable, and Unhealthy when every session is
|
||||
/// faulted.
|
||||
/// </returns>
|
||||
public Task<HealthCheckResult> CheckHealthAsync(
|
||||
HealthCheckContext context,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
int ready = 0;
|
||||
int faulted = 0;
|
||||
int starting = 0;
|
||||
int closing = 0;
|
||||
|
||||
foreach (GatewaySession session in _sessionRegistry.Snapshot())
|
||||
{
|
||||
switch (session.State)
|
||||
{
|
||||
case SessionState.Ready:
|
||||
ready++;
|
||||
break;
|
||||
case SessionState.Faulted:
|
||||
faulted++;
|
||||
break;
|
||||
case SessionState.Closing:
|
||||
case SessionState.Closed:
|
||||
// Counted but excluded from the verdict: a session on its way out is an
|
||||
// expected lifecycle stage, not a failure, and Snapshot() still returns
|
||||
// Closed sessions until they are removed from the registry.
|
||||
closing++;
|
||||
break;
|
||||
default:
|
||||
// Creating / StartingWorker / WaitingForPipe / Handshaking /
|
||||
// InitializingWorker — mid-startup, not yet usable but not wrong.
|
||||
// Unspecified lands here too; it is the proto zero value and should not occur.
|
||||
starting++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
int total = ready + faulted + starting + closing;
|
||||
int usable = ready + starting;
|
||||
|
||||
Dictionary<string, object> data = new(StringComparer.Ordinal)
|
||||
{
|
||||
["total"] = total,
|
||||
["ready"] = ready,
|
||||
["faulted"] = faulted,
|
||||
["starting"] = starting,
|
||||
["closing"] = closing,
|
||||
};
|
||||
|
||||
HealthCheckResult result = (faulted, usable) switch
|
||||
{
|
||||
(0, _) => HealthCheckResult.Healthy(Describe(total, ready, faulted), data),
|
||||
(_, 0) => HealthCheckResult.Unhealthy(Describe(total, ready, faulted), data: data),
|
||||
_ => HealthCheckResult.Degraded(Describe(total, ready, faulted), data: data),
|
||||
};
|
||||
|
||||
return Task.FromResult(result);
|
||||
}
|
||||
|
||||
private static string Describe(int total, int ready, int faulted)
|
||||
{
|
||||
if (total == 0)
|
||||
{
|
||||
return "No MXAccess sessions are open.";
|
||||
}
|
||||
|
||||
return faulted == 0
|
||||
? $"{ready} of {total} MXAccess sessions ready."
|
||||
: $"{ready} of {total} MXAccess sessions ready, {faulted} faulted.";
|
||||
}
|
||||
}
|
||||
@@ -70,15 +70,27 @@ public static class GatewayApplication
|
||||
});
|
||||
StaticWebAssetsLoader.UseStaticWebAssets(builder.Environment, builder.Configuration);
|
||||
|
||||
ApplyDefaultSecretsStorePath(builder.Configuration);
|
||||
|
||||
// Resolve ${secret:...} references in configuration BEFORE any config consumer (TLS, Kestrel,
|
||||
// GatewayOptions/Ldap/Galaxy validators) reads a value, using a standalone secrets provider
|
||||
// (envelope-decrypted via the master key). A token referencing a missing secret fails fast
|
||||
// here (SecretNotFoundException); config with no tokens is untouched (no-op), so this is safe
|
||||
// to always run. CreateBuilder is synchronous and single-shot at bootstrap, so the two awaits
|
||||
// are driven via GetAwaiter().GetResult() (no sync-context deadlock risk during host startup).
|
||||
// The content root is passed explicitly because this container is a throwaway
|
||||
// ServiceCollection with no IHostEnvironment in it. Without it the library cannot tell
|
||||
// "no content root exists" from "no host is registered", so it skips the
|
||||
// under-content-root rule — and the migrator below CREATES the store before the real host
|
||||
// ever validates. The boot then fails a moment later, having already left an empty
|
||||
// database with its -wal/-shm siblings at the very path the rule rejects. That artifact is
|
||||
// what made the 2026-08-09 outage read as "the database is there, it's just empty".
|
||||
// DO NOT simplify this to the 3-argument overload: it still compiles, the app still boots
|
||||
// when the path is correct, and the guard silently stops running at the one moment that
|
||||
// matters.
|
||||
#pragma warning disable ASP0000 // deliberate throwaway container, disposed here, shares no singletons
|
||||
using (var secretsProvider = new ServiceCollection()
|
||||
.AddZbSecrets(builder.Configuration, "Secrets")
|
||||
.AddZbSecrets(builder.Configuration, "Secrets", builder.Environment.ContentRootPath)
|
||||
.BuildServiceProvider())
|
||||
#pragma warning restore ASP0000
|
||||
{
|
||||
@@ -106,7 +118,13 @@ public static class GatewayApplication
|
||||
.AddTypeActivatedCheck<AuthStoreHealthCheck>(
|
||||
"auth-store",
|
||||
failureStatus: null,
|
||||
tags: new[] { ZbHealthTags.Ready });
|
||||
tags: new[] { ZbHealthTags.Ready })
|
||||
// Active, not Ready: a gateway holding no sessions is legitimately ready to serve.
|
||||
// See SessionHealthCheck for why zero sessions is healthy.
|
||||
.AddTypeActivatedCheck<SessionHealthCheck>(
|
||||
"mxaccess-sessions",
|
||||
failureStatus: null,
|
||||
tags: new[] { ZbHealthTags.Active });
|
||||
builder.Services.AddSingleton<GatewayMetrics>();
|
||||
builder.AddZbTelemetry(o =>
|
||||
{
|
||||
@@ -180,6 +198,60 @@ public static class GatewayApplication
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Supplies the default location of the encrypted secrets store when nothing configured one.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The store used to default to a bare relative <c>mxgateway-secrets.db</c>, which resolves
|
||||
/// against the working directory and therefore normally lands inside the application directory.
|
||||
/// That is the shape that lost every API key on a production host: the upgrade procedure renames
|
||||
/// the application directory away, the store goes with it, and a fresh empty one appears in its
|
||||
/// place with no error. In development the same default writes a database into the source tree.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// This sets a default for an <em>unset</em> key; it never relocates a value someone configured.
|
||||
/// That distinction matters — <see cref="Configuration.GatewayConfigPathRules"/> deliberately
|
||||
/// rejects bad configured paths rather than quietly moving them, because silently relocating a
|
||||
/// credential store is worse than a boot error. Choosing where to put a value nobody specified
|
||||
/// is a different act from overriding one they did.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The location mirrors <c>AuthenticationOptions.SqlitePath</c> so both gateway stores sit
|
||||
/// together, and the mechanism is the one SEC-33 already used for
|
||||
/// <c>MxGateway:Galaxy:SnapshotCachePath</c> below — same problem, same fix, same file. It also
|
||||
/// matches what <c>docs/GatewayConfiguration.md</c> already tells operators to
|
||||
/// pass to the <c>secret</c> CLI — an absolute default also removes the CLI/gateway divergence
|
||||
/// that a working-directory-relative path can cause. On non-Windows hosts
|
||||
/// <see cref="Environment.SpecialFolder.CommonApplicationData"/> is typically not writable by a
|
||||
/// normal user, so a local run there must set <c>Secrets__SqlitePath</c> explicitly, exactly as
|
||||
/// it already must for the auth store.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>This deliberately differs from the <c>ZB.MOM.WW.Secrets</c> library default</b>, which is
|
||||
/// <see cref="Environment.SpecialFolder.LocalApplicationData"/>-derived so the family's
|
||||
/// cross-platform apps still boot locally without an override. The gateway keeps
|
||||
/// <c>CommonApplicationData</c> because it runs as a machine-wide Windows service and its other
|
||||
/// two stores — the auth database and the Galaxy snapshot — already live there; splitting them
|
||||
/// would be the greater inconsistency. The value set here always wins, so the library default is
|
||||
/// unreachable in this app. Do not "fix" the difference by deleting this method: that would
|
||||
/// silently move the store, which is the failure this whole rule exists to prevent.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="configuration">The configuration to supply the default into.</param>
|
||||
private static void ApplyDefaultSecretsStorePath(IConfiguration configuration)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(configuration["Secrets:SqlitePath"]))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
configuration["Secrets:SqlitePath"] = Path.Combine(
|
||||
Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData),
|
||||
"MxGateway",
|
||||
"mxgateway-secrets.db");
|
||||
}
|
||||
|
||||
private static void ConfigureSelfSignedTls(WebApplicationBuilder builder)
|
||||
{
|
||||
if (!Security.Tls.KestrelTlsInspector.RequiresGeneratedCertificate(builder.Configuration))
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
using System.Threading.Channels;
|
||||
using Microsoft.Extensions.Options;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
@@ -37,7 +38,7 @@ public sealed class EventStreamService(
|
||||
// non-blocking. When this subscriber's channel is full the pump applies the per-subscriber
|
||||
// backpressure policy and completes this subscriber's channel with a SessionManagerException
|
||||
// (SessionManagerErrorCode.EventQueueOverflow). That terminal fault surfaces here when the
|
||||
// reader's MoveNextAsync throws, and it propagates to the gRPC client unchanged. The overflow
|
||||
// reader's WaitToReadAsync throws, and it propagates to the gRPC client unchanged. The overflow
|
||||
// metric, and (in the legacy single-subscriber FailFast case) the session fault + fault metric,
|
||||
// are recorded by the distributor's overflow handler so the session, the pump, and other
|
||||
// subscribers are isolated from this subscriber's slowness.
|
||||
@@ -106,9 +107,14 @@ public sealed class EventStreamService(
|
||||
options.Value.Sessions.MaxEventSubscribersPerSession);
|
||||
}
|
||||
|
||||
IAsyncEnumerator<MxEvent> reader = subscriber.Reader
|
||||
.ReadAllAsync(cancellationToken)
|
||||
.GetAsyncEnumerator(cancellationToken);
|
||||
// Consume the subscriber channel directly (WaitToReadAsync + an inner TryRead drain)
|
||||
// rather than through ReadAllAsync's IAsyncEnumerable wrapper. This is the hottest
|
||||
// per-event path in the gateway and the wrapper added a second async state machine hop
|
||||
// per event for no behavioral benefit: WaitToReadAsync observes cancellation and a
|
||||
// faulted completion exactly as MoveNextAsync did, and TryRead drains what is already
|
||||
// buffered without allocating a wait. StreamEventsAsync itself stays an async iterator —
|
||||
// its `yield return` is what feeds the gRPC writer.
|
||||
ChannelReader<MxEvent> reader = subscriber.Reader;
|
||||
|
||||
// GWC-15: register this subscriber's channel as a live backlog source instead of
|
||||
// reconciling the queue-depth gauge on every event. The gauge previously read the
|
||||
@@ -151,15 +157,14 @@ public sealed class EventStreamService(
|
||||
|
||||
while (true)
|
||||
{
|
||||
MxEvent mxEvent;
|
||||
bool hasMore;
|
||||
try
|
||||
{
|
||||
if (!await reader.MoveNextAsync().ConfigureAwait(false))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
mxEvent = reader.Current;
|
||||
// A cleanly completed channel returns false here (end of stream); a channel
|
||||
// completed WITH a fault rethrows that fault from the wait once the buffer
|
||||
// is drained — the same surface MoveNextAsync presented, so the terminal
|
||||
// SessionManagerException(EventQueueOverflow) still propagates unchanged.
|
||||
hasMore = await reader.WaitToReadAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (WorkerClientException workerException)
|
||||
{
|
||||
@@ -173,24 +178,36 @@ public sealed class EventStreamService(
|
||||
throw;
|
||||
}
|
||||
|
||||
// Per-RPC filter stays at the subscriber boundary: each request may resume
|
||||
// from a different AfterWorkerSequence, so the shared pump fans raw events and
|
||||
// this loop drops the ones at or below the caller's watermark.
|
||||
if (mxEvent.WorkerSequence <= afterWorkerSequence)
|
||||
if (!hasMore)
|
||||
{
|
||||
continue;
|
||||
break;
|
||||
}
|
||||
|
||||
// The queue-depth gauge is maintained lazily via the backlog registration above
|
||||
// (GWC-15): the metric reads this subscriber's channel Count only when scraped,
|
||||
// so there is no per-event gauge bookkeeping on this hot path.
|
||||
yield return mxEvent;
|
||||
// Drain everything already buffered before waiting again. TryRead never throws;
|
||||
// a fault left on the channel is observed by the next WaitToReadAsync above.
|
||||
while (reader.TryRead(out MxEvent? mxEvent))
|
||||
{
|
||||
// Per-RPC filter stays at the subscriber boundary: each request may resume
|
||||
// from a different AfterWorkerSequence, so the shared pump fans raw events
|
||||
// and this loop drops the ones at or below the caller's watermark. It
|
||||
// applies to every live event, drained or awaited alike.
|
||||
if (mxEvent.WorkerSequence <= afterWorkerSequence)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// The queue-depth gauge is maintained lazily via the backlog registration
|
||||
// above (GWC-15): the metric reads this subscriber's channel Count only when
|
||||
// scraped, so there is no per-event gauge bookkeeping on this hot path.
|
||||
yield return mxEvent;
|
||||
}
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
await reader.DisposeAsync().ConfigureAwait(false);
|
||||
|
||||
// Nothing to dispose for the reader: consuming the ChannelReader directly means
|
||||
// there is no enumerator wrapper holding the cancellation registration.
|
||||
//
|
||||
// Remove this subscriber's live backlog contribution before disposing the lease so
|
||||
// the gauge stops counting a channel that is about to be completed; after this the
|
||||
// gauge reflects only the remaining subscribers (zero when none remain).
|
||||
|
||||
@@ -101,6 +101,12 @@ public sealed class MxAccessGatewayService(
|
||||
try
|
||||
{
|
||||
requestValidator.ValidateInvoke(request);
|
||||
|
||||
// PERF(followup): this resolve and the sessionManager.InvokeAsync below look the same
|
||||
// session up twice (a dictionary hit each, so measured cost is negligible). Collapsing
|
||||
// them needs a SessionManager overload taking an already-resolved GatewaySession, which
|
||||
// would duplicate InvokeAsync's fault mapping (SessionNotFound / state checks / metrics)
|
||||
// at a second entry point — deliberately not worth it until a profile says otherwise.
|
||||
GatewaySession session = ResolveSession(request.SessionId);
|
||||
MxCommand command = request.Command;
|
||||
BulkConstraintPlan? bulkConstraintPlan = await ApplyConstraintsAsync(
|
||||
@@ -461,6 +467,14 @@ public sealed class MxAccessGatewayService(
|
||||
string? correlationId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// An identity with no read constraints allows every tag, so the per-item enforcer call below
|
||||
// can only answer "allowed" — the whole loop (and the plan it would build) is dead work.
|
||||
// Returning null is exactly what the denied.Count == 0 exit below returns.
|
||||
if (!constraintEnforcer.HasReadConstraints(identity))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
Dictionary<int, SubscribeResult> denied = [];
|
||||
List<string> allowed = [];
|
||||
for (int index = 0; index < tagAddresses.Count; index++)
|
||||
@@ -491,16 +505,23 @@ public sealed class MxAccessGatewayService(
|
||||
return null;
|
||||
}
|
||||
|
||||
MxCommand filtered = command.Clone();
|
||||
if (filtered.Kind == MxCommandKind.AddItemBulk)
|
||||
// Build the filtered command directly instead of cloning the original and clearing it:
|
||||
// the clone deep-copied every denied address only to drop it. The payload's other fields
|
||||
// (server_handle) are copied across explicitly. Nothing aliases the request here — these
|
||||
// bulk payloads carry only strings — and the worker-bound graph is still the unaliased copy
|
||||
// MapCommand makes.
|
||||
MxCommand filtered = new() { Kind = command.Kind };
|
||||
if (command.Kind == MxCommandKind.AddItemBulk)
|
||||
{
|
||||
filtered.AddItemBulk.TagAddresses.Clear();
|
||||
filtered.AddItemBulk.TagAddresses.Add(allowed);
|
||||
AddItemBulkCommand payload = new() { ServerHandle = command.AddItemBulk.ServerHandle };
|
||||
payload.TagAddresses.Add(allowed);
|
||||
filtered.AddItemBulk = payload;
|
||||
}
|
||||
else
|
||||
{
|
||||
filtered.SubscribeBulk.TagAddresses.Clear();
|
||||
filtered.SubscribeBulk.TagAddresses.Add(allowed);
|
||||
SubscribeBulkCommand payload = new() { ServerHandle = command.SubscribeBulk.ServerHandle };
|
||||
payload.TagAddresses.Add(allowed);
|
||||
filtered.SubscribeBulk = payload;
|
||||
}
|
||||
|
||||
return new SubscribeBulkConstraintPlan(filtered, tagAddresses.Count, denied, allowed.Count > 0);
|
||||
@@ -517,6 +538,11 @@ public sealed class MxAccessGatewayService(
|
||||
// Mirrors FilterTagBulkAsync but produces BulkReadResult denial entries
|
||||
// so the reply payload merges into BulkReadReply.Results, not
|
||||
// BulkSubscribeReply.Results.
|
||||
if (!constraintEnforcer.HasReadConstraints(identity))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
Dictionary<int, BulkReadResult> denied = [];
|
||||
List<string> allowed = [];
|
||||
for (int index = 0; index < tagAddresses.Count; index++)
|
||||
@@ -548,9 +574,14 @@ public sealed class MxAccessGatewayService(
|
||||
return null;
|
||||
}
|
||||
|
||||
MxCommand filtered = command.Clone();
|
||||
filtered.ReadBulk.TagAddresses.Clear();
|
||||
filtered.ReadBulk.TagAddresses.Add(allowed);
|
||||
MxCommand filtered = new() { Kind = command.Kind };
|
||||
ReadBulkCommand payload = new()
|
||||
{
|
||||
ServerHandle = command.ReadBulk.ServerHandle,
|
||||
TimeoutMs = command.ReadBulk.TimeoutMs,
|
||||
};
|
||||
payload.TagAddresses.Add(allowed);
|
||||
filtered.ReadBulk = payload;
|
||||
|
||||
return new ReadBulkConstraintPlan(filtered, tagAddresses.Count, denied, allowed.Count > 0);
|
||||
}
|
||||
@@ -572,6 +603,11 @@ public sealed class MxAccessGatewayService(
|
||||
// Parameterising on TEntry + getItemHandle keeps a single filter
|
||||
// routine for all four and avoids duplicating CheckWriteHandleAsync
|
||||
// calls.
|
||||
if (!constraintEnforcer.HasWriteConstraints(identity))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
Dictionary<int, BulkWriteResult> denied = [];
|
||||
List<TEntry> allowed = [];
|
||||
for (int index = 0; index < entries.Count; index++)
|
||||
@@ -609,33 +645,74 @@ public sealed class MxAccessGatewayService(
|
||||
return null;
|
||||
}
|
||||
|
||||
MxCommand filtered = command.Clone();
|
||||
ReplaceWriteBulkEntries(filtered, allowed);
|
||||
return new WriteBulkConstraintPlan(filtered, entries.Count, denied, allowed.Count > 0);
|
||||
return new WriteBulkConstraintPlan(
|
||||
BuildFilteredWriteBulkCommand(command, allowed),
|
||||
entries.Count,
|
||||
denied,
|
||||
allowed.Count > 0);
|
||||
}
|
||||
|
||||
private static void ReplaceWriteBulkEntries<TEntry>(MxCommand command, IReadOnlyList<TEntry> allowed)
|
||||
/// <summary>
|
||||
/// Builds the allowed-only bulk-write command. The allowed entries are carried over by
|
||||
/// reference rather than deep-cloned: the caller only reads this command (TrackCommandReply),
|
||||
/// and the copy the worker mutates and owns is the one <c>MapCommand</c> clones — the same
|
||||
/// no-aliasing boundary as before. Cloning the whole command here and clearing it copied
|
||||
/// every denied entry's payload (including <c>WriteSecured</c> values) for nothing.
|
||||
/// </summary>
|
||||
/// <typeparam name="TEntry">The per-family bulk-write entry message type.</typeparam>
|
||||
/// <param name="command">The original command, read for its kind and payload scalars.</param>
|
||||
/// <param name="allowed">The entries that survived constraint filtering, in original order.</param>
|
||||
/// <returns>A command of the same kind carrying only the allowed entries.</returns>
|
||||
private static MxCommand BuildFilteredWriteBulkCommand<TEntry>(MxCommand command, IReadOnlyList<TEntry> allowed)
|
||||
where TEntry : class
|
||||
{
|
||||
MxCommand filtered = new() { Kind = command.Kind };
|
||||
switch (command.Kind)
|
||||
{
|
||||
case MxCommandKind.WriteBulk:
|
||||
command.WriteBulk.Entries.Clear();
|
||||
command.WriteBulk.Entries.Add((IEnumerable<WriteBulkEntry>)allowed);
|
||||
{
|
||||
WriteBulkCommand payload = new() { ServerHandle = command.WriteBulk.ServerHandle };
|
||||
payload.Entries.Add((IEnumerable<WriteBulkEntry>)allowed);
|
||||
filtered.WriteBulk = payload;
|
||||
break;
|
||||
}
|
||||
|
||||
case MxCommandKind.Write2Bulk:
|
||||
command.Write2Bulk.Entries.Clear();
|
||||
command.Write2Bulk.Entries.Add((IEnumerable<Write2BulkEntry>)allowed);
|
||||
{
|
||||
Write2BulkCommand payload = new() { ServerHandle = command.Write2Bulk.ServerHandle };
|
||||
payload.Entries.Add((IEnumerable<Write2BulkEntry>)allowed);
|
||||
filtered.Write2Bulk = payload;
|
||||
break;
|
||||
}
|
||||
|
||||
case MxCommandKind.WriteSecuredBulk:
|
||||
command.WriteSecuredBulk.Entries.Clear();
|
||||
command.WriteSecuredBulk.Entries.Add((IEnumerable<WriteSecuredBulkEntry>)allowed);
|
||||
{
|
||||
WriteSecuredBulkCommand payload = new() { ServerHandle = command.WriteSecuredBulk.ServerHandle };
|
||||
payload.Entries.Add((IEnumerable<WriteSecuredBulkEntry>)allowed);
|
||||
filtered.WriteSecuredBulk = payload;
|
||||
break;
|
||||
}
|
||||
|
||||
case MxCommandKind.WriteSecured2Bulk:
|
||||
command.WriteSecured2Bulk.Entries.Clear();
|
||||
command.WriteSecured2Bulk.Entries.Add((IEnumerable<WriteSecured2BulkEntry>)allowed);
|
||||
{
|
||||
WriteSecured2BulkCommand payload = new() { ServerHandle = command.WriteSecured2Bulk.ServerHandle };
|
||||
payload.Entries.Add((IEnumerable<WriteSecured2BulkEntry>)allowed);
|
||||
filtered.WriteSecured2Bulk = payload;
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
// Only the four bulk-write kinds above reach FilterWriteBulkAsync, so this is
|
||||
// unreachable. It throws rather than falling back to the unmodified command,
|
||||
// because that fallback failed OPEN: a fifth bulk-write kind added upstream
|
||||
// without a case here would silently ship the DENIED entries to the worker while
|
||||
// still reporting them denied to the caller. Failing loud on a kind nobody can
|
||||
// reach today is strictly safer than a constraint bypass nobody would notice.
|
||||
throw new UnreachableException(
|
||||
$"Command kind {command.Kind} reached bulk-write constraint filtering without a filter case.");
|
||||
}
|
||||
|
||||
return filtered;
|
||||
}
|
||||
|
||||
private async Task<BulkConstraintPlan?> FilterHandleBulkAsync(
|
||||
@@ -647,6 +724,11 @@ public sealed class MxAccessGatewayService(
|
||||
string? correlationId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!constraintEnforcer.HasReadConstraints(identity))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
Dictionary<int, SubscribeResult> denied = [];
|
||||
List<int> allowed = [];
|
||||
for (int index = 0; index < itemHandles.Count; index++)
|
||||
@@ -677,9 +759,10 @@ public sealed class MxAccessGatewayService(
|
||||
return null;
|
||||
}
|
||||
|
||||
MxCommand filtered = command.Clone();
|
||||
filtered.AdviseItemBulk.ItemHandles.Clear();
|
||||
filtered.AdviseItemBulk.ItemHandles.Add(allowed);
|
||||
MxCommand filtered = new() { Kind = command.Kind };
|
||||
AdviseItemBulkCommand payload = new() { ServerHandle = command.AdviseItemBulk.ServerHandle };
|
||||
payload.ItemHandles.Add(allowed);
|
||||
filtered.AdviseItemBulk = payload;
|
||||
|
||||
return new SubscribeBulkConstraintPlan(filtered, itemHandles.Count, denied, allowed.Count > 0);
|
||||
}
|
||||
|
||||
@@ -71,7 +71,17 @@ public sealed class MxAccessGrpcMapper
|
||||
};
|
||||
}
|
||||
|
||||
return reply.Reply.Clone();
|
||||
// GWC-07 / IPC-05: ownership transfer, not a deep clone — the same rule MapEvent follows,
|
||||
// applied to the other (and larger, on bulk reads) hot-path message. The enclosing
|
||||
// WorkerCommandReply is parsed fresh from a single pipe frame in WorkerClient's read loop
|
||||
// and is single-consumer by construction: CompleteCommand's TryRemove hands it to exactly
|
||||
// one PendingCommand awaiter, that awaiter is the gRPC Invoke handler, and the handler's
|
||||
// one call is this mapping. Nothing else aliases or reads reply.Reply afterwards — the
|
||||
// enclosing WorkerCommandReply is discarded here. We therefore move the inner
|
||||
// MxCommandReply into the gRPC response instead of copying it; the handler owning it
|
||||
// outright is also what makes BulkConstraintPlan.MergeDeniedInto's in-place splice safe.
|
||||
// If a second consumer of the same WorkerCommandReply is ever added, restore a .Clone().
|
||||
return reply.Reply;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -28,7 +28,9 @@ public sealed class GatewayMetrics : IDisposable
|
||||
private readonly Histogram<double> _workerStartupLatencyHistogram;
|
||||
private readonly Histogram<double> _commandLatencyHistogram;
|
||||
private readonly Histogram<double> _eventStreamSendLatencyHistogram;
|
||||
private readonly Dictionary<string, long> _commandFailuresByMethod = new(StringComparer.OrdinalIgnoreCase);
|
||||
// Concurrent (not Dictionary + _syncRoot) because CommandFailed runs on every failing gRPC call:
|
||||
// the command counters are recorded outside the lock, so their breakdown map must be too.
|
||||
private readonly ConcurrentDictionary<string, long> _commandFailuresByMethod = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly ConcurrentDictionary<string, long> _eventsByFamily = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly ConcurrentDictionary<string, long> _eventsBySession = new(StringComparer.Ordinal);
|
||||
private readonly Dictionary<string, long> _retryAttemptsByArea = new(StringComparer.OrdinalIgnoreCase);
|
||||
@@ -41,9 +43,16 @@ public sealed class GatewayMetrics : IDisposable
|
||||
private readonly ConcurrentDictionary<long, Func<int>> _eventStreamBacklogSources = new();
|
||||
private long _nextEventStreamBacklogSourceId;
|
||||
|
||||
// GWC-30: the same pull model for the worker event queue depth. It replaces a pushed scalar that
|
||||
// every WorkerClient wrote twice per event (staged, consumed) under _syncRoot — a process-wide
|
||||
// lock on the hottest path, and last-writer-wins across sessions, so the gauge reported one
|
||||
// arbitrary session's backlog instead of the gateway's. Each client registers a source returning
|
||||
// its own undelivered depth; the gauge sums them at collection time only.
|
||||
private readonly ConcurrentDictionary<long, Func<int>> _workerEventQueueDepthSources = new();
|
||||
private long _nextWorkerEventQueueDepthSourceId;
|
||||
|
||||
private int _openSessions;
|
||||
private int _workersRunning;
|
||||
private int _workerEventQueueDepth;
|
||||
private int _alarmProviderMode;
|
||||
private long _sessionsOpened;
|
||||
private long _sessionsClosed;
|
||||
@@ -201,10 +210,10 @@ public sealed class GatewayMetrics : IDisposable
|
||||
/// <param name="method">Name of the command method.</param>
|
||||
public void CommandStarted(string method)
|
||||
{
|
||||
lock (_syncRoot)
|
||||
{
|
||||
_commandsStarted++;
|
||||
}
|
||||
// GWC-30: the three command counters run two-to-three times per gRPC call, so they use
|
||||
// Interlocked rather than _syncRoot — the same idiom as EventReceived. Nothing here needs a
|
||||
// consistent multi-field view; GetSnapshot reads each with Interlocked.Read.
|
||||
Interlocked.Increment(ref _commandsStarted);
|
||||
|
||||
_commandsStartedCounter.Add(1, new KeyValuePair<string, object?>("method", method));
|
||||
}
|
||||
@@ -216,10 +225,7 @@ public sealed class GatewayMetrics : IDisposable
|
||||
/// <param name="duration">Elapsed time to complete the command.</param>
|
||||
public void CommandSucceeded(string method, TimeSpan duration)
|
||||
{
|
||||
lock (_syncRoot)
|
||||
{
|
||||
_commandsSucceeded++;
|
||||
}
|
||||
Interlocked.Increment(ref _commandsSucceeded);
|
||||
|
||||
KeyValuePair<string, object?> methodTag = new("method", method);
|
||||
_commandsSucceededCounter.Add(1, methodTag);
|
||||
@@ -234,11 +240,8 @@ public sealed class GatewayMetrics : IDisposable
|
||||
/// <param name="duration">Elapsed time before command failed.</param>
|
||||
public void CommandFailed(string method, string category, TimeSpan duration)
|
||||
{
|
||||
lock (_syncRoot)
|
||||
{
|
||||
_commandsFailed++;
|
||||
Increment(_commandFailuresByMethod, method);
|
||||
}
|
||||
Interlocked.Increment(ref _commandsFailed);
|
||||
Increment(_commandFailuresByMethod, method);
|
||||
|
||||
KeyValuePair<string, object?> methodTag = new("method", method);
|
||||
KeyValuePair<string, object?> categoryTag = new("category", category);
|
||||
@@ -275,29 +278,24 @@ public sealed class GatewayMetrics : IDisposable
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the worker event queue depth; delegates to SetWorkerEventQueueDepth.
|
||||
/// Registers a live depth source for the worker event queue-depth gauge and returns a handle
|
||||
/// that removes it when disposed. Each <c>WorkerClient</c> registers once and reports its own
|
||||
/// undelivered (staged + queued) event count, so the gauge is the gateway-wide sum instead of
|
||||
/// the last value any one session happened to push (GWC-30).
|
||||
/// </summary>
|
||||
/// <param name="depth">Queue depth value.</param>
|
||||
public void SetEventQueueDepth(int depth)
|
||||
/// <param name="depth">
|
||||
/// Returns this worker client's current undelivered event count. Invoked only at collection
|
||||
/// time; must be cheap and non-blocking (a <see cref="Volatile.Read(ref int)"/> of an
|
||||
/// interlocked counter). Negative readings — which a racing decrement can produce — are
|
||||
/// clamped to zero when summed.
|
||||
/// </param>
|
||||
/// <returns>A handle whose disposal unregisters the source. Safe to dispose more than once.</returns>
|
||||
public IDisposable RegisterWorkerEventQueueDepthSource(Func<int> depth)
|
||||
{
|
||||
SetWorkerEventQueueDepth(depth);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sets the worker event queue depth to the given value.
|
||||
/// </summary>
|
||||
/// <param name="depth">Queue depth value.</param>
|
||||
public void SetWorkerEventQueueDepth(int depth)
|
||||
{
|
||||
if (depth < 0)
|
||||
{
|
||||
throw new ArgumentOutOfRangeException(nameof(depth), depth, "Queue depth cannot be negative.");
|
||||
}
|
||||
|
||||
lock (_syncRoot)
|
||||
{
|
||||
_workerEventQueueDepth = depth;
|
||||
}
|
||||
ArgumentNullException.ThrowIfNull(depth);
|
||||
long id = Interlocked.Increment(ref _nextWorkerEventQueueDepthSourceId);
|
||||
_workerEventQueueDepthSources[id] = depth;
|
||||
return new GaugeSourceRegistration(_workerEventQueueDepthSources, id);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -318,7 +316,7 @@ public sealed class GatewayMetrics : IDisposable
|
||||
ArgumentNullException.ThrowIfNull(backlog);
|
||||
long id = Interlocked.Increment(ref _nextEventStreamBacklogSourceId);
|
||||
_eventStreamBacklogSources[id] = backlog;
|
||||
return new EventStreamBacklogRegistration(this, id);
|
||||
return new GaugeSourceRegistration(_eventStreamBacklogSources, id);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -460,21 +458,23 @@ public sealed class GatewayMetrics : IDisposable
|
||||
/// <returns>The current metrics snapshot.</returns>
|
||||
public GatewayMetricsSnapshot GetSnapshot()
|
||||
{
|
||||
// Compute the live gRPC stream backlog outside _syncRoot: the sources are the subscriber
|
||||
// channels' Count (their own locks) and must not run under this lock. GWC-15.
|
||||
// Compute the live queue depths outside _syncRoot: the sources are the subscriber channels'
|
||||
// Count (their own locks) and the worker clients' interlocked counters, neither of which may
|
||||
// run under this lock. GWC-15, GWC-30.
|
||||
int workerEventQueueDepth = GetWorkerEventQueueDepth();
|
||||
int grpcEventStreamQueueDepth = GetGrpcEventStreamQueueDepth();
|
||||
lock (_syncRoot)
|
||||
{
|
||||
return new GatewayMetricsSnapshot(
|
||||
OpenSessions: _openSessions,
|
||||
WorkersRunning: _workersRunning,
|
||||
WorkerEventQueueDepth: _workerEventQueueDepth,
|
||||
WorkerEventQueueDepth: workerEventQueueDepth,
|
||||
GrpcEventStreamQueueDepth: grpcEventStreamQueueDepth,
|
||||
SessionsOpened: _sessionsOpened,
|
||||
SessionsClosed: _sessionsClosed,
|
||||
CommandsStarted: _commandsStarted,
|
||||
CommandsSucceeded: _commandsSucceeded,
|
||||
CommandsFailed: _commandsFailed,
|
||||
CommandsStarted: Interlocked.Read(ref _commandsStarted),
|
||||
CommandsSucceeded: Interlocked.Read(ref _commandsSucceeded),
|
||||
CommandsFailed: Interlocked.Read(ref _commandsFailed),
|
||||
EventsReceived: Interlocked.Read(ref _eventsReceived),
|
||||
QueueOverflows: _queueOverflows,
|
||||
Faults: _faults,
|
||||
@@ -521,22 +521,19 @@ public sealed class GatewayMetrics : IDisposable
|
||||
}
|
||||
}
|
||||
|
||||
private int GetWorkerEventQueueDepth()
|
||||
{
|
||||
lock (_syncRoot)
|
||||
{
|
||||
return _workerEventQueueDepth;
|
||||
}
|
||||
}
|
||||
// Sums the undelivered event backlog across every live worker client (GWC-30).
|
||||
private int GetWorkerEventQueueDepth() => SumSources(_workerEventQueueDepthSources);
|
||||
|
||||
// Sums the live backlog across every registered event-stream subscriber. Runs at collection
|
||||
// time (ObservableGauge scrape) or when GetSnapshot projects the value — never on the
|
||||
// per-event path. Enumerating ConcurrentDictionary.Values never throws on concurrent
|
||||
// Sums the live backlog across every registered event-stream subscriber.
|
||||
private int GetGrpcEventStreamQueueDepth() => SumSources(_eventStreamBacklogSources);
|
||||
|
||||
// Runs at collection time (ObservableGauge scrape) or when GetSnapshot projects the value —
|
||||
// never on a per-event path. Enumerating ConcurrentDictionary.Values never throws on concurrent
|
||||
// register/unregister; a source removed mid-enumeration simply drops from this sample.
|
||||
private int GetGrpcEventStreamQueueDepth()
|
||||
private static int SumSources(ConcurrentDictionary<long, Func<int>> sources)
|
||||
{
|
||||
int total = 0;
|
||||
foreach (Func<int> source in _eventStreamBacklogSources.Values)
|
||||
foreach (Func<int> source in sources.Values)
|
||||
{
|
||||
int value = source();
|
||||
if (value > 0)
|
||||
@@ -548,11 +545,6 @@ public sealed class GatewayMetrics : IDisposable
|
||||
return total;
|
||||
}
|
||||
|
||||
private void UnregisterEventStreamBacklogSource(long id)
|
||||
{
|
||||
_eventStreamBacklogSources.TryRemove(id, out _);
|
||||
}
|
||||
|
||||
private int GetAlarmProviderMode()
|
||||
{
|
||||
lock (_syncRoot)
|
||||
@@ -572,9 +564,10 @@ public sealed class GatewayMetrics : IDisposable
|
||||
values.AddOrUpdate(key, 1, static (_, currentValue) => currentValue + 1);
|
||||
}
|
||||
|
||||
// Handle returned by RegisterEventStreamBacklogSource. Disposal (once) removes the source
|
||||
// from the gauge's live sum. Idempotent so a double dispose from a stream teardown is safe.
|
||||
private sealed class EventStreamBacklogRegistration(GatewayMetrics metrics, long id) : IDisposable
|
||||
// Handle returned by the pull-model gauge registrations. Disposal (once) removes the source from
|
||||
// that gauge's live sum. Idempotent so a double dispose from a stream or worker-client teardown
|
||||
// is safe, and shared by both gauges so the two registrations cannot drift apart.
|
||||
private sealed class GaugeSourceRegistration(ConcurrentDictionary<long, Func<int>> sources, long id) : IDisposable
|
||||
{
|
||||
private int _disposed;
|
||||
|
||||
@@ -582,7 +575,7 @@ public sealed class GatewayMetrics : IDisposable
|
||||
{
|
||||
if (Interlocked.Exchange(ref _disposed, 1) == 0)
|
||||
{
|
||||
metrics.UnregisterEventStreamBacklogSource(id);
|
||||
sources.TryRemove(id, out _);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,281 @@
|
||||
using ZB.MOM.WW.Audit;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Security.Audit;
|
||||
|
||||
/// <summary>
|
||||
/// Drains <see cref="ChannelAuditWriter"/> onto the durable <see cref="IAuditEventSink"/>,
|
||||
/// owns the one-time schema bootstrap, and sweeps audit rows past their retention window.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Batching is the point: up to <see cref="MaxBatchSize"/> buffered events are committed in a
|
||||
/// single transaction, so a burst of constraint denials costs a handful of commits instead of
|
||||
/// one per denied tag. The bootstrap runs here — before the writer starts enqueueing — so no
|
||||
/// audit write ever pays a <c>CREATE TABLE IF NOT EXISTS</c> round-trip.
|
||||
/// <para>
|
||||
/// Every failure mode ends in synchronous audit rather than silent loss: a batch that will not
|
||||
/// commit is retried one event at a time so only the offending row is dropped, and a drain loop
|
||||
/// that dies detaches the writer, which reverts every producer to the direct write path.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="writer">The channel writer whose buffered events are drained.</param>
|
||||
/// <param name="sink">The durable sink events are committed to.</param>
|
||||
/// <param name="security">Security options carrying the audit retention window.</param>
|
||||
/// <param name="timeProvider">Clock used for the retention cutoff and sweep interval.</param>
|
||||
/// <param name="logger">Logger for bootstrap, drain and sweep diagnostics.</param>
|
||||
public sealed class AuditDrainService(
|
||||
ChannelAuditWriter writer,
|
||||
IAuditEventSink sink,
|
||||
SecurityOptions security,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<AuditDrainService> logger) : BackgroundService
|
||||
{
|
||||
/// <summary>Maximum number of audit events committed in one transaction per drain pass.</summary>
|
||||
public const int MaxBatchSize = 64;
|
||||
|
||||
/// <summary>How often the retention sweep runs while the gateway is up.</summary>
|
||||
public static readonly TimeSpan RetentionSweepInterval = TimeSpan.FromHours(1);
|
||||
|
||||
/// <summary>Upper bound on how long shutdown waits for the remaining buffered events.</summary>
|
||||
private static readonly TimeSpan ShutdownDrainCap = TimeSpan.FromSeconds(2);
|
||||
|
||||
/// <summary>
|
||||
/// Bootstraps the audit table, runs one retention sweep, then attaches the drain so the
|
||||
/// writer switches from synchronous write-through to enqueueing.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Token to observe for cancellation.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public override async Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
await sink.EnsureInitializedAsync(cancellationToken).ConfigureAwait(false);
|
||||
await SweepRetentionAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
// Audit is best-effort: a bootstrap failure must not take the gateway down. The
|
||||
// sink's own latch will retry the schema check on the first write.
|
||||
logger.LogWarning(exception, "Audit store bootstrap failed; audit writes will retry the schema check.");
|
||||
}
|
||||
|
||||
writer.AttachDrain();
|
||||
|
||||
await base.StartAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Detaches the drain (so late writes go straight to the sink) and gives the buffered
|
||||
/// events a bounded window to reach the store.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Token to observe for cancellation.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public override async Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
writer.DetachDrain();
|
||||
writer.CompleteWriting();
|
||||
|
||||
await base.StopAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
using CancellationTokenSource drainCap = new(ShutdownDrainCap);
|
||||
try
|
||||
{
|
||||
await DrainPendingAsync(drainCap.Token).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Shutdown drain exceeded {CapSeconds}s; remaining buffered audit events were not persisted.",
|
||||
ShutdownDrainCap.TotalSeconds);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Commits every event currently buffered, in batches of at most <see cref="MaxBatchSize"/>.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Token to observe for cancellation.</param>
|
||||
/// <returns>The number of events persisted.</returns>
|
||||
/// <exception cref="OperationCanceledException">
|
||||
/// The drain was cancelled — at shutdown this is the 2-second cap expiring, which the caller
|
||||
/// reports as unpersisted audit rather than as a store fault.
|
||||
/// </exception>
|
||||
public async Task<int> DrainPendingAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
int persisted = 0;
|
||||
List<AuditEvent> batch = new(MaxBatchSize);
|
||||
|
||||
while (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
batch.Clear();
|
||||
while (batch.Count < MaxBatchSize && writer.Reader.TryRead(out AuditEvent? auditEvent))
|
||||
{
|
||||
batch.Add(auditEvent);
|
||||
}
|
||||
|
||||
if (batch.Count == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await sink.InsertBatchAsync(batch, cancellationToken).ConfigureAwait(false);
|
||||
persisted += batch.Count;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Cancellation is the shutdown cap, not a store fault: surface it so StopAsync
|
||||
// reports unpersisted audit instead of misreporting it as a failed write.
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Failed to commit a batch of {Count} audit events; retrying them individually.",
|
||||
batch.Count);
|
||||
|
||||
persisted += await InsertIndividuallyAsync(batch, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
return persisted;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Deletes audit rows older than <c>MxGateway:Security:AuditRetentionDays</c>, and reports the
|
||||
/// running total of audit events dropped by channel pressure since startup.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Token to observe for cancellation.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async Task SweepRetentionAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
DateTimeOffset cutoff = timeProvider.GetUtcNow() - TimeSpan.FromDays(security.AuditRetentionDays);
|
||||
|
||||
try
|
||||
{
|
||||
int deleted = await sink.DeleteOlderThanAsync(cutoff, cancellationToken).ConfigureAwait(false);
|
||||
if (deleted > 0)
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Audit retention sweep removed {Deleted} events older than {Cutoff:o} ({RetentionDays} days).",
|
||||
deleted,
|
||||
cutoff,
|
||||
security.AuditRetentionDays);
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Audit retention sweep failed; it will be retried on the next interval.");
|
||||
}
|
||||
|
||||
long dropped = writer.DroppedCount;
|
||||
if (dropped > 0)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"{Dropped} audit events have been dropped since startup because the audit channel was full.",
|
||||
dropped);
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
await Task.WhenAll(
|
||||
DrainLoopAsync(stoppingToken),
|
||||
RetentionLoopAsync(stoppingToken)).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
// Re-inserts a failed batch one event at a time so a single unwritable row costs only itself
|
||||
// rather than the up-to-MaxBatchSize good events that happened to share its transaction.
|
||||
private async Task<int> InsertIndividuallyAsync(
|
||||
IReadOnlyList<AuditEvent> batch,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
int persisted = 0;
|
||||
|
||||
foreach (AuditEvent auditEvent in batch)
|
||||
{
|
||||
try
|
||||
{
|
||||
await sink.InsertAsync(auditEvent, cancellationToken).ConfigureAwait(false);
|
||||
persisted++;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(
|
||||
exception,
|
||||
"Dropped audit event {EventId} (action {Action}); it could not be persisted individually.",
|
||||
auditEvent.EventId,
|
||||
auditEvent.Action);
|
||||
}
|
||||
}
|
||||
|
||||
logger.LogWarning(
|
||||
"Recovered {Persisted} of {Count} audit events from a failed batch.",
|
||||
persisted,
|
||||
batch.Count);
|
||||
|
||||
return persisted;
|
||||
}
|
||||
|
||||
private async Task DrainLoopAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
while (await writer.Reader.WaitToReadAsync(stoppingToken).ConfigureAwait(false))
|
||||
{
|
||||
await DrainPendingAsync(stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Shutdown; StopAsync performs the final bounded drain.
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
// A dead drain loop would silently discard every later audit write, because producers
|
||||
// keep enqueueing into a channel nobody reads. Detaching (below) reverts them to the
|
||||
// synchronous path, so audit degrades in latency rather than disappearing.
|
||||
logger.LogError(exception, "Audit drain loop failed; reverting to synchronous audit writes.");
|
||||
}
|
||||
finally
|
||||
{
|
||||
writer.DetachDrain();
|
||||
|
||||
// Detaching alone leaves a racer that already passed the attached check enqueueing into
|
||||
// a channel this loop will never read again — those events would sit in the buffer until
|
||||
// StopAsync's final drain. Completing the writer as well makes that racer's TryWrite
|
||||
// return false, which is the write-through branch, so the event reaches the store now.
|
||||
// TryComplete is idempotent, so StopAsync's own CompleteWriting stays safe either way.
|
||||
writer.CompleteWriting();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RetentionLoopAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
using PeriodicTimer timer = new(RetentionSweepInterval, timeProvider);
|
||||
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
|
||||
{
|
||||
await SweepRetentionAsync(stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Shutdown.
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
// Retention is unbounded growth if it stops: say so loudly rather than letting the
|
||||
// audit table grow forever behind a silently dead timer loop.
|
||||
logger.LogError(exception, "Audit retention loop failed; expired audit rows will no longer be swept.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,24 +3,26 @@ using ZB.MOM.WW.Audit;
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Security.Audit;
|
||||
|
||||
/// <summary>
|
||||
/// Best-effort <see cref="IAuditWriter"/> over the MxGateway-owned
|
||||
/// <see cref="SqliteCanonicalAuditStore"/>. It honours the canonical
|
||||
/// Best-effort, <em>synchronous</em> <see cref="IAuditWriter"/> over the MxGateway-owned
|
||||
/// <see cref="IAuditEventSink"/>. It honours the canonical
|
||||
/// <see cref="IAuditWriter"/> contract: a failed audit write is swallowed and logged
|
||||
/// rather than propagated, so it can never abort the user-facing action that produced it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is the single sink through which ALL MxGateway audit flows — the library admin
|
||||
/// verbs (via <see cref="CanonicalForwardingApiKeyAuditStore"/>) and the gateway's own
|
||||
/// dashboard / constraint-denial producers, which write canonical events directly. The
|
||||
/// best-effort wrapping here also closes the gap that the library's
|
||||
/// This is the durable bottom of the audit pipeline. Callers reach it two ways: through
|
||||
/// <see cref="ChannelAuditWriter"/> — the registered <see cref="IAuditWriter"/>, which
|
||||
/// enqueues and lets <see cref="AuditDrainService"/> batch events onto the sink — and
|
||||
/// directly, when there is no drain to batch behind (the <c>apikey</c> CLI, and any host
|
||||
/// shutdown window), where writing through immediately is the only way the event survives.
|
||||
/// The best-effort wrapping here also closes the gap that the library's
|
||||
/// <c>SqliteApiKeyAuditStore.AppendAsync</c> propagated exceptions.
|
||||
/// </remarks>
|
||||
public sealed class CanonicalAuditWriter(
|
||||
SqliteCanonicalAuditStore store,
|
||||
IAuditEventSink sink,
|
||||
ILogger<CanonicalAuditWriter> logger) : IAuditWriter
|
||||
{
|
||||
/// <summary>
|
||||
/// Persists a canonical audit event to the underlying <see cref="SqliteCanonicalAuditStore"/>.
|
||||
/// Persists a canonical audit event to the underlying <see cref="IAuditEventSink"/>.
|
||||
/// Any failure is caught, logged, and swallowed rather than propagated to the caller.
|
||||
/// </summary>
|
||||
/// <param name="auditEvent">The canonical audit event to persist.</param>
|
||||
@@ -32,7 +34,7 @@ public sealed class CanonicalAuditWriter(
|
||||
|
||||
try
|
||||
{
|
||||
await store.InsertAsync(auditEvent, cancellationToken).ConfigureAwait(false);
|
||||
await sink.InsertAsync(auditEvent, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
using System.Threading.Channels;
|
||||
using ZB.MOM.WW.Audit;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Security.Audit;
|
||||
|
||||
/// <summary>
|
||||
/// Bounded, non-blocking <see cref="IAuditWriter"/>: <see cref="WriteAsync"/> enqueues onto a
|
||||
/// fixed-capacity channel and returns, leaving <see cref="AuditDrainService"/> to batch the
|
||||
/// events onto the durable <see cref="IAuditEventSink"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The canonical <see cref="IAuditWriter"/> contract is already best-effort — a failed audit
|
||||
/// write is swallowed rather than propagated. The channel makes the <em>bound</em> on that
|
||||
/// promise explicit: audit can cost the calling RPC at most one enqueue, never a SQLite
|
||||
/// round-trip, and at most <see cref="ChannelCapacity"/> events of memory. This matters on the
|
||||
/// constraint-denial path, where a partially denied bulk RPC previously awaited one insert per
|
||||
/// denied tag, serially, against the same database file the authentication hot path reads.
|
||||
/// <para>
|
||||
/// When the channel is full the newest write is dropped (<see cref="BoundedChannelFullMode.DropWrite"/>)
|
||||
/// and counted in <see cref="DroppedCount"/>. Dropping is the deliberate choice over blocking:
|
||||
/// a stalled audit database must degrade audit completeness, not stall the gateway. Drops are
|
||||
/// logged, and <see cref="AuditDrainService"/> reports the running total on its sweep.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Until a drain attaches (<see cref="AttachDrain"/>), and again after it detaches, writes go
|
||||
/// straight through to <see cref="CanonicalAuditWriter"/>. Enqueueing into a channel nobody will
|
||||
/// ever read would silently discard audit in the processes that have no hosted services — the
|
||||
/// <c>apikey</c> admin CLI and the DI-only tests — so those keep the original synchronous path.
|
||||
/// The same fallback covers a completed channel, so no combination of attach/detach can leave
|
||||
/// producers writing into a buffer that will never be read.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class ChannelAuditWriter : IAuditWriter
|
||||
{
|
||||
/// <summary>
|
||||
/// Maximum number of audit events buffered before writes start being dropped. Sized to
|
||||
/// absorb a fully denied bulk RPC (the gateway's bulk request cap) plus headroom, so a
|
||||
/// realistic burst is buffered rather than lost.
|
||||
/// </summary>
|
||||
public const int ChannelCapacity = 4096;
|
||||
|
||||
private readonly CanonicalAuditWriter _directWriter;
|
||||
private readonly ILogger<ChannelAuditWriter> _logger;
|
||||
private readonly Channel<AuditEvent> _channel;
|
||||
|
||||
private long _droppedCount;
|
||||
private int _drainAttached;
|
||||
private int _dropLogged;
|
||||
|
||||
/// <summary>Creates the writer and its bounded buffer.</summary>
|
||||
/// <param name="directWriter">The synchronous writer used when no drain is attached.</param>
|
||||
/// <param name="logger">Logger for drop diagnostics.</param>
|
||||
public ChannelAuditWriter(CanonicalAuditWriter directWriter, ILogger<ChannelAuditWriter> logger)
|
||||
{
|
||||
_directWriter = directWriter;
|
||||
_logger = logger;
|
||||
|
||||
// DropWrite discards the incoming item and still reports success to the producer, so the
|
||||
// itemDropped callback is the only place a drop can be observed and counted.
|
||||
_channel = Channel.CreateBounded<AuditEvent>(
|
||||
new BoundedChannelOptions(ChannelCapacity)
|
||||
{
|
||||
FullMode = BoundedChannelFullMode.DropWrite,
|
||||
SingleReader = true,
|
||||
SingleWriter = false,
|
||||
},
|
||||
itemDropped: RecordDrop);
|
||||
}
|
||||
|
||||
/// <summary>Gets the number of audit events dropped because the channel was full.</summary>
|
||||
public long DroppedCount => Interlocked.Read(ref _droppedCount);
|
||||
|
||||
/// <summary>Gets the reader the drain service consumes buffered events from.</summary>
|
||||
public ChannelReader<AuditEvent> Reader => _channel.Reader;
|
||||
|
||||
/// <summary>
|
||||
/// Marks a drain as running, so subsequent writes enqueue instead of writing through.
|
||||
/// Called by <see cref="AuditDrainService"/> once its one-time bootstrap has completed.
|
||||
/// </summary>
|
||||
public void AttachDrain() => Volatile.Write(ref _drainAttached, 1);
|
||||
|
||||
/// <summary>
|
||||
/// Marks the drain as no longer running, so writes revert to the synchronous path. Called at
|
||||
/// shutdown, and whenever the drain loop dies, so late audit is still persisted rather than
|
||||
/// buffered into a channel with no reader.
|
||||
/// </summary>
|
||||
public void DetachDrain() => Volatile.Write(ref _drainAttached, 0);
|
||||
|
||||
/// <summary>
|
||||
/// Enqueues a canonical audit event for the drain to persist. Never blocks and never throws.
|
||||
/// It also never touches the store on the caller's thread while a drain is attached — with one
|
||||
/// exception: once the channel has been completed (shutdown, or a drain loop that died), the
|
||||
/// enqueue fails and this falls through to the synchronous write, which is what keeps the event
|
||||
/// rather than stranding it in a buffer nobody reads.
|
||||
/// </summary>
|
||||
/// <param name="auditEvent">The canonical audit event to persist.</param>
|
||||
/// <param name="cancellationToken">Token honoured only by the direct write-through path.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public Task WriteAsync(AuditEvent auditEvent, CancellationToken cancellationToken = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(auditEvent);
|
||||
|
||||
if (Volatile.Read(ref _drainAttached) == 0)
|
||||
{
|
||||
return _directWriter.WriteAsync(auditEvent, cancellationToken);
|
||||
}
|
||||
|
||||
// Under DropWrite a full channel still reports SUCCESS — the discard surfaces through the
|
||||
// itemDropped callback. So a false here does not mean "full", it means the channel has
|
||||
// been completed and no drain will ever read it again (shutdown, or a re-attach onto a
|
||||
// dead channel). Writing through is the only outcome that keeps the event.
|
||||
if (!_channel.Writer.TryWrite(auditEvent))
|
||||
{
|
||||
return _directWriter.WriteAsync(auditEvent, cancellationToken);
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// <summary>Signals that no further events will be enqueued, so the drain loop can finish.</summary>
|
||||
public void CompleteWriting() => _channel.Writer.TryComplete();
|
||||
|
||||
private void RecordDrop(AuditEvent auditEvent)
|
||||
{
|
||||
Interlocked.Increment(ref _droppedCount);
|
||||
|
||||
// Log the first drop only; the running total is reported on the drain's periodic sweep,
|
||||
// so a sustained overload cannot turn audit pressure into a log flood.
|
||||
if (Interlocked.Exchange(ref _dropLogged, 1) == 0)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Audit channel is full ({Capacity} events); dropping audit event {EventId} (action {Action}). "
|
||||
+ "Audit is best-effort and bounded; further drops are reported in aggregate.",
|
||||
ChannelCapacity,
|
||||
auditEvent.EventId,
|
||||
auditEvent.Action);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
using ZB.MOM.WW.Audit;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Security.Audit;
|
||||
|
||||
/// <summary>
|
||||
/// Durable sink the audit pipeline persists canonical <see cref="AuditEvent"/>s through.
|
||||
/// It exists so the write path (<see cref="CanonicalAuditWriter"/>) and the batching drain
|
||||
/// (<see cref="AuditDrainService"/>) depend on the storage contract rather than on the
|
||||
/// concrete <see cref="SqliteCanonicalAuditStore"/>.
|
||||
/// </summary>
|
||||
public interface IAuditEventSink
|
||||
{
|
||||
/// <summary>
|
||||
/// Bootstraps the backing storage. Called once at startup so no write path pays a schema
|
||||
/// round-trip; implementations must be idempotent and safe to call concurrently.
|
||||
/// </summary>
|
||||
/// <param name="cancellationToken">Token to observe for cancellation.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
Task EnsureInitializedAsync(CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>Persists a single canonical audit event.</summary>
|
||||
/// <param name="auditEvent">The canonical event to persist.</param>
|
||||
/// <param name="cancellationToken">Token to observe for cancellation.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
Task InsertAsync(AuditEvent auditEvent, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>Persists a batch of canonical audit events as one unit of work.</summary>
|
||||
/// <param name="auditEvents">The canonical events to persist.</param>
|
||||
/// <param name="cancellationToken">Token to observe for cancellation.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
Task InsertBatchAsync(IReadOnlyList<AuditEvent> auditEvents, CancellationToken cancellationToken);
|
||||
|
||||
/// <summary>Deletes every audit row that occurred strictly before <paramref name="cutoffUtc"/>.</summary>
|
||||
/// <param name="cutoffUtc">The retention cutoff; rows older than this are removed.</param>
|
||||
/// <param name="cancellationToken">Token to observe for cancellation.</param>
|
||||
/// <returns>The number of rows deleted.</returns>
|
||||
Task<int> DeleteOlderThanAsync(DateTimeOffset cutoffUtc, CancellationToken cancellationToken);
|
||||
}
|
||||
@@ -18,11 +18,27 @@ namespace ZB.MOM.WW.MxGateway.Server.Security.Audit;
|
||||
/// <c>IApiKeyAuditStore</c> registration is overridden by
|
||||
/// <see cref="CanonicalForwardingApiKeyAuditStore"/>, which forwards onto this store via
|
||||
/// <see cref="CanonicalAuditWriter"/>. The library's <c>schema_version</c> /
|
||||
/// <c>api_key_audit</c> tables are not touched here; the <c>audit_event</c> table is
|
||||
/// created idempotently (<c>CREATE TABLE IF NOT EXISTS</c>) on each write so it
|
||||
/// self-bootstraps regardless of migration ordering.
|
||||
/// <c>api_key_audit</c> tables are not touched here.
|
||||
/// <para>
|
||||
/// The <c>audit_event</c> table is created idempotently, but the <c>CREATE TABLE IF NOT
|
||||
/// EXISTS</c> is <em>latched</em>: <see cref="AuditDrainService"/> runs
|
||||
/// <see cref="EnsureInitializedAsync"/> once at startup, and every later insert/list/delete
|
||||
/// then skips the DDL round-trip. Keeping the (now free) check on each path rather than
|
||||
/// dropping it means the store still self-bootstraps for callers that use it without the
|
||||
/// hosted drain — the <c>apikey</c> CLI and the DI-only tests — regardless of migration
|
||||
/// ordering. The latch is deliberately racy: a lost race merely re-runs an idempotent
|
||||
/// <c>CREATE TABLE IF NOT EXISTS</c>, and a failure leaves the latch open so the next call
|
||||
/// retries.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class SqliteCanonicalAuditStore(AuthSqliteConnectionFactory connectionFactory)
|
||||
/// <param name="connectionFactory">Factory for connections to the shared auth database file.</param>
|
||||
/// <param name="logger">
|
||||
/// Optional logger for row-level read diagnostics. Optional because the store is also constructed
|
||||
/// directly by the <c>apikey</c> CLI path and by DI-free tests, which have no logger to hand.
|
||||
/// </param>
|
||||
public sealed class SqliteCanonicalAuditStore(
|
||||
AuthSqliteConnectionFactory connectionFactory,
|
||||
ILogger<SqliteCanonicalAuditStore>? logger = null) : IAuditEventSink
|
||||
{
|
||||
private const string CreateTableSql =
|
||||
"""
|
||||
@@ -40,14 +56,111 @@ public sealed class SqliteCanonicalAuditStore(AuthSqliteConnectionFactory connec
|
||||
);
|
||||
""";
|
||||
|
||||
/// <summary>Inserts a canonical audit event into the <c>audit_event</c> table.</summary>
|
||||
/// <param name="auditEvent">The canonical event to persist.</param>
|
||||
/// <param name="cancellationToken">Token to observe for cancellation.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async Task InsertAsync(AuditEvent auditEvent, CancellationToken cancellationToken)
|
||||
private const string InsertSql =
|
||||
"""
|
||||
INSERT INTO audit_event
|
||||
(event_id, occurred_at_utc, actor, action, outcome,
|
||||
category, target, source_node, correlation_id, details_json)
|
||||
VALUES
|
||||
($event_id, $occurred_at_utc, $actor, $action, $outcome,
|
||||
$category, $target, $source_node, $correlation_id, $details_json);
|
||||
""";
|
||||
|
||||
/// <summary>0 until the <c>audit_event</c> table has been created at least once by this instance.</summary>
|
||||
private int _tableEnsured;
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task EnsureInitializedAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
await using SqliteConnection connection =
|
||||
await connectionFactory.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await EnsureTableAsync(connection, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task InsertAsync(AuditEvent auditEvent, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(auditEvent);
|
||||
|
||||
return InsertBatchAsync([auditEvent], cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// One connection, one transaction and one prepared command for the whole batch: the drain
|
||||
/// pays a single commit for up to <see cref="AuditDrainService.MaxBatchSize"/> events rather
|
||||
/// than one round-trip per event.
|
||||
/// </remarks>
|
||||
public async Task InsertBatchAsync(IReadOnlyList<AuditEvent> auditEvents, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(auditEvents);
|
||||
|
||||
if (auditEvents.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await using SqliteConnection connection =
|
||||
await connectionFactory.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await EnsureTableAsync(connection, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await using SqliteTransaction transaction =
|
||||
(SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
await using (SqliteCommand command = connection.CreateCommand())
|
||||
{
|
||||
command.Transaction = transaction;
|
||||
command.CommandText = InsertSql;
|
||||
|
||||
SqliteParameter eventId = command.Parameters.Add("$event_id", SqliteType.Text);
|
||||
SqliteParameter occurredAtUtc = command.Parameters.Add("$occurred_at_utc", SqliteType.Text);
|
||||
SqliteParameter actor = command.Parameters.Add("$actor", SqliteType.Text);
|
||||
SqliteParameter action = command.Parameters.Add("$action", SqliteType.Text);
|
||||
SqliteParameter outcome = command.Parameters.Add("$outcome", SqliteType.Text);
|
||||
SqliteParameter category = command.Parameters.Add("$category", SqliteType.Text);
|
||||
SqliteParameter target = command.Parameters.Add("$target", SqliteType.Text);
|
||||
SqliteParameter sourceNode = command.Parameters.Add("$source_node", SqliteType.Text);
|
||||
SqliteParameter correlationId = command.Parameters.Add("$correlation_id", SqliteType.Text);
|
||||
SqliteParameter detailsJson = command.Parameters.Add("$details_json", SqliteType.Text);
|
||||
|
||||
foreach (AuditEvent auditEvent in auditEvents)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(auditEvent);
|
||||
|
||||
eventId.Value = auditEvent.EventId.ToString();
|
||||
occurredAtUtc.Value = auditEvent.OccurredAtUtc.ToString("O", CultureInfo.InvariantCulture);
|
||||
actor.Value = auditEvent.Actor;
|
||||
action.Value = auditEvent.Action;
|
||||
outcome.Value = auditEvent.Outcome.ToString();
|
||||
category.Value = (object?)auditEvent.Category ?? DBNull.Value;
|
||||
target.Value = (object?)auditEvent.Target ?? DBNull.Value;
|
||||
sourceNode.Value = (object?)auditEvent.SourceNode ?? DBNull.Value;
|
||||
correlationId.Value = (object?)auditEvent.CorrelationId?.ToString() ?? DBNull.Value;
|
||||
detailsJson.Value = (object?)auditEvent.DetailsJson ?? DBNull.Value;
|
||||
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
await transaction.CommitAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// The comparison goes through SQLite's <c>datetime()</c> rather than comparing the stored
|
||||
/// ISO-8601 text directly. Text comparison is only correct while every row is UTC-normalized
|
||||
/// ISO-8601 — which <see cref="AuditEvent.OccurredAtUtc"/> guarantees for rows written through
|
||||
/// this store, but not for rows that entered the table any other way (a repair script, an
|
||||
/// older schema, a future producer). On a mixed-format column a text comparison silently
|
||||
/// deletes live audit: <c>2026-05-17T09:00:00-05:00</c> is two hours AFTER a
|
||||
/// <c>2026-05-17T12:00:00+00:00</c> cutoff yet sorts before it. Comparing instants is correct
|
||||
/// regardless of how the text got there, and anything <c>datetime()</c> cannot parse yields
|
||||
/// NULL and is therefore never deleted — audit that cannot be dated is kept, not swept.
|
||||
/// </remarks>
|
||||
public async Task<int> DeleteOlderThanAsync(DateTimeOffset cutoffUtc, CancellationToken cancellationToken)
|
||||
{
|
||||
await using SqliteConnection connection =
|
||||
await connectionFactory.OpenConnectionAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
@@ -56,25 +169,14 @@ public sealed class SqliteCanonicalAuditStore(AuthSqliteConnectionFactory connec
|
||||
await using SqliteCommand command = connection.CreateCommand();
|
||||
command.CommandText =
|
||||
"""
|
||||
INSERT INTO audit_event
|
||||
(event_id, occurred_at_utc, actor, action, outcome,
|
||||
category, target, source_node, correlation_id, details_json)
|
||||
VALUES
|
||||
($event_id, $occurred_at_utc, $actor, $action, $outcome,
|
||||
$category, $target, $source_node, $correlation_id, $details_json);
|
||||
DELETE FROM audit_event
|
||||
WHERE datetime(occurred_at_utc) < datetime($cutoff);
|
||||
""";
|
||||
command.Parameters.AddWithValue("$event_id", auditEvent.EventId.ToString());
|
||||
command.Parameters.AddWithValue("$occurred_at_utc", auditEvent.OccurredAtUtc.ToString("O", CultureInfo.InvariantCulture));
|
||||
command.Parameters.AddWithValue("$actor", auditEvent.Actor);
|
||||
command.Parameters.AddWithValue("$action", auditEvent.Action);
|
||||
command.Parameters.AddWithValue("$outcome", auditEvent.Outcome.ToString());
|
||||
command.Parameters.AddWithValue("$category", (object?)auditEvent.Category ?? DBNull.Value);
|
||||
command.Parameters.AddWithValue("$target", (object?)auditEvent.Target ?? DBNull.Value);
|
||||
command.Parameters.AddWithValue("$source_node", (object?)auditEvent.SourceNode ?? DBNull.Value);
|
||||
command.Parameters.AddWithValue("$correlation_id", (object?)auditEvent.CorrelationId?.ToString() ?? DBNull.Value);
|
||||
command.Parameters.AddWithValue("$details_json", (object?)auditEvent.DetailsJson ?? DBNull.Value);
|
||||
command.Parameters.AddWithValue(
|
||||
"$cutoff",
|
||||
cutoffUtc.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture));
|
||||
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
return await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>Returns the most recent canonical audit events, newest first.</summary>
|
||||
@@ -113,7 +215,7 @@ public sealed class SqliteCanonicalAuditStore(AuthSqliteConnectionFactory connec
|
||||
events.Add(new AuditEvent
|
||||
{
|
||||
EventId = Guid.Parse(reader.GetString(0)),
|
||||
OccurredAtUtc = ParseUtc(reader.GetString(1)),
|
||||
OccurredAtUtc = ParseUtcOrMinValue(reader.GetString(1), reader.GetString(0)),
|
||||
Actor = reader.GetString(2),
|
||||
Action = reader.GetString(3),
|
||||
Outcome = Enum.Parse<AuditOutcome>(reader.GetString(4)),
|
||||
@@ -128,13 +230,42 @@ public sealed class SqliteCanonicalAuditStore(AuthSqliteConnectionFactory connec
|
||||
return events;
|
||||
}
|
||||
|
||||
private static async Task EnsureTableAsync(SqliteConnection connection, CancellationToken cancellationToken)
|
||||
// Latched bootstrap: after the first success this is a single volatile read, so the DDL
|
||||
// round-trip is paid once per process rather than once per audit write.
|
||||
private async Task EnsureTableAsync(SqliteConnection connection, CancellationToken cancellationToken)
|
||||
{
|
||||
if (Volatile.Read(ref _tableEnsured) == 1)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await using SqliteCommand command = connection.CreateCommand();
|
||||
command.CommandText = CreateTableSql;
|
||||
await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
Volatile.Write(ref _tableEnsured, 1);
|
||||
}
|
||||
|
||||
private static DateTimeOffset ParseUtc(string value) =>
|
||||
DateTimeOffset.Parse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind);
|
||||
// Reading is defensive where writing is not: an insert always round-trips "O", but the table is
|
||||
// append-only shared state that an operator (or a future migration) can put an unparseable
|
||||
// timestamp into, and the retention sweep deliberately keeps such a row — SQLite's datetime()
|
||||
// yields NULL for it, so the DELETE's comparison is never true. A throwing Parse here would let
|
||||
// that single row take out the dashboard's whole recent-audit view. MinValue instead sorts the
|
||||
// row to the far past and keeps every other column readable, which is what an operator looking
|
||||
// at the view actually needs.
|
||||
private DateTimeOffset ParseUtcOrMinValue(string value, string eventId)
|
||||
{
|
||||
if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out DateTimeOffset parsed))
|
||||
{
|
||||
return parsed;
|
||||
}
|
||||
|
||||
// Debug, not warning: the row is still returned and the timestamp text itself is not logged
|
||||
// (audit rows are not secrets, but the value is attacker-influenceable in the worst case).
|
||||
logger?.LogDebug(
|
||||
"Audit event {EventId} has an unparseable occurred_at_utc; reporting it as DateTimeOffset.MinValue.",
|
||||
eventId);
|
||||
|
||||
return DateTimeOffset.MinValue;
|
||||
}
|
||||
}
|
||||
|
||||
+32
-3
@@ -97,18 +97,47 @@ public static class AuthStoreServiceCollectionExtensions
|
||||
sp.GetService<TimeProvider>() ?? TimeProvider.System));
|
||||
DecorateVerifierWithCache(services, security);
|
||||
|
||||
// GetService, not GetRequiredService, for the same reason the writer registration below
|
||||
// gives: the DI-only unit tests build a bare ServiceCollection with no AddLogging(). The
|
||||
// store's logger is optional and only carries row-level read diagnostics.
|
||||
services.AddSingleton(sp =>
|
||||
new SqliteCanonicalAuditStore(sp.GetRequiredService<AuthSqliteConnectionFactory>()));
|
||||
new SqliteCanonicalAuditStore(
|
||||
sp.GetRequiredService<AuthSqliteConnectionFactory>(),
|
||||
sp.GetService<ILogger<SqliteCanonicalAuditStore>>()));
|
||||
services.AddSingleton<IAuditEventSink>(sp => sp.GetRequiredService<SqliteCanonicalAuditStore>());
|
||||
// Resolve the logger defensively: the production host always registers ILogger<T>, but the
|
||||
// DI-only auth/CLI/dashboard unit tests build a bare ServiceCollection without AddLogging().
|
||||
// Fall back to NullLogger there so the audit writer (and the IApiKeyAuditStore override that
|
||||
// depends on it) still resolve. The write path is best-effort regardless.
|
||||
services.AddSingleton<IAuditWriter>(sp =>
|
||||
services.AddSingleton(sp =>
|
||||
new CanonicalAuditWriter(
|
||||
sp.GetRequiredService<SqliteCanonicalAuditStore>(),
|
||||
sp.GetRequiredService<IAuditEventSink>(),
|
||||
sp.GetService<ILogger<CanonicalAuditWriter>>()
|
||||
?? Microsoft.Extensions.Logging.Abstractions.NullLogger<CanonicalAuditWriter>.Instance));
|
||||
|
||||
// The registered IAuditWriter is the bounded, asynchronous one: audit producers — above
|
||||
// all IConstraintEnforcer.RecordDenialAsync, which fires once per denied tag inside bulk
|
||||
// RPC loops — enqueue and return instead of awaiting a SQLite insert each. No producer
|
||||
// signature changes; the seam is entirely here. AuditDrainService batches the buffered
|
||||
// events onto the sink, owns the one-time schema bootstrap and sweeps expired rows. Where
|
||||
// no hosted service runs (the `apikey` CLI, the DI-only tests) the channel writer falls
|
||||
// back to CanonicalAuditWriter's synchronous path, so audit is never silently buffered
|
||||
// into a channel nobody drains.
|
||||
services.AddSingleton(sp =>
|
||||
new ChannelAuditWriter(
|
||||
sp.GetRequiredService<CanonicalAuditWriter>(),
|
||||
sp.GetService<ILogger<ChannelAuditWriter>>()
|
||||
?? Microsoft.Extensions.Logging.Abstractions.NullLogger<ChannelAuditWriter>.Instance));
|
||||
services.AddSingleton<IAuditWriter>(sp => sp.GetRequiredService<ChannelAuditWriter>());
|
||||
services.AddSingleton(sp => new AuditDrainService(
|
||||
sp.GetRequiredService<ChannelAuditWriter>(),
|
||||
sp.GetRequiredService<IAuditEventSink>(),
|
||||
security,
|
||||
sp.GetService<TimeProvider>() ?? TimeProvider.System,
|
||||
sp.GetService<ILogger<AuditDrainService>>()
|
||||
?? Microsoft.Extensions.Logging.Abstractions.NullLogger<AuditDrainService>.Instance));
|
||||
services.AddHostedService(sp => sp.GetRequiredService<AuditDrainService>());
|
||||
|
||||
// OVERRIDE the library's IApiKeyAuditStore (AddZbApiKeyAuth registered the library's
|
||||
// SqliteApiKeyAuditStore via TryAddSingleton) with an adapter that canonicalizes every
|
||||
// library-emitted ApiKeyAuditEntry onto AuditEvent and forwards it through IAuditWriter.
|
||||
|
||||
@@ -213,7 +213,10 @@ public sealed class CachingApiKeyVerifier : IApiKeyVerifier, IApiKeyCacheInvalid
|
||||
// DashboardApiKeyManagementService.ValidateKeyId each restrict a key id to
|
||||
// char.IsAsciiLetterOrDigit || '.' || '-'. Key ids are never library-generated, so no path can
|
||||
// mint one containing '_'.
|
||||
private static string? TryParseKeyId(string? authorizationHeader)
|
||||
//
|
||||
// Internal rather than private so the parse rules can be pinned directly by test: the guard's
|
||||
// correctness depends on this returning the full key id.
|
||||
internal static string? TryParseKeyId(string? authorizationHeader)
|
||||
{
|
||||
if (string.IsNullOrEmpty(authorizationHeader))
|
||||
{
|
||||
@@ -226,15 +229,29 @@ public sealed class CachingApiKeyVerifier : IApiKeyVerifier, IApiKeyCacheInvalid
|
||||
? header[bearer.Length..].Trim()
|
||||
: header;
|
||||
|
||||
string[] parts = token.ToString().Split('_');
|
||||
if (parts.Length < 3
|
||||
|| !string.Equals(parts[0], TokenPrefix, StringComparison.Ordinal)
|
||||
|| parts[1].Length == 0)
|
||||
// Scanned rather than split, for the same reason as the interceptor's copy: Split would
|
||||
// allocate a token copy, an array and a string per segment on every cache miss to produce
|
||||
// one key id. Two IndexOf scans allocate only that key id.
|
||||
//
|
||||
// The '_' checked immediately after the prefix is what makes "mxgw" the whole first segment
|
||||
// (so "mxgwabc_..." is still rejected), and the second separator must exist because the
|
||||
// split form required three segments — a token with no secret delimiter is not a key token.
|
||||
if (!token.StartsWith(TokenPrefix, StringComparison.Ordinal)
|
||||
|| token.Length <= TokenPrefix.Length
|
||||
|| token[TokenPrefix.Length] != '_')
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return parts[1];
|
||||
ReadOnlySpan<char> afterPrefix = token[(TokenPrefix.Length + 1)..];
|
||||
int separator = afterPrefix.IndexOf('_');
|
||||
if (separator <= 0)
|
||||
{
|
||||
// -1 is a token with no second separator; 0 is an empty key id.
|
||||
return null;
|
||||
}
|
||||
|
||||
return new string(afterPrefix[..separator]);
|
||||
}
|
||||
|
||||
private void IndexCacheKey(string keyId, string cacheKey)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user