Compare commits
27 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2f0cfe33bf | |||
| b021e292ee | |||
| e107019a2a | |||
| e088dfabf7 | |||
| 2e7245dd55 | |||
| 22a878c36e | |||
| b79e119ada | |||
| a2538039c0 | |||
| 1533cd3909 | |||
| 9552b79481 | |||
| 5f4ecabc8a | |||
| 4b8ba7a5bd | |||
| 19cbf7be72 | |||
| c94066dc36 | |||
| df7e20db1d | |||
| 5c8075996f | |||
| 6803bab79a | |||
| b42cbd0730 | |||
| 09f2285957 | |||
| d769244ac0 | |||
| 1a5a12a416 | |||
| 4f5371fdec | |||
| b86c6bb47f | |||
| abb0930359 | |||
| 8bd364f562 | |||
| d6c2ca33f1 | |||
| dca09e7e06 |
+77
-20
@@ -2,15 +2,19 @@
|
||||
#
|
||||
# Scope note: the x86 Worker (src/ZB.MOM.WW.MxGateway.Worker) and Worker.Tests target
|
||||
# .NET Framework 4.8 / x86 and need MXAccess COM installed, so they build ONLY on a Windows host.
|
||||
# They are out of scope for this Linux `portable` job — the `windows` job below (self-hosted windev
|
||||
# runner) covers them, and live-MXAccess integration runs are scheduled-only so they never gate a push.
|
||||
# They are out of scope for this Linux `portable` job — the `windows-x86` job below builds and
|
||||
# tests them on windev (10.100.0.48) over SSH (see scripts/ci/run-windev-ci.sh), and the nightly
|
||||
# `nightly-windev` job additionally runs the live-MXAccess smoke, which is scheduled-only so it
|
||||
# never gates a push.
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
schedule:
|
||||
# Nightly live-MXAccess smoke (windev). Push/PR runs skip the live-mxaccess job.
|
||||
# 06:00 UTC nightly: the `nightly-windev` job re-runs the x86 Worker build + full Worker.Tests on
|
||||
# windev, then the live-MXAccess smoke (MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1) and a full-slnx build.
|
||||
# Push/PR runs skip it — the job is gated `if: github.event_name == 'schedule'`.
|
||||
- cron: '0 6 * * *'
|
||||
|
||||
jobs:
|
||||
@@ -49,6 +53,13 @@ jobs:
|
||||
- name: Build NonWindows solution
|
||||
run: dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx -c Release
|
||||
|
||||
# GitHub-hosted runners ship pwsh; the self-hosted act image does not. Install it as a
|
||||
# .NET global tool (the SDK is already set up above) so the codegen check's `shell: pwsh` works.
|
||||
- name: Install PowerShell (pwsh)
|
||||
run: |
|
||||
dotnet tool install --global PowerShell
|
||||
echo "$HOME/.dotnet/tools" >> "$GITHUB_PATH"
|
||||
|
||||
# IPC-01 / IPC-19 / IPC-20: descriptor set + Contracts/Generated must match the current protos.
|
||||
- name: Codegen / descriptor freshness
|
||||
shell: pwsh
|
||||
@@ -93,32 +104,78 @@ jobs:
|
||||
with:
|
||||
distribution: temurin
|
||||
java-version: '17'
|
||||
# GitHub-hosted runners ship gradle on PATH; the self-hosted act image does not, and the repo
|
||||
# has no gradle wrapper. act also can't resolve the gradle/actions monorepo action, so install
|
||||
# gradle directly (pinned to 9.5.1, matching the local homebrew build in CLAUDE.md/memory).
|
||||
- name: Install Gradle 9.5.1
|
||||
run: |
|
||||
curl -sSL "https://services.gradle.org/distributions/gradle-9.5.1-bin.zip" -o /tmp/gradle.zip
|
||||
sudo unzip -q -d /opt/gradle /tmp/gradle.zip
|
||||
echo "/opt/gradle/gradle-9.5.1/bin" >> "$GITHUB_PATH"
|
||||
- name: Gradle test
|
||||
working-directory: clients/java
|
||||
run: gradle test
|
||||
- name: Revert spurious protobuf-version churn (no .proto changed)
|
||||
run: git checkout -- clients/java/src/main/generated/main/java/mxaccess_gateway/v1/MxaccessGateway.java || true
|
||||
# Both generated aggregates can pick up protobuf-runtime-version churn on regen; revert
|
||||
# both so verify-clean still catches a real, uncommitted proto/codegen change elsewhere.
|
||||
run: |
|
||||
git checkout -- clients/java/src/main/generated/main/java/mxaccess_gateway/v1/MxaccessGateway.java || true
|
||||
git checkout -- clients/java/src/main/generated/main/java/mxaccess_worker/v1/MxaccessWorker.java || true
|
||||
- name: Verify generated tree is clean
|
||||
run: git diff --exit-code -- clients/java/src/main/generated
|
||||
|
||||
windows:
|
||||
# Self-hosted windev runner (10.100.0.48): the only host that can build the x86 / net48 Worker
|
||||
# and MXAccess-adjacent code. Not required for a green portable build; runs where a runner exists.
|
||||
runs-on: [self-hosted, windows]
|
||||
windows-x86:
|
||||
# TST-25: x86 / net48 Worker build + Worker.Tests, executed on windev (10.100.0.48) over SSH.
|
||||
# Runs on a Linux runner so it ALWAYS schedules — act_runner host-mode on Windows is broken (its
|
||||
# hostexecutor writes each step's script to a path it cannot resolve, and it cannot stage JS
|
||||
# actions), and a `runs-on` gate with no matching runner wedges the queue in "queued" forever.
|
||||
# An unreachable windev fails this job RED (loud), never leaves it stuck. A red job can mean the
|
||||
# Windows tier is down rather than the change — see docs/GatewayTesting.md § Continuous Integration.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Build x86 Worker
|
||||
run: dotnet build src/ZB.MOM.WW.MxGateway.Worker/ZB.MOM.WW.MxGateway.Worker.csproj -p:Platform=x86
|
||||
- name: Worker tests
|
||||
run: dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86
|
||||
- name: Worker x86 build + tests on windev
|
||||
env:
|
||||
# WINDEV_SSH_KEY is stored base64-encoded (single line) so Gitea's line-oriented secret
|
||||
# masker redacts it in the step env echo; run-windev-ci.sh decodes it. A raw multiline PEM
|
||||
# would leak in cleartext here (TST-25 acceptance: no key material in logs). Known-hosts is
|
||||
# public and comes from the committed scripts/ci/windev.known_hosts pin (no secret needed).
|
||||
WINDEV_SSH_KEY: ${{ secrets.WINDEV_SSH_KEY }}
|
||||
WINDEV_SSH_USER: ${{ vars.WINDEV_SSH_USER }}
|
||||
CI_SHA: ${{ github.sha }}
|
||||
# `test` = x86 build + Worker.Tests (~50s measured on windev). Demote to `build` only if the
|
||||
# test step ever exceeds ~10 min; the x86 build alone still guards the net48/CS0246/x86 class.
|
||||
run: ./scripts/ci/run-windev-ci.sh test
|
||||
|
||||
live-mxaccess:
|
||||
# Opt-in, scheduled only — needs installed MXAccess COM + live provider state. Never gates a push.
|
||||
if: ${{ github.event_name == 'schedule' }}
|
||||
runs-on: [self-hosted, windows, mxaccess]
|
||||
env:
|
||||
MXGATEWAY_RUN_LIVE_MXACCESS_TESTS: '1'
|
||||
nightly-windev:
|
||||
# TST-25: scheduled deep run — full Worker.Tests + live-MXAccess smoke + full-slnx build on windev.
|
||||
# Gated to the schedule event via `if:` (not an unsatisfiable `runs-on`), so push/PR runs mark it
|
||||
# skipped, not queued. Never gates a push. On failure it opens a Gitea issue so a red nightly is
|
||||
# visible even though nobody watches the Actions page.
|
||||
if: github.event_name == 'schedule'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Live MXAccess smoke
|
||||
run: dotnet test src/ZB.MOM.WW.MxGateway.IntegrationTests/ZB.MOM.WW.MxGateway.IntegrationTests.csproj --filter FullyQualifiedName~WorkerLiveMxAccessSmokeTests
|
||||
- name: Full Worker.Tests + live-MXAccess smoke on windev
|
||||
env:
|
||||
# WINDEV_SSH_KEY is stored base64-encoded (single line) so Gitea's line-oriented secret
|
||||
# masker redacts it in the step env echo; run-windev-ci.sh decodes it. A raw multiline PEM
|
||||
# would leak in cleartext here (TST-25 acceptance: no key material in logs). Known-hosts is
|
||||
# public and comes from the committed scripts/ci/windev.known_hosts pin (no secret needed).
|
||||
WINDEV_SSH_KEY: ${{ secrets.WINDEV_SSH_KEY }}
|
||||
WINDEV_SSH_USER: ${{ vars.WINDEV_SSH_USER }}
|
||||
CI_SHA: ${{ github.sha }}
|
||||
run: ./scripts/ci/run-windev-ci.sh live
|
||||
- name: Report a red nightly as a Gitea issue
|
||||
if: failure()
|
||||
run: |
|
||||
curl -sSf -X POST \
|
||||
-H "Authorization: token ${{ github.token }}" \
|
||||
-H "Content-Type: application/json" \
|
||||
"${{ github.server_url }}/api/v1/repos/${{ github.repository }}/issues" \
|
||||
-d "{\"title\":\"nightly-windev failed on ${{ github.sha }}\",\"body\":\"The scheduled windev Worker + live-MXAccess run failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} . A red nightly may mean the Windows tier is down rather than the change — see docs/GatewayTesting.md (Continuous Integration).\"}"
|
||||
|
||||
# NOTE: there is intentionally no native `windows` runner job. act_runner v0.6.1 host-mode on
|
||||
# Windows is broken and Windows containers are impractical for the net48/x86/MXAccess Worker, so the
|
||||
# Windows tier runs via the SSH-driven `windows-x86` (per-push) and `nightly-windev` (scheduled)
|
||||
# jobs above. Restore a native job only alongside a Windows runner that actually works.
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
# MxAccessGateway — Architecture Review Update (post-remediation)
|
||||
|
||||
Date: 2026-07-12. Branch: `main` at `4f5371f` (merge of `fix/archreview-p2` into main, performed as part of this review so the target includes P0 + P1 + P2 + CI). Working tree: macOS (static review only; NonWindows build + .NET client tests were run to validate the merge, not the findings).
|
||||
|
||||
## Scope and method
|
||||
|
||||
This is a re-run of the 2026-07-08 review ([`../00-overall.md`](../00-overall.md)), executed the same way: six parallel review agents, one per domain, each with three passes — (1) verify every prior finding's tracked status against actual code at HEAD, (2) deep-review the remediation code itself for new defects, (3) a fresh sweep for anything the first review missed. The domain reports carry the full evidence:
|
||||
|
||||
| Report | Domain |
|
||||
|---|---|
|
||||
| [10-gateway-core.md](10-gateway-core.md) | Gateway server: sessions, worker lifecycle, IPC client, gRPC streaming, alarms |
|
||||
| [20-worker.md](20-worker.md) | Worker process: STA pump, COM lifetime, pipe session, event queue |
|
||||
| [30-contracts-ipc.md](30-contracts-ipc.md) | Protos, frame protocol (both sides), codegen pipeline |
|
||||
| [40-security-dashboard.md](40-security-dashboard.md) | API-key auth, LDAP dashboard, SignalR hubs, metrics, diagnostics |
|
||||
| [50-clients.md](50-clients.md) | .NET, Go, Java, Python, Rust clients; cross-client parity |
|
||||
| [60-testing-docs-gaps.md](60-testing-docs-gaps.md) | Test architecture, documentation currency, CI/ops, repo-wide gaps |
|
||||
|
||||
## Overall verdict
|
||||
|
||||
The remediation was real. Every one of the prior review's 1 Critical and 12 High findings is verifiably fixed at HEAD, and **not a single finding marked `Done` in the tracker turned out to be false** — the worst cases are partial (ten `Done` claims carry material caveats, detailed below). The new top line is **0 Critical, 1 High** across 47 new findings, and the one High is an automation gap, not a product defect. The risk profile has genuinely shifted from "correctness defects in the core" to "sharp edges in the newly added machinery plus guard gaps around the Windows tier".
|
||||
|
||||
Three patterns dominate the new findings:
|
||||
|
||||
1. **The remediation's own edges.** Several fixes solve the tracked defect but leave a residual failure mode one layer out: the DrainEvents bound is count-only, so a byte-heavy drain still builds a session-killing oversized frame (WRK-21/IPC-23 — the exact failure the bound exists to prevent); the event-staging channel that decoupled the read loop is unbounded and invisible to the queue-depth gauge (GWC-24); the new gRPC failure limiter can be turned into a lockout-DoS against a legitimate key holder because it partitions on the attacker-supplied key id *before* verification (SEC-31).
|
||||
2. **ReplayGap shipped with broken ends.** The server emits `oldest_available_sequence = 0` when the ring is empty — a spec-compliant client resume then underflows to `2^64−1` and silently drops every event forever (GWC-25, one-line fix) — and while all five client *libraries* surface the sentinel, the Python CLI crashes on it and the Go CLI destroys it (CLI-35/36).
|
||||
3. **Guard gaps the CI can't see.** With the Windows jobs removed, the x86 worker build, all of Worker.Tests (including the new priority-scheduler tests), and the live-MXAccess smoke have zero automation (TST-25); the committed Go and Python *worker* bindings are stale on main right now (missing `max_frame_bytes`; IPC-25); and CI's unconditional `git checkout` of the two Java aggregate files masks real Java drift for any message-level proto change (IPC-24).
|
||||
|
||||
## Prior-finding verification
|
||||
|
||||
- **All Critical/High fixes verified sound at HEAD**, including the alarm-feed split (GWC-01), faulted-session reaping (GWC-02), sparse-array bound (GWC-03), ReadBulk watchdog heartbeat (WRK-01), Go loud overflow (CLI-01), Rust MXSTATUS validation (CLI-03), owner re-validation (TST-02), and CI existence (TST-03, with the Windows-job caveat above).
|
||||
- **`Done` claims with material caveats (10):**
|
||||
- **IPC-04 / WRK-21** — DrainEvents cap is count-based only; oversized reply still faults the whole session and loses drained events.
|
||||
- **IPC-09** — codegen guards have two holes: Java drift masking in CI (IPC-24) and no guard at all over Go/Python worker bindings, which are stale at HEAD (IPC-25).
|
||||
- **WRK-12** — flush coalescing is correct in the writer but never engages on the event hot path: the drain loop awaits each write+flush serially (`WorkerPipeSession.cs:374-382`), so events still cost one flush each.
|
||||
- **WRK-07** — the two-class priority scheduler landed but `docs/MxAccessWorkerInstanceDesign.md` still specifies a five-level order (WRK-26).
|
||||
- **SEC-01** — path-rooting shipped, but `IsRootedForAnyPlatform` accepts Windows paths on Unix, so the shipped appsettings still materializes a literal `C:\ProgramData\...` auth DB on macOS at runtime (SEC-33); Galaxy `SnapshotCachePath` has no rooting check.
|
||||
- **SEC-06** — production plaintext-LDAP hard-stop is enforced, but the committed dev service-account password is still in `appsettings.json`, unrotated (SEC-36).
|
||||
- **CLI-15** — library-complete in all five clients, but the Python CLI crashes on ReplayGap and the Go CLI drops it (CLI-35/36).
|
||||
- **CLI-02** — Rust packaging fix is sound; the required README/ClientPackaging doc updates were skipped (CLI-42).
|
||||
- **CLI-18/21/26/29** — versions aligned to the already-published 0.1.2 despite breaking API changes since; next publish collides or ships a different API under the same version (CLI-39).
|
||||
- **Docs claim** (`ClientLibrariesDesign.md`) that typed helpers use `hresult < 0` — false for .NET/Go/Java, which still use `!= 0` (CLI-08 remains open inside the new surfaces; CLI-38).
|
||||
- **Incidentally fixed open items:** CLI-24 (Java single-consumer javadoc) and CLI-34 (gitignore coverage) can be closed in the tracker.
|
||||
- All other open findings were confirmed still present at HEAD; none silently regressed.
|
||||
|
||||
## Severity roll-up (new findings only)
|
||||
|
||||
| Domain | Critical | High | Medium | Low | Info | IDs |
|
||||
|---|:-:|:-:|:-:|:-:|:-:|---|
|
||||
| Gateway core | — | — | 2 | 4 | 1 | GWC-24..30 |
|
||||
| Worker | — | — | 1 | 7 | — | WRK-21..28 |
|
||||
| Contracts & IPC | — | — | 3 | 5 | 2 | IPC-23..32 |
|
||||
| Security & dashboard | — | — | 1 | 4 | 1 | SEC-31..36 |
|
||||
| Clients | — | — | 5 | 6 | — | CLI-35..45 |
|
||||
| Testing, docs & gaps | — | 1 | 2 | 2 | — | TST-25..29 |
|
||||
| **Total** | **0** | **1** | **14** | **28** | **4** | **47** |
|
||||
|
||||
(Prior review: 1 Critical, 12 High, 54 Medium, 39 Low.)
|
||||
|
||||
## High and headline findings
|
||||
|
||||
### High
|
||||
|
||||
- **TST-25 — the entire Windows/x86 test tier has zero automation.** With the `windows`/`live-mxaccess` CI jobs removed (`abb0930`), the x86 worker build, all of Worker.Tests — including the tests for the *new* priority write scheduler, negotiated frame max, and drain bound — and the live-MXAccess smoke run only when someone remembers the manual windev worktree process, while `docs/GatewayTesting.md` and `check-codegen.ps1` still describe the removed job as "the definitive guard" (TST-26). Cheapest fix: an SSH-driven scheduled step against windev rather than waiting on a Windows act_runner. ([60](60-testing-docs-gaps.md))
|
||||
|
||||
### Medium — headline items
|
||||
|
||||
- **WRK-21 / IPC-23 — DrainEvents byte-blindness is still session-fatal.** The 10,000-count cap doesn't bound bytes; array-heavy events averaging >~1.7 KiB build a reply over `MaxMessageBytes`, the writer's rejection is rethrown out of `WorkerPipeSession.RunAsync`, the worker exits, and the already-drained events are lost. Cap the reply by serialized size as well as count. ([20](20-worker.md), [30](30-contracts-ipc.md))
|
||||
- **GWC-25 — ReplayGap `oldest_available_sequence = 0` on an empty ring.** Violates the proto contract; a compliant client resumes at `0 − 1`, underflows, and the live filter silently drops every event forever. Reachable via the default 300 s age eviction. One-line fix (`_highestSequenceSeen + 1`). ([10](10-gateway-core.md))
|
||||
- **GWC-24 — the event staging channel is unbounded and invisible.** The GWC-04 decoupling removed end-to-end pipe backpressure; a slow-but-live consumer grows gateway memory without bound, without fault, and the queue-depth gauge doesn't count it. ([10](10-gateway-core.md))
|
||||
- **SEC-31 — failure-limiter lockout DoS.** The gRPC limiter partitions on the key id parsed from the *unauthenticated* token and blocks before verification: any peer who knows a key id (non-secret) can send 10 garbage requests/min and lock out the legitimate holder indefinitely. Partition by peer, or count failures only after verification. ([40](40-security-dashboard.md))
|
||||
- **IPC-25 / IPC-24 — codegen guard holes, one already live.** Go and Python worker bindings on main lack `max_frame_bytes` (stale since the 2026-07-09 proto change; nothing checks them); CI's unconditional checkout of `MxaccessGateway.java`/`MxaccessWorker.java` reverts exactly the files where real message-level Java drift would appear. ([30](30-contracts-ipc.md))
|
||||
- **CLI-35/36/37/38 — the new client surfaces re-diverge.** Python CLI crashes on ReplayGap; Go CLI prints a nil event instead of the gap; four clients branch status checks on `success` while the proto mandates `category` (only .NET complies — identical replies pass four clients and throw in one); the `< 0` HRESULT rule is documented but implemented only in Rust/Python. ([50](50-clients.md))
|
||||
- **CLI-39 — version constants aligned to an already-published version.** 0.1.2 now labels a breaking-changed API (Rust stream item type, Python stream union); bump before any publish. ([50](50-clients.md))
|
||||
- **TST-27 — `ShowTagValues` documented as "Reserved" but now live.** SEC-25 wired it into the SignalR mirror redaction; a security-relevant flag is documented as a no-op. ([60](60-testing-docs-gaps.md))
|
||||
|
||||
## Cross-cutting themes
|
||||
|
||||
1. **Residual failure modes one layer out from each fix.** The remediation pattern to watch: the tracked defect is fixed, but the same failure re-emerges at the next boundary (drain bound vs frame max; read-loop decoupling vs unbounded staging; per-frame rejection vs event frames still session-fatal one layer up, IPC-30). Fixes here should be specified against the *invariant* ("no diagnostics command may kill a session", "gateway memory per session is bounded") rather than the symptom.
|
||||
2. **The reconnect/replay story needs an end-to-end walk.** Server sentinel arithmetic (GWC-25), two broken CLIs (CLI-35/36), and no .NET session re-attach (CLI-05) mean the shipped, on-by-default resilience feature still can't be exercised cleanly end-to-end by a user in every language.
|
||||
3. **Cross-client semantic parity regressed inside the new code.** The typed-command epic achieved surface parity but re-introduced behavioral divergence: HRESULT sign semantics (3v2 split), success-vs-category (4v1), credential-scrub strength, malformed-AuthenticateUser handling, and even credential env-var names (CLI-45). A single conformance checklist per behavior, tested per client, is the fix shape.
|
||||
4. **Automation guards still trail the code.** CI exists and is green, but the Windows tier is unguarded (TST-25), two vendored binding sets drifted already (IPC-25), Java drift is masked by the pipeline itself (IPC-24), and the descriptor-freshness *test* is blind to enums/services (IPC-27).
|
||||
5. **The same-commit docs rule was violated by the remediation itself.** TST-26 (removed CI jobs still documented), TST-27 (`ShowTagValues`), WRK-26 (five-level priority spec vs two-class implementation), CLI-42 (Rust packaging docs) — worth one batch pass, since the repo rule exists precisely to prevent this accumulation.
|
||||
6. **Security posture: structural controls hold; new machinery needs a second pass.** Fail-closed interceptor, constant-time compare, redaction seam, owner-scoped streams all verified intact. The new limiter (SEC-31/32), the 15 s verification-cache window vs revocation/expiry (SEC-34), the Unix path-rooting residual (SEC-33), and the still-committed dev LDAP password (SEC-36) are the follow-ups.
|
||||
|
||||
## Prioritized roadmap
|
||||
|
||||
**P0 — correctness/safety, all small**
|
||||
1. GWC-25: emit `_highestSequenceSeen + 1` as `oldest_available_sequence` on an empty ring (one line) + CLI-35/36 CLI ReplayGap handling.
|
||||
2. WRK-21/IPC-23 (+ IPC-30): byte-aware DrainEvents reply capping; no diagnostics command may be session-fatal.
|
||||
3. SEC-31: re-partition the failure limiter (per-peer, or post-verification counting); SEC-32 while in there.
|
||||
4. IPC-25: regenerate Go/Python worker bindings and add them to check-codegen; IPC-24: replace CI's unconditional Java checkout with a real churn-vs-drift discriminator.
|
||||
|
||||
**P1 — process and hardening**
|
||||
5. TST-25: automated Windows tier (SSH-driven windev job for x86 build + Worker.Tests; scheduled live smoke) and fix the docs that claim it exists (TST-26).
|
||||
6. GWC-24: bound the staging channel (or fault on sustained growth) and include it in the queue-depth gauge.
|
||||
7. Client conformance pass: `< 0` HRESULT + `category` branching everywhere (CLI-37/38, closes CLI-08), env-var name alignment (CLI-45), version bump before publish (CLI-39).
|
||||
8. Doc-drift batch: TST-27, WRK-26, CLI-42, plus SEC-33/36 residuals.
|
||||
|
||||
**P2 — the rest**
|
||||
9. New Lows (WRK-22 cancelled-write ghost frames, WRK-24 tiny negotiated max, WRK-27 alarm-poll watchdog bypass, IPC-26/27, SEC-34 cache/revocation window, GWC-26/27/28) alongside the pre-existing open backlog (GWC-05/09/10…, WRK-02/03/05…, SEC-13…27, CLI-05…33, TST-05…24), which this review confirmed unchanged.
|
||||
|
||||
## What is demonstrably good
|
||||
|
||||
The prior review's "preserve under change" list survived the remediation intact: STA pump and COM teardown discipline, inline-proven session locking, frame-protocol validation (now with negotiated maxima and headroom cross-validation at startup), additive-only proto evolution with byte-identical vendored copies, fail-closed auth with constant-time comparison, and the layered fake-worker test architecture. New entries worth preserving: the owner-scoped `StreamEvents` trust boundary, the per-correlation `CommandTooLarge` → `ResourceExhausted` path (fault isolation done right — the model the DrainEvents fix should copy), the pooled single-write gateway framing, and the two-class priority write scheduler's control-before-event guarantee.
|
||||
@@ -0,0 +1,114 @@
|
||||
# Gateway Server Core — Post-Remediation Re-Review
|
||||
|
||||
- **Date:** 2026-07-12
|
||||
- **Commit:** `4f5371f` (`main` — merge of P0/P1/P2 remediation tiers + CI)
|
||||
- **Baseline:** `59856b8` (pre-remediation)
|
||||
- **Method:** static review on macOS; every cited file read in full at HEAD. No build, no test, no source change.
|
||||
- **Scope:** `src/ZB.MOM.WW.MxGateway.Server` — `Sessions/`, `Workers/`, `Grpc/` (command invoke + `EventStreamService`), `Alarms/`. Excludes Security/authn/authz, dashboard, metrics endpoints (other agent) except where session/stream behavior touches them.
|
||||
- **Prior report:** `archreview/10-gateway-core.md` (2026-07-08); remediation design `archreview/remediation/10-gateway-core.md`; status source `archreview/remediation/00-tracking.md`.
|
||||
|
||||
## Prior-finding verification
|
||||
|
||||
Every `Done` claim was verified against code read at HEAD; every open finding was re-checked for incidental fixes. **All 10 Done claims hold and are sound. All 12 open findings are still open (none incidentally fixed); GWC-13 was partially improved.** No Done claim failed verification.
|
||||
|
||||
| ID | Claimed | Verified? | Evidence (HEAD) | Notes |
|
||||
|----|---------|-----------|-----------------|-------|
|
||||
| GWC-01 | Done | ✅ holds | `Workers/WorkerClient.cs:79-92` (`_events` now `SingleReader = true` with invariant comment), `:267-282` (`Interlocked.CompareExchange` claimed-once guard in `ReadEventsAsync`, throws on second consumer); `Sessions/GatewaySession.cs:553-562` (`AttachInternalEventSubscriber`, `isInternal: true`); `Sessions/SessionManager.cs:195-208` (`ReadAlarmEventsAsync` lease-based); `Alarms/GatewayAlarmMonitor.cs:232-234` (monitor consumes mapped `MxEvent`s through the distributor, no raw drain) | Sound. The dual-drain race is structurally closed and future regressions fail loudly. Residual one-time gap at monitor startup → new GWC-26. |
|
||||
| GWC-02 | Done | ✅ holds | `Sessions/GatewaySession.cs:755-772` (`MarkFaulted` stamps `_faultedAtUtc` once), `:847-853` + `:1663-1667` (`IsFaultedReapable(Core)` with `FaultedGraceSeconds`), `:1637` (faulted arm inside the atomic `TryBeginCloseIfExpired` re-check); `Sessions/SessionManager.cs:22` (`FaultedReason`), `:287-293` (sweep ladder lease → faulted → detach-grace); `Configuration/SessionOptions.cs:58` (`FaultedGraceSeconds`, default 0 = next sweep); `Configuration/GatewayOptionsValidator.cs:265-266` | Sound; uses the existing TOCTOU-safe close gate as designed. Clock consistency verified: fault stamp and sweeper share the same `TimeProvider` (`SessionManager.cs:450-454`, `:474-477`). |
|
||||
| GWC-03 | Done | ✅ holds | `Sessions/SparseArrayExpander.cs:72-76` (configured cap checked **before** allocation, message names `MxGateway:Events:MaxSparseArrayLength`), `:78-82` (`Array.MaxLength` backstop kept); `Configuration/EventOptions.cs:39` (default 1,000,000); `Sessions/GatewaySession.cs:1120-1126` (wired from `EventOptions` at the choke point); `Configuration/GatewayOptionsValidator.cs:303-304` (range-validated at startup) | Sound. Note the rejection still throws `RpcException` — the new check inherits open GWC-17's layering leak. |
|
||||
| GWC-04 | Done | ✅ holds | `Workers/WorkerClient.cs:28-33`/`:93-100` (unbounded `_eventStaging`, single-reader/single-writer), `:518-554` (`DispatchEnvelope` fully synchronous; replies/heartbeats/faults/acks never queue behind events), `:565-573` (`StageWorkerEvent` non-blocking `TryWrite`), `:580-593` (`EventWriteLoopAsync` owns the timed write + sustained-overflow `ProtocolViolation` fault, `:610-647`); loop registered in `WaitForBackgroundTasksAsync` (`:1069`) and completed on close/fault/dispose (`:363-365`, `:803-805`, `:837-839`) | Fix works as designed for the reported defect (reply-behind-event stall). However the *unbounded* staging channel introduces a backpressure regression → new GWC-24. |
|
||||
| GWC-05 | Not started | ✅ still open | `Sessions/SessionWorkerClientFactory.cs:158-166` — plain `NamedPipeServerStream(..., PipeOptions.Asynchronous)`, no ACL, no `CurrentUserOnly` | Local pipe-squatting DoS unchanged; `gateway.md` pipe-security contract still unmet. |
|
||||
| GWC-06 | Done | ✅ holds | `Grpc/MxAccessGatewayService.cs:155-163` — `Stopwatch.GetTimestamp()` / `Stopwatch.GetElapsedTime(ts)`, no per-event allocation | |
|
||||
| GWC-07 | Done | ✅ holds | `Grpc/MxAccessGrpcMapper.cs:64-82` — `MapEvent` transfers ownership of `workerEvent.Event` (no `.Clone()`), with the ownership-transfer invariant documented and tied to GWC-01's single-reader guard | Downstream sharing audited: subscribers/replay ring are read-only; the dashboard broadcaster redacts on a deep clone (per SEC-25 change log), so the shared instance is never mutated. Safe. |
|
||||
| GWC-08 | Done | ✅ holds | `Workers/WorkerFrameWriter.cs:57-76` — single `ArrayPool` rent, LE prefix + `WriteTo(Span)`, one `WriteAsync`, `finally` return; `Workers/WorkerFrameReader.cs:51-79` — pooled payload buffer, length-bounded read/parse, returned in `finally`; validation order preserved (`:37-49` before rent) | Sound. Residual nit: the reader still allocates a fresh 4-byte prefix array per frame (`WorkerFrameReader.cs:33`) → GWC-30 (Info). |
|
||||
| GWC-09 | Not started | ✅ still open | `Workers/WorkerProcessStartedProbe.cs:5-19` (no-op `HasExited` check throwing `WorkerProcessLaunchException`); `Workers/WorkerProcessLauncher.cs:290-298` (`ShouldRetryStartupProbe` still excludes `WorkerProcessLaunchException`); `Configuration/WorkerOptions.cs:19-22` (both dead options remain) | Retry pipeline still can never retry. |
|
||||
| GWC-10 | Not started | ✅ still open | `Workers/WorkerEnvelopeValidator.cs:15-39` — validates version/session/body only; no sequence tracking anywhere in `WorkerClient` | Related new gateway-side emission defect → GWC-28. |
|
||||
| GWC-11 | Not started | ✅ still open | `Sessions/GatewaySession.cs:18` (`_workerClient` not volatile); lock-free reads at `:280` (`WorkerProcessId`), `:285` (new public `WorkerClient` property — one **more** unguarded read than at baseline), `:1533` (`CloseAsync`), `:1684`, `:1719`, `:1831` (`DisposeAsync`) | |
|
||||
| GWC-12 | Not started | ✅ still open | `Workers/WorkerClient.cs:353-360` — plain `if (_disposed) return; _disposed = true;`, no interlock | |
|
||||
| GWC-13 | Not started | ✅ still open (partially improved) | `Sessions/GatewaySession.cs:1910-1978` — still a 25 ms poll loop, but the delay now routes through the injected `TimeProvider` (`:1940-1944`), improving testability | Design doc recommended defer-until-used; acceptable. |
|
||||
| GWC-14 | Done | ✅ holds | `Sessions/SessionEventDistributor.cs:100-118` (preallocated circular `ReplayEntry[]`, head+count, invariants documented), `:776-810` (allocation-free append with capacity overwrite), `:814` (`ReplayEntryAt` modular indexing), `:818-831` (age eviction advances head) | Wraparound math verified (`_replayBuffer.Length == _replayBufferCapacity`; capacity-0 guarded before any modulo). Query paths preserve ascending order. Correct. |
|
||||
| GWC-15 | Done | ✅ holds | `Grpc/EventStreamService.cs:123-124` (per-stream backlog source registration; gauge reads `Reader.Count` only at scrape), `:197` (disposed before the lease in `finally`); `Metrics/GatewayMetrics.cs:314-320`, `:560-573` (idempotent registration handle) | |
|
||||
| GWC-16 | Not started | ✅ still open | `Grpc/MxAccessGatewayService.cs:104` (`ResolveSession`) + `:122-123` (`sessionManager.InvokeAsync(request.SessionId, …)`) → `Sessions/SessionManager.cs:163` (`GetRequiredSession`) — still two registry lookups per command | |
|
||||
| GWC-17 | Not started | ✅ still open | `Sessions/SparseArrayExpander.cs:296-297` (`RpcException(InvalidArgument)` below the gRPC layer); invoked from `Sessions/GatewaySession.cs:1043-1045` (comment explicitly documents the leak) | GWC-03's new cap rejection flows through the same leak. |
|
||||
| GWC-18 | Not started | ✅ still open | `Sessions/GatewaySession.cs:13` — `public sealed class GatewaySession` declares no `IAsyncDisposable`; `DisposeAsync` at `:1741` | |
|
||||
| GWC-19 | Not started | ✅ still open | `Sessions/GatewaySession.cs:1684-1688` — `KillWorker(string)` present; grep across `src/` + `clients/` finds zero callers (only `KillWorkerWithCloseGateAsync` is used) | |
|
||||
| GWC-20 | Not started | ✅ still open | `Configuration/WorkerOptions.cs:30-31` (`HeartbeatIntervalSeconds` doc/name unchanged) bound to the check interval at `Sessions/SessionWorkerClientFactory.cs:86`; `Workers/WorkerClient.cs:458` heartbeat loop still uses raw `Task.Delay` without `TimeProvider` | |
|
||||
| GWC-21 | Not started | ✅ still open | `Sessions/SessionWorkerClientFactory.cs:83-89` populates 4 of 6 `WorkerClientOptions`; `EventChannelFullModeTimeout` / `HeartbeatStuckCeiling` fixed at defaults (`Workers/WorkerClientOptions.cs:13`, `:24`) | Post-GWC-04 the full-mode timeout is now the *sole* structural backpressure bound (see GWC-24), which raises the value of exposing it. |
|
||||
| GWC-22 | Not started | ✅ still open | `Grpc/EventStreamService.cs:200` — `metrics.StreamDisconnected("Detached")` hardcoded in the `finally`; fault/overflow paths (`:164-174`) do not differentiate | |
|
||||
| GWC-23 | N/A | ✅ unchanged | Validator comment still documents the knowingly-dead knob | No action, as agreed. |
|
||||
|
||||
Also verified in passing (other-domain fixes landing in gateway-core files, all sound): IPC-03 gateway half — pre-checked command envelope size failing only the offending correlation (`Workers/WorkerClient.cs:209-221`, `CommandTooLarge` → `ResourceExhausted` at `Grpc/MxAccessGatewayService.cs:955`); IPC-02 — `GatewayHello.MaxFrameBytes` conveyed (`WorkerClient.cs:986-988`) with startup cross-validation `Worker.MaxMessageBytes >= gRPC cap + 64 KiB reserve` (`Configuration/GatewayOptionsValidator.cs:481-495`); IPC-04 — `DrainEvents max_events` bounded at 10,000 (`Grpc/MxAccessGrpcRequestValidator.cs:13`, `:81-84`); TST-02 — owner-scoped `StreamEvents` attach (`Grpc/EventStreamService.cs:57-62`).
|
||||
|
||||
## New findings
|
||||
|
||||
IDs continue from the prior register.
|
||||
|
||||
### GWC-24 — The GWC-04 staging channel is unbounded: sustained slow consumption now grows memory instead of faulting, and the backlog is invisible to metrics
|
||||
|
||||
- **Severity:** Medium
|
||||
- **Category:** Stability / backpressure regression
|
||||
- **Location:** `Workers/WorkerClient.cs:93-100` (unbounded `_eventStaging`), `:565-573` (`StageWorkerEvent` `TryWrite` always succeeds), `:610-647` (fault fires only when a *single* timed write exceeds `EventChannelFullModeTimeout`), `:616`/`:627` (`_eventQueueDepth` gauge counts `_events` only)
|
||||
- **Description:** Before GWC-04, a full `_events` channel blocked the read loop, which propagated backpressure through the pipe to the worker's own bounded queue — end-to-end, structurally bounded. Now the read loop stages events into an unbounded channel and only `EventWriteLoopAsync` faces the bounded `_events`. The sustained-overflow fault fires only when one `WriteAsync` waits longer than 5 s; if the consumer (the distributor pump) drains slower than the worker produces but each individual write completes within the window, `_eventStaging` grows without bound and without any fault — and the growth is invisible, because `SetWorkerEventQueueDepth` reflects only `_events`, never the staging backlog. The code comment (`:28-33`) asserts staging "only fills during the bounded EventChannelFullModeTimeout window", which is true only for a full stall, not for sustained slow drain. Mitigating factor: the pump is non-blocking (per-subscriber `TryWrite` fan-out, `SessionEventDistributor.cs:582-594`), so a persistently slow-but-live consumer requires the pump itself to be starved — unlikely but no longer structurally impossible. The repo's fail-fast backpressure invariant now rests on pump liveness rather than on channel bounds.
|
||||
- **Recommendation:** Bound the staging channel (e.g. `2 × EventChannelCapacity`) and treat a `TryWrite` miss there as an immediate `ProtocolViolation` fault (a full staging channel means the event writer is already saturated); alternatively track total staged-but-undelivered depth and fault past a ceiling. Include the staging depth in the worker-event queue-depth gauge either way.
|
||||
|
||||
### GWC-25 — `ReplayGap` sentinel carries `oldest_available_sequence = 0` when the ring is empty, violating the proto resume contract and dead-streaming a compliant client
|
||||
|
||||
- **Severity:** Medium
|
||||
- **Category:** Correctness / contract
|
||||
- **Location:** `Sessions/SessionEventDistributor.cs:463-467` (`RegisterWithReplay`, `_replayCount == 0` branch: `oldestAvailableSequence = 0`); emitted verbatim by `Grpc/EventStreamService.cs:133-139` + `:210-222`; contract at `src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_gateway.proto:755-763`
|
||||
- **Description:** The proto contract states "`oldest_available_sequence` itself IS still retained: a client that wishes to resume without incurring another gap MUST set `after_worker_sequence = oldest_available_sequence - 1`". When a resume finds a gap with an *empty* ring (all entries age-evicted — default `ReplayRetentionSeconds = 300` and `EvictAged` runs inside `RegisterWithReplay` at `:458` — or `ReplayBufferCapacity = 0` with the cursor behind `_highestSequenceSeen`), the sentinel carries `oldest_available_sequence = 0`. A contract-compliant client computes `0 − 1` on a uint64 → `2^64−1`, and its next resume passes `after_worker_sequence = ulong.MaxValue`: `RegisterWithReplay` replays nothing, reports no gap, and the live filter `mxEvent.WorkerSequence <= afterWorkerSequence` (`Grpc/EventStreamService.cs:179`) then drops **every** live event — a silently dead stream with no error. All five clients shipped the `oldest − 1` resume formula in CLI-15 (per the tracking change log), so the failure mode is reachable end-to-end.
|
||||
- **Recommendation:** In the empty-ring-with-gap branch, emit `oldest_available_sequence = _highestSequenceSeen + 1` (the next sequence that can possibly be delivered), so `oldest − 1 = highestSeen` resumes cleanly; keep 0 only for the never-populated case where no gap is reported anyway. Alternatively define the 0 special case in the proto and fix all five clients — the server-side fix is one line and needs no client changes, so prefer it. Add a distributor test for gap-with-empty-ring and an e2e resume after full age eviction.
|
||||
|
||||
### GWC-26 — Alarm monitor attaches its event subscriber only *after* SubscribeAlarms + first reconcile; transitions in that window bypass the feed, and a missed Acknowledge is never broadcast
|
||||
|
||||
- **Severity:** Low
|
||||
- **Category:** Stability / alarm-feed completeness (residual of GWC-01)
|
||||
- **Location:** `Alarms/GatewayAlarmMonitor.cs:213-214` (subscribe + reconcile) precede the attach at `:232-234`; late-subscriber semantics at `Sessions/SessionEventDistributor.cs:579-582`; `ApplyReconcile` presence-delta-only broadcast at `Alarms/GatewayAlarmMonitor.cs:514-553`
|
||||
- **Description:** The distributor pump has been running since `MarkReady` (dashboard mirror registers first, `Sessions/GatewaySession.cs:445-449`, `:621`), and a subscriber registered mid-stream misses previously fanned events by design. Alarm transitions arriving between the worker's `SubscribeAlarms` activation and the monitor's internal-subscriber registration (two full worker command round-trips later) are fanned only to the dashboard mirror. Raise/Clear are repaired by the next reconcile, but an Acknowledge in the window updates `_alarms` silently on the next reconcile snapshot replace (`:547-551`) without any `Broadcast` — live `StreamAlarms` subscribers keep showing the alarm unacked until it clears. This is the same defect class GWC-01 fixed, shrunk from "random half of all events, forever" to "a one-shot window of a few hundred ms per monitor lifecycle".
|
||||
- **Recommendation:** Attach the internal subscriber before invoking `SubscribeAlarmsAsync` (take the lease from `AttachInternalEventSubscriber` explicitly, then subscribe, then drain), or buffer the lease's channel during subscribe/reconcile. Also consider broadcasting an acked-state delta from `ApplyReconcile` when an existing reference's `CurrentState` changes.
|
||||
|
||||
### GWC-27 — `AttachInternalEventSubscriber` / `ReadAlarmEventsAsync` bypass the readiness gate; a premature attach permanently poisons the session's event distributor
|
||||
|
||||
- **Severity:** Low
|
||||
- **Category:** Latent lifecycle hazard
|
||||
- **Location:** `Sessions/GatewaySession.cs:553-562` (no `_state == Ready` check, contrast `AttachEventSubscriber` `:889-916`); `Sessions/SessionManager.cs:195-208` (public `ReadAlarmEventsAsync`, also unchecked)
|
||||
- **Description:** The internal-attach path starts the distributor pump unconditionally. If invoked before the session is Ready, the pump's source (`MapWorkerEventsAsync` → `GetReadyWorkerClientAsync`, `:740-749`/`:1910`) throws `SessionNotReady`; `PumpAsync` completes all subscribers with the error and latches `_completed` (`Sessions/SessionEventDistributor.cs:603-612`, `:662-680`). Because `_eventDistributorStarted` is never reset (`Sessions/GatewaySession.cs:527-531`) and the distributor is replaced only in `DisposeAsync`, every later registration — including gRPC `StreamEvents` attaches — receives an immediately-completed channel carrying the stale error: a Ready session with permanently dead event streaming. Not reachable today (the alarm monitor, the only caller, attaches after `OpenSessionAsync` returns a Ready session), but the hazard is one future call site away and fails in the worst way (silent, permanent, per-session).
|
||||
- **Recommendation:** Mirror `AttachEventSubscriber`'s readiness check at the top of `AttachInternalEventSubscriber` (throw `SessionManagerErrorCode.SessionNotReady` before `EnsureDistributorCreated`).
|
||||
|
||||
### GWC-28 — Gateway→worker envelope `sequence` is stamped at envelope creation, not at write, so concurrent invokes can emit non-monotonic sequences on the wire
|
||||
|
||||
- **Severity:** Low
|
||||
- **Category:** IPC contract / conventions
|
||||
- **Location:** `Workers/WorkerClient.cs:1031-1044` (`CreateEnvelope` stamps `Interlocked.Increment(ref _nextSequence)`); enqueue at `:957-972`; single write loop `:390-409`; the gateway `WorkerFrameWriter` does not re-stamp (`Workers/WorkerFrameWriter.cs:35-77`)
|
||||
- **Description:** Two concurrent `InvokeAsync` callers can stamp sequences 1 and 2 but win the channel `WriteAsync` race in the order 2, 1, putting out-of-order sequences on the pipe. This is exactly the defect WRK-04 fixed on the worker side (its writer now stamps at the point of writing under the write lock); the gateway half was not given the same treatment. Benign today — nothing validates inbound sequence on either side (GWC-10/IPC-10 both open) — but it violates gateway.md's "monotonic per sender" contract and would start faulting sessions the moment worker-side validation is added.
|
||||
- **Recommendation:** Move the stamp into `WriteLoopAsync` immediately before `_writer.WriteAsync` (single consumer, naturally serialized), matching the worker's WRK-04 fix; coordinate with GWC-10 if enforcement is added.
|
||||
|
||||
### GWC-29 — `Invoke` deep-clones the entire request only to discard the cloned command
|
||||
|
||||
- **Severity:** Low
|
||||
- **Category:** Performance (hot path)
|
||||
- **Location:** `Grpc/MxAccessGatewayService.cs:119-121`; `Grpc/MxAccessGrpcMapper.cs:27-37`
|
||||
- **Description:** `MxCommandRequest invokeRequest = request.Clone()` deep-clones the request *including its command payload* (potentially a large bulk-write graph), then immediately overwrites `invokeRequest.Command` with `commandToInvoke`, discarding the cloned command. `MapCommand` then performs the one clone that is actually needed (`request.Command.Clone()`). Net: one full wasted command deep-clone on every `Invoke` — the same cost class GWC-07/IPC-05 eliminated on the event and envelope paths, still present on the command path.
|
||||
- **Recommendation:** Skip the request clone: add a `MapCommand(MxCommand command)` overload (or construct a minimal request) since `MapCommand` reads only the command, and pass `commandToInvoke` directly.
|
||||
|
||||
### GWC-30 — Frame reader still allocates a fresh 4-byte prefix array per frame
|
||||
|
||||
- **Severity:** Info
|
||||
- **Category:** Performance (residual of GWC-08)
|
||||
- **Location:** `Workers/WorkerFrameReader.cs:33`
|
||||
- **Description:** The GWC-08 design suggested a per-reader scratch buffer for the length prefix; the payload was pooled but the `new byte[sizeof(uint)]` per frame remains. Gen-0, 4 bytes — negligible; recorded only so the next hot-path pass knows it was seen, not missed.
|
||||
- **Recommendation:** Optional: a reusable per-reader prefix field (the reader is single-consumer by construction).
|
||||
|
||||
## Severity roll-up (new findings)
|
||||
|
||||
| Severity | Count | IDs |
|
||||
|----------|:-:|-----|
|
||||
| Critical | 0 | — |
|
||||
| High | 0 | — |
|
||||
| Medium | 2 | GWC-24, GWC-25 |
|
||||
| Low | 4 | GWC-26, GWC-27, GWC-28, GWC-29 |
|
||||
| Info | 1 | GWC-30 |
|
||||
|
||||
## Still good / preserved
|
||||
|
||||
The qualities the prior review praised survived the remediation churn intact. Session state writes remain single-lock disciplined (`_syncRoot` everywhere, including the new fault-timestamp and faulted-reap paths); the close gate still serializes Close/Kill/sweep with atomic TOCTOU re-checks (`TryBeginCloseIfExpired` gained the faulted arm without weakening the reattach-wins race resolution). The replay→live handoff argument still holds in the circular-array rewrite — the `_replayLock`-atomic snapshot+register in `RegisterWithReplay` is unchanged in structure and its no-gap/no-duplicate documentation is exemplary. The rewritten ring's eviction, ordering, and wraparound logic are correct. The GWC-01 fix is a model remediation: it doesn't just fix the bug, it converts the whole bug class into a loud `InvalidOperationException` via `SingleReader = true` plus the claimed-once guard. Repo invariants were respected throughout: no synthesized events (the `ReplayGap` sentinel is a documented control signal, correctly never fanned or drained elsewhere), one worker per session, and the frame wire format is byte-identical after the pooled-I/O rewrite. Convention adherence (file-scoped namespaces, `sealed`, `Async` suffixes, MXAccess-aligned naming) remains strong in all new code; `GatewaySession` has grown to 2,128 lines and the split-out recommendation from the prior report stands.
|
||||
@@ -0,0 +1,104 @@
|
||||
# Worker Process — Architecture Review (re-run)
|
||||
|
||||
## Scope & method
|
||||
|
||||
Date 2026-07-12 · commit `4f5371f` (`main`) · macOS static review — the worker targets .NET Framework 4.8 x86 and was **not built or tested** here; every claim cites file:line evidence read directly from the working tree at HEAD. Scope: `src/ZB.MOM.WW.MxGateway.Worker` (STA runtime/pump, `MxAccessSession` COM lifetime, `Ipc/` pipe session + frame reader/writer, event queue, alarm consumers, program lifecycle/exit codes) and `src/ZB.MOM.WW.MxGateway.Worker.Tests`. This is a verification pass over the 2026-07-08 review (`archreview/20-worker.md`, remediation IDs WRK-01..WRK-20 in `archreview/remediation/20-worker.md` / `00-tracking.md`) plus a deep review of everything changed since the pre-remediation baseline `59856b8` and a lighter fresh sweep.
|
||||
|
||||
Changed-in-scope since `59856b8` (17 files, +890/−86): `Ipc/WorkerFrameWriter.cs` (priority scheduler, sequence-at-write, batch flush), `Ipc/WorkerFrameWritePriority.cs` (new), `Ipc/WorkerFrameProtocolOptions.cs` (negotiated max adoption), `Ipc/WorkerPipeSession.cs` (event priority tag, DrainEvents bound, sequence removal), `Sta/StaRuntime.cs` (pump refreshes activity), `Conversion/MxStatusProxyConverter.cs` (FieldInfo cache), `MxAccess/MxAccessEventQueue.cs` + `MxAccessValueCache.cs` (clone elimination + cache snapshot), comment strips in the two alarm consumers, and six test files.
|
||||
|
||||
## Executive summary
|
||||
|
||||
- All seven Done-claimed fixes (WRK-01, 04, 06, 07, 11, 12, 15) are present at HEAD and fundamentally sound; none is a false Done. Two carry caveats: WRK-12's flush coalescing never engages on the pure event hot path it was aimed at (WRK-25), and WRK-07 landed without updating the component design doc that still specifies a five-level priority order (WRK-26).
|
||||
- The new cooperative frame-write scheduler is well built — enqueue-then-contend, control-before-event drain under a single lock, sequence stamped at the moment of writing, per-frame rejections isolated from stream failures, no deadlock or lost-wakeup path found. Its weakest edges are cancellation (a cancelled `WriteAsync` leaves its frame queued and it is still written later, WRK-22) and sequence numbers consumed by rejected frames (WRK-23).
|
||||
- The most significant new defect is that the DrainEvents bound is count-based only: 10,000 large events can still exceed the negotiated frame max, and because the drained events were already removed from the queue, the oversized-reply rejection both loses those events and unwinds the whole session as a protocol violation (WRK-21) — the exact "session-killing reply frame" the bound was added to prevent.
|
||||
- All thirteen open findings (WRK-02/03/05/08/09/10/13/14/16/17/18/19 and the WRK-02/03 slice of WRK-20) remain present at HEAD, none incidentally fixed. The highest-value remaining items are still WRK-02 (silent STA thread death) and WRK-05 (one alarm-poll failure kills the session).
|
||||
- Fresh sweep: the alarm poll loop runs outside the command dispatcher, so it never gets the watchdog's in-flight suppression — a single slow `PollOnce` COM call faults `StaHung` after the 15 s grace instead of the 75 s ceiling other commands get (WRK-27).
|
||||
|
||||
## Prior-finding verification
|
||||
|
||||
| ID | Claimed status | Verified? | Evidence (file:line at HEAD) | Notes |
|
||||
|----|----------------|-----------|------------------------------|-------|
|
||||
| WRK-01 | Done | **Yes** | `Sta/StaRuntime.cs:95-100` (`PumpPendingMessages` → `MarkActivity()`); wiring `MxAccess/MxAccessStaSession.cs:208` (`pumpStep: () => staRuntime.PumpPendingMessages()`), `MxAccess/MxAccessSession.cs:885` (per-tag wait routes through `pumpStep`); docs `docs/MxAccessWorkerInstanceDesign.md:690-703`; tests `StaRuntimeTests.PumpPendingMessages_RefreshesLastActivity`, `WorkerPipeSessionTests.RunAsync_LongInFlightCommandThatKeepsPumping_DoesNotFaultAndDeliversReply` | Sound. A ReadBulk that keeps pumping stays fresh past the 75 s ceiling; a stuck (non-pumping) STA still faults. Residual by design: a single blocking COM call cannot pump and still faults at the ceiling. See WRK-27 for the alarm-poll asymmetry the fix does not cover. |
|
||||
| WRK-02 | Not started | **Still open** | `Sta/StaRuntime.cs:265-269` (loop exception stored in write-only `startupException`, `startedEvent.Set()`); `Sta/StaRuntime.cs:161-189` (`InvokeAsync` has no terminated check; enqueues into a consumer-less queue after thread death) | `finally` at :272 cancels only work queued *at* death; anything enqueued after death hangs forever, exception never logged/faulted. Unchanged since baseline. |
|
||||
| WRK-03 | Not started | **Still open** | `Ipc/WorkerPipeSession.cs:717-720` (`if (!_acceptingCommands) { return; }` — no reply, no log) | Silent drop during shutdown race unchanged. |
|
||||
| WRK-04 | Done | **Yes** | `Ipc/WorkerFrameWriter.cs:236-240` (sequence stamped inside `WriteFrameAsync`, which runs only under `_writeLock` per :116-125); `Ipc/WorkerPipeSession.cs:1025-1035` (`CreateBaseEnvelope` no longer sets `Sequence`; `NextSequence()`/`_nextSequence` removed from the session); test `WorkerFrameProtocolTests.WriteAsync_UnderConcurrentCalls_StampsGapFreeMonotonicSequence` | Sound: wire order and stamped sequence agree under concurrency and priority reordering. Nuance: rejected frames consume sequence numbers (WRK-23). |
|
||||
| WRK-06 | Done | **Yes** | `Conversion/MxStatusProxyConverter.cs:23` (static `ConcurrentDictionary<Type, StatusFields>`), :122-142 (`GetFields`/`ResolveField`, throw-through `GetOrAdd` so a bad type is not cached), :96-108 (per-field `GetValue`+`Convert.ToInt32` retained), :186-207 (plain `readonly struct`, net48-safe) | Sound; exception messages preserved (missing-field from `ResolveField`, null-value from `ReadInt32Field`). Test `Convert_RepeatedForSameType_ProducesIdenticalResults`. |
|
||||
| WRK-07 | Done | **Yes, with doc caveat** | `Ipc/WorkerFrameWritePriority.cs:10-17` (Control/Event); `Ipc/WorkerFrameWriter.cs:88-98` (priority enqueue), :200-216 (`DequeueNext` control-first), :141-157 (per-frame rejection vs stream-failure semantics), :218-232 (`FailAllQueued`); `Ipc/WorkerPipeSession.cs:374-382` (events tagged `Event`); test `WriteAsync_WhenControlAndEventQueuedTogether_WritesControlFirst` | Scheduler is correct and deadlock-free (analysis below). But it is a **two-class** scheduler while `docs/MxAccessWorkerInstanceDesign.md:607-613` still specifies the five-level order (faults > replies > shutdown acks > heartbeats > events) — within Control it is FIFO. Doc not updated in the same change (WRK-26). |
|
||||
| WRK-05 | Not started | **Still open** | `MxAccess/MxAccessStaSession.cs:278-291` (any single poll exception → `RecordFault` → loop stops); drain loop turns fault into session termination `Ipc/WorkerPipeSession.cs:356-365` | One transient alarm E_FAIL still kills the whole session (default `standbyFactory: null`, `Ipc/WorkerPipeSession.cs:60`). |
|
||||
| WRK-08 | Not started | **Still open** | `Ipc/WorkerPipeSession.cs:298-304` (drain loop cancelled in `finally` after shutdown ack, residual queue unshipped); `docs/MxAccessWorkerInstanceDesign.md:705-718` shutdown section still silent on the discard | |
|
||||
| WRK-09 | Not started | **Still open** | No `AppDomain`/`UnhandledException`/`UnobservedTaskException` reference anywhere under `src/ZB.MOM.WW.MxGateway.Worker/` (grep at HEAD) | |
|
||||
| WRK-10 | Not started | **Still open** | `WorkerApplication.cs:112-117, 123-127, 133-137` (all three catch blocks log `exception_type` only, never the redacted message) | The redactor (`Bootstrap/WorkerLogRedactor.cs`) still exists and works (nonce redaction verified via `WorkerConsoleLogger.cs:36`), so this stays a cheap win. |
|
||||
| WRK-11 | Done | **Yes** | `MxAccess/MxAccessEventQueue.cs:146-160` (stamps sequence/timestamp on the caller's instance, no `Clone()`, ownership invariant documented :11-21); `MxAccess/MxAccessValueCache.cs:43-70` (cache deep-copies `Value`/`SourceTimestamp`/`Statuses.Clone()` so the snapshot never aliases the queue-owned event); callers verified single-ownership (`MxAccess/MxAccessBaseEventSink.cs:205-256` builds a fresh event per enqueue; `postPublish` runs after enqueue at :243-256 and only *reads* the event — safe against the concurrent drain-side serialization, which is also read-only) | Sound. Tests `Enqueue_TakesOwnershipOfPassedEventInstance`, `Set_StoresIndependentSnapshot_UnaffectedByLaterEventMutation`. |
|
||||
| WRK-12 | Done | **Yes, but partially ineffective** | `Ipc/WorkerFrameWriter.cs:120-124, 160-182` (write each frame, one `FlushAsync` per drained batch, complete after flush; flush failure fails the whole written batch); test `WriteAsync_WhenBatchDrainedTogether_FlushesOnce` | The writer-side mechanism is correct, but the production event drain loop awaits each event write to completion before issuing the next (`Ipc/WorkerPipeSession.cs:374-382`), so the writer's queue never accumulates an event batch — see WRK-25. The tracking-log claim "a burst of N events costs 1 flush not N" does not hold on the event path. |
|
||||
| WRK-13 | Not started | **Still open** | `Ipc/WorkerFrameWriter.cs:261-266` — still `new byte[frameLength]` per frame (now a single combined prefix+payload buffer and one stream write, a modest improvement over baseline, but still unpooled vs the reader's `ArrayPool` at `Ipc/WorkerFrameReader.cs:55-77`) | |
|
||||
| WRK-14 | Not started | **Still open** | `Ipc/WorkerPipeSession.cs:28-44` `_camelCase` vs `MxAccess/MxAccessSession.cs:11-14` and `Sta/StaRuntime.cs:10-24` bare `camelCase` | |
|
||||
| WRK-15 | Done | **Yes** | `docs/WorkerSta.md:23,29` and `docs/MxAccessWorkerInstanceDesign.md:254` now state the actual thread name `MxGateway.Worker.STA` (matches `Sta/StaRuntime.cs:61`); heartbeat-counter note fixed at `docs/MxAccessWorkerInstanceDesign.md:651-654` ("populated from the live event queue") | |
|
||||
| WRK-16 | Not started | **Still open** | `Ipc/WorkerPipeSession.cs:940-1023` (seven `CreateEnvelope`/`CreateBaseEnvelope` pairs); `Ipc/WorkerPipeClient.cs` (7 public constructors) | |
|
||||
| WRK-17 | Not started | **Still open** | `WorkerApplication.cs:110-119` maps every `WorkerFrameProtocolException` (including `EndOfStream`) to `ProtocolViolation` (6); `PipeConnectionFailed` (5) reserved for `IOException`/`TimeoutException` at :121-129 | |
|
||||
| WRK-18 | Not started | **Still open** | `Ipc/WorkerPipeSession.cs:356-365` (drain-fault writes fault then throws `InvalidOperationException`) → generic catch `WorkerApplication.cs:131-139` → exit 1 | Overflow doc text at `docs/MxAccessWorkerInstanceDesign.md:614-621` ("stop accepting new commands") also still stale. |
|
||||
| WRK-19 | Not started | **Still open** | `Sta/StaCommandDispatcher.cs` has no logger and no start/end log (grep: no `Information`/`Error` emit sites) | |
|
||||
| WRK-20 | Not started | **Partially closed as a side effect** | Added since baseline: pump-refresh + long-in-flight-no-fault tests (WRK-01), gap-free concurrent-sequence test (WRK-04), control-before-event and flush-once tests (WRK-07/12), converter cache, queue ownership, cache snapshot, DrainEvents-bound tests | Still missing (because their fixes are open): STA-thread-death test (WRK-02) and command-refused-during-shutdown test (WRK-03). |
|
||||
|
||||
Done claims that failed verification outright: **none**. Defective/incomplete aspects of Done claims: WRK-12 effectiveness (WRK-25) and WRK-07 documentation (WRK-26).
|
||||
|
||||
## Deep review of the new code
|
||||
|
||||
**Frame-write scheduler correctness.** Enqueue happens before the lock wait (`Ipc/WorkerFrameWriter.cs:87-98`), every caller then contends for `_writeLock` and the winner drains all queued frames control-first (:100-113, :125-182), so there is no lost-wakeup: a frame enqueued while another holder drains is either drained by that holder or by its own caller's subsequent lock acquisition. Completions use `RunContinuationsAsynchronously` (:28), stream writes run with `CancellationToken.None` so frames are never half-written (:116-118, :266), `_nextSequence` is only touched under the lock (:44-48), and stream failure fails the current frame, the written-but-unflushed batch, and everything queued (:148-157, :165-176) so no caller hangs. Per-frame rejections (`InvalidEnvelope`/`MessageTooLarge`/version/session, :192-198) correctly fail only that frame. Event ordering is preserved because the single drain loop awaits each event write (`Ipc/WorkerPipeSession.cs:374-382`). No deadlock or STA-affinity issue found — the writer runs entirely on thread-pool/loop threads.
|
||||
|
||||
**Negotiated max adoption.** `AdoptNegotiatedMaxMessageBytes` (`Ipc/WorkerFrameProtocolOptions.cs:124-140`) is applied in `ValidateGatewayHello` before `WorkerHello` is written and before the message loop starts (`Ipc/WorkerPipeSession.cs:235-239` then :192), so the single-threaded-handshake mutation claim holds; the reader/writer read the property per frame and pick the negotiated value up for all post-hello frames. 0-keeps-default and >256 MiB-rejected are implemented and tested. Gap: no lower bound (WRK-24).
|
||||
|
||||
**DrainEvents bound.** `max_events = 0` and over-large requests are clamped to `MaxDrainEventsPerReply = 10_000` (`Ipc/WorkerPipeSession.cs:21-26, 545-557`), matching the gateway validator's `MaxDrainEventsPerRequest = 10_000` (`src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGrpcRequestValidator.cs:13,81-85`). Semantics change from "drain all" to "up to 10,000" is reasonable for a diagnostic command. The bound is count-only — see WRK-21.
|
||||
|
||||
**Watchdog / long COM calls.** The heartbeat loop and `ReportWatchdogFaultIfNeededAsync` (`Ipc/WorkerPipeSession.cs:790-881`) are unchanged in structure: suppression while `CurrentCommandCorrelationId` is non-empty up to `HeartbeatStuckCeiling` (75 s default, `Ipc/WorkerPipeSessionOptions.cs:19`), and WRK-01's pump refresh now keeps pumping commands fresh indefinitely. Exit-code/fault mapping unchanged (WRK-17/18 still apply).
|
||||
|
||||
## New findings
|
||||
|
||||
**WRK-21 — Medium (stability).** The DrainEvents bound is count-based, so an oversized reply still kills the session — and now also loses the drained events.
|
||||
Evidence: `Ipc/WorkerPipeSession.cs:545-557` clamps only the event *count*; the reply is written via `WriteControlReplyAsync` (:481, :485-496) with no catch; a reply whose serialized size exceeds `MaxMessageBytes` throws `WorkerFrameProtocolException(MessageTooLarge)` from the writer (`Ipc/WorkerFrameWriter.cs:250-255`) — a "per-frame" rejection at the writer, but `HandleControlCommandAsync`/`DispatchGatewayEnvelopeAsync` propagate it out of `RunMessageLoopAsync`, so the session dies and `WorkerApplication.cs:110-119` exits `ProtocolViolation` (6). The 10,000 drained events were already removed from `MxAccessEventQueue` by `runtimeSession.DrainEvents(maxEvents)` (:551) and are gone. 10,000 events at ~1.7 KiB each (large string/array `MxValue`s) exceeds the default 16 MiB + 64 KiB frame max, so the comment's stated goal ("cannot pack the whole queue into one session-killing reply frame", :21-26) is not met for large events.
|
||||
Recommendation: bound by bytes as well as count (stop adding events once the reply's `CalculateSize()` approaches `MaxMessageBytes` minus headroom), or catch `MessageTooLarge` on control-reply writes and reply with a `ResourceExhausted`-style error for that correlation instead of unwinding the session. The same catch would also stop an oversized *STA command* reply from faulting the whole session at `Ipc/WorkerPipeSession.cs:640-655`.
|
||||
|
||||
**WRK-22 — Low (stability).** A cancelled `WriteAsync` leaves its frame queued, and the frame is still written later by an unrelated drain.
|
||||
Evidence: the frame is enqueued before the cancellable lock wait (`Ipc/WorkerFrameWriter.cs:87-103`); if `_writeLock.WaitAsync(cancellationToken)` throws, nothing removes the frame, so the next lock-holder writes it. Concrete scenario: at shutdown the event-drain loop's token is cancelled (`Ipc/WorkerPipeSession.cs:298-304`) while its event frame waits for the lock; the shutdown-ack write then drains queues — ack first (Control), then the orphaned event — so an event frame is emitted *after* `WorkerShutdownAck`, and a write reported as cancelled to its caller still reaches the wire.
|
||||
Recommendation: on cancellation, remove the frame from its queue under `_gate` (mark the `PendingFrame` cancelled and have `DequeueNext` skip completed frames), or document that cancellation only abandons the wait, never the write.
|
||||
|
||||
**WRK-23 — Low (conventions/diagnostics).** Rejected frames consume sequence numbers, producing wire gaps.
|
||||
Evidence: `Ipc/WorkerFrameWriter.cs:236-255` stamps `++_nextSequence` (:240) before the empty-payload and `MessageTooLarge` checks (:243-255), so a per-frame rejection leaves a hole in the on-wire sequence. `gateway.md:328` calls `sequence` a monotonic *diagnostic* counter, so nothing breaks, but the new test (`WorkerFrameProtocolTests.WriteAsync_UnderConcurrentCalls_StampsGapFreeMonotonicSequence`) asserts a gap-free stream, documenting a stronger expectation than the code guarantees once a rejection occurs.
|
||||
Recommendation: run the size/empty checks before stamping (move the stamp to just before `WriteTo`), keeping sequences contiguous.
|
||||
|
||||
**WRK-24 — Low (stability/hardening).** `AdoptNegotiatedMaxMessageBytes` has no lower-bound sanity check.
|
||||
Evidence: `Ipc/WorkerFrameProtocolOptions.cs:124-140` — 0 keeps the default and >256 MiB throws, but any value from 1 byte up is adopted; a gateway bug or a foreign/old gateway sending e.g. 512 would leave the session alive but unable to write any frame (every write fails `MessageTooLarge`) and unable to read most inbound frames, failing confusingly mid-session rather than at the handshake. The gateway's own validator floors its config at 1024 (`src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs:10,230-233`), but the worker's declared intent (:14-18, "a nonsensical negotiated value is rejected at the handshake") is only enforced at the top end.
|
||||
Recommendation: reject negotiated values below a sane floor (e.g. 64 KiB, or at least the gateway's 1024 minimum) with the same `InvalidConfiguration` error.
|
||||
|
||||
**WRK-25 — Low (performance).** The WRK-12 flush coalescing never engages on the event hot path it targeted.
|
||||
Evidence: the event drain loop awaits each event's `WriteAsync` — which completes only after that frame is written *and flushed* — before writing the next (`Ipc/WorkerPipeSession.cs:374-382`), so the writer's event queue holds at most one frame from the drain loop and every event still costs one flush. The batch path (`Ipc/WorkerFrameWriter.cs:120-124, 160-182`) only coalesces when independent producers happen to queue behind a blocked in-progress write (control frames, or a slow pipe) — the tracking-log claim "a burst of N events now costs 1 flush not N" does not describe production behavior.
|
||||
Recommendation: issue the drained batch's writes without awaiting each (collect the tasks and `Task.WhenAll` after the loop — the writer already guarantees per-queue FIFO order for a single producer), or add an explicit `WriteBatchAsync`; either makes the existing coalescing effective.
|
||||
|
||||
**WRK-26 — Low (conventions/doc drift).** The write-priority and overflow sections of the component design doc were not updated with the WRK-07 change.
|
||||
Evidence: `docs/MxAccessWorkerInstanceDesign.md:607-613` still specifies the five-level priority (faults > replies > shutdown acks > heartbeats > events) while the implementation is a two-class scheduler (Control FIFO, then Event — `Ipc/WorkerFrameWritePriority.cs:10-17`); within Control a fault does not jump queued heartbeats/replies. `docs/WorkerFrameProtocol.md` describes the negotiated max (:25-30) but says nothing about priority classes, flush coalescing, or the per-frame vs stream-failure semantics. CLAUDE.md requires affected docs to change in the same commit as the source.
|
||||
Recommendation: rewrite `docs/MxAccessWorkerInstanceDesign.md:607-613` to describe the implemented two-class scheduler (and note the accepted FIFO-within-control decision), and add a short "write scheduling" paragraph to `docs/WorkerFrameProtocol.md`. Fold in the stale overflow-policy text (WRK-18's doc half) while there.
|
||||
|
||||
**WRK-27 — Low (stability).** The alarm poll runs outside the dispatcher, so it never gets the watchdog's in-flight-command suppression.
|
||||
Evidence: `MxAccess/MxAccessStaSession.cs:245-253` invokes `handler.PollOnce()` via `staRuntime.InvokeAsync` directly, bypassing `StaCommandDispatcher`; `CaptureHeartbeat` takes `CurrentCommandCorrelationId` only from the dispatcher (`MxAccess/MxAccessStaSession.cs:364-381`), so during a long `PollOnce` the heartbeat shows no in-flight command and the watchdog fires `StaHung` as soon as staleness exceeds `HeartbeatGrace` (15 s default) rather than the 75 s ceiling granted to dispatched commands (`Ipc/WorkerPipeSession.cs:843-861`). A healthy-but-slow `GetXmlCurrentAlarms2` (large alarm sets, busy provider) blocking the STA >15 s faults a healthy session. Partially defensible — a blocked STA can't deliver data events either — but the 15 s vs 75 s asymmetry between polls and commands is unintentional.
|
||||
Recommendation: surface poll-in-progress to the heartbeat snapshot (e.g. a synthetic correlation id or a boolean the suppression branch also honors), or run the poll through the dispatcher.
|
||||
|
||||
**WRK-28 — Low (conventions).** The 10,000 drain cap is a duplicated magic constant with a comment-only synchronization contract.
|
||||
Evidence: `Ipc/WorkerPipeSession.cs:26` (`MaxDrainEventsPerReply = 10_000`, "Kept in step with that public ceiling") vs `src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGrpcRequestValidator.cs:13` (`MaxDrainEventsPerRequest = 10_000`). Both projects already share the Contracts assembly (`net10.0;net48`), which could own the constant.
|
||||
Recommendation: move the ceiling into `ZB.MOM.WW.MxGateway.Contracts` (e.g. next to `GatewayContractInfo`) and reference it from both sides.
|
||||
|
||||
## Severity roll-up (new findings)
|
||||
|
||||
| Severity | Count | IDs |
|
||||
|----------|-------|-----|
|
||||
| High | 0 | — |
|
||||
| Medium | 1 | WRK-21 |
|
||||
| Low | 7 | WRK-22, WRK-23, WRK-24, WRK-25, WRK-26, WRK-27, WRK-28 |
|
||||
|
||||
## Still good / preserved
|
||||
|
||||
The strengths recorded in the 2026-07-08 review are intact at HEAD: the STA core still runs the canonical `MsgWaitForMultipleObjectsEx` + `PeekMessage`/`DispatchMessage` loop with the wake event and 50 ms idle pump (`Sta/StaRuntime.cs:255-261`, `Sta/StaMessagePump.cs`); COM lifetime discipline (`FinalReleaseComObject`, sink-detach-before-release, UnAdvise → RemoveItem → Unregister order) is unchanged in `MxAccess/MxAccessSession.cs` and both alarm consumers (the only alarm-consumer diffs since baseline are XML-doc comment removals — commit `b86c6bb`); partial pipe reads, hello validation, dispatcher backpressure (128-entry bound), record-after-COM-success handle bookkeeping, and value-cache eviction on `RemoveItem` all match the prior positive observations; nothing synthesizes events (`MxAccess/MxAccessBaseEventSink.cs:160-171`); nonce/credential redaction still works and is applied by the console logger (`Bootstrap/WorkerLogRedactor.cs:18`, `Bootstrap/WorkerConsoleLogger.cs:36`). The new writer preserves the "written and flushed before the caller's task completes" contract, the ownership-transfer refactor is properly invariant-documented on the queue class, and the new tests (sequence monotonicity, control-before-event, flush-once, negotiated-max bounds, drain bound, pump refresh, long-in-flight no-fault, ownership/snapshot independence) close the WRK-01/04/06/07/11/12 slices of the WRK-20 gap. net48 constraints are respected throughout the new code (plain ctors, `readonly struct`, no init-only members).
|
||||
|
||||
## Verification needed on Windows (windev)
|
||||
|
||||
- This review is static: the x86 worker was not built; `Worker.Tests` were not run. The tracking log records windev-green runs for the P0/P1/P2 batches (352–356 passed), but any fix for WRK-21..28 needs an x86 build + filtered `WorkerFrameProtocolTests`/`WorkerPipeSessionTests` run on windev.
|
||||
- WRK-21 deserves a windev repro test: enqueue ~10,000 large events, issue `DrainEvents max_events=0`, assert the session survives (currently expected to die with exit code 6).
|
||||
- WRK-22's post-ack event emission is best confirmed against the real gateway `WorkerClient` (does it tolerate/ignore frames after `WorkerShutdownAck`?) — gateway side is outside this domain's scope.
|
||||
@@ -0,0 +1,98 @@
|
||||
# Contracts & IPC Protocol — Architecture Review (re-run)
|
||||
|
||||
Date: 2026-07-12 · Commit: `4f5371f` (main) · macOS static review (no build/test; sources read at HEAD)
|
||||
Prior review: [../30-contracts-ipc.md](../30-contracts-ipc.md) (2026-07-08, baseline `59856b8`) · Remediation plan: [../remediation/30-contracts-ipc.md](../remediation/30-contracts-ipc.md)
|
||||
|
||||
## Scope & method
|
||||
|
||||
Same domain as the prior review: `src/ZB.MOM.WW.MxGateway.Contracts` (three protos, `Generated/` hygiene, net10.0;net48 multi-targeting), the frame protocol on both sides (`src/ZB.MOM.WW.MxGateway.Worker/Ipc/*`, `src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerFrame*` + `WorkerClient`), the codegen/descriptor pipeline (`scripts/check-codegen.ps1`, `scripts/publish-client-proto-inputs.ps1`, `clients/proto/*`, vendored client proto copies incl. `clients/rust/protos`), and proto-evolution hygiene. Method: static reading at HEAD, `git diff 59856b8..HEAD` for the delta, verification of each remediation claim in `archreview/remediation/00-tracking.md` against source.
|
||||
|
||||
## Proto evolution since baseline — verified additive-only
|
||||
|
||||
`git diff 59856b8..HEAD -- '*.proto'` shows exactly two things: (1) one additive field, `uint32 max_frame_bytes = 4;` on `GatewayHello` (`src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_worker.proto:47`, next free tag, documented zero-means-default semantics — safe for old peers); (2) new vendored Rust proto copies (`clients/rust/protos/*.proto`), byte-identical to the Contracts sources (`cmp` clean for all three files). No renumbering, no repurposing, `reserved 1` / `reserved "session_id"` still present on the retired fields (`mxaccess_gateway.proto:915-916,929-930`), wire-compat policy headers intact. `galaxy_repository.proto` retention comment intact (`ZB.MOM.WW.MxGateway.Contracts.csproj:37-42`). Proto hygiene held.
|
||||
|
||||
## Prior-finding verification
|
||||
|
||||
| ID | Claimed | Verified? | Evidence (HEAD) | Notes |
|
||||
|----|---------|-----------|-----------------|-------|
|
||||
| IPC-01 | Done | **Yes** | Descriptor refreshed commit `309296f` (contains `MxSparseArray` ×2, `max_frame_bytes` ×1 via `strings`); semantic freshness test `src/ZB.MOM.WW.MxGateway.Tests/Contracts/ClientProtoInputTests.cs:21-59`; CI runs `scripts/check-codegen.ps1` + the test project (`.gitea/workflows/ci.yml:60-65`) | Test has blind spots — see IPC-27 |
|
||||
| IPC-02 | Done | **Yes** | Gateway sends `MaxFrameBytes` in hello (`Server/Workers/WorkerClient.cs:988` from `FrameOptions.MaxMessageBytes`, wired from `Worker.MaxMessageBytes` at `Sessions/SessionWorkerClientFactory.cs:73-76`); worker adopts after version+nonce checks, before the message loop (`Worker/Ipc/WorkerPipeSession.cs:239`; `WorkerFrameProtocolOptions.cs:124-140`): 0 → keep default, >256 MiB (`MaxNegotiableFrameBytes`, :19) → handshake fault. Docs: `docs/WorkerFrameProtocol.md:25-30`, `docs/GatewayConfiguration.md:120` | Adoption is single mutation pre-loop on the shared options instance; reader/writer read the property per frame — no TOCTOU. Handshake-fault path writes a fault then throws (`WorkerPipeSession.cs:198-201`) — fail-fast as designed |
|
||||
| IPC-03 | Done | **Yes** | `EnvelopeOverheadReserveBytes` = 64 KiB (`Server/Workers/WorkerFrameProtocolOptions.cs:20`); default pipe max = 16 MiB + reserve (`Server/Configuration/WorkerOptions.cs:44-45`); startup cross-validation with `long` arithmetic (no overflow) at `GatewayOptionsValidator.cs:476-498`; per-command pre-check at enqueue (`WorkerClient.cs:207-221`, `WorkerClientErrorCode.CommandTooLarge` at `WorkerClientErrorCode.cs:19`) → `ResourceExhausted` (`Grpc/MxAccessGatewayService.cs:955`); write-loop `MessageTooLarge` kept fatal as genuine desync (`WorkerClient.cs:390-408`) | Envelope not mutated between pre-check and write (gateway stamps `Sequence` at creation, `WorkerClient.cs:1039`) — no check/write TOCTOU. One doc gap: see IPC-28 |
|
||||
| IPC-04 | Done | **Partly** | Public validator cap 10 000 (`Grpc/MxAccessGrpcRequestValidator.cs:13,81-84`); worker clamp `MaxDrainEventsPerReply` = 10 000, 0 → cap (`Worker/Ipc/WorkerPipeSession.cs:26,547-550`) | Cap is **count-based only**; a byte-oversized reply still kills the worker — residual finding IPC-23 |
|
||||
| IPC-05 | Done | **Yes** | Second clone removed (`WorkerClient.cs:1000-1007`, transfers `command` into the envelope); `MapEvent` transfers ownership with an audit comment (`Grpc/MxAccessGrpcMapper.cs:64-76`); `MapCommand`'s isolating clone kept (:34); `docs/Grpc.md` clone prose updated (~:218-220) | |
|
||||
| IPC-06 | Done | **Yes** | `gateway.md:303-337`: hand-copied sketch replaced by proto-as-source-of-truth prose; `string correlation_id` correct (:311-313); all ten arms incl. `worker_shutdown_ack` listed (:316-324) | |
|
||||
| IPC-07 | Done | **Yes** | `docs/Grpc.md:13,32` say seven RPCs incl. `QueryActiveAlarms`; handler section at :97-99; rules-table row at :180 | |
|
||||
| IPC-08 | Open | **Confirmed open** | Zero `WorkerCancel` references in `src/ZB.MOM.WW.MxGateway.Server` outside `Generated/` (grep); worker dispatch still present (`WorkerPipeSession.cs:414-416`); `gateway.md:319` lists the arm ("best-effort cancellation") and :751-757 still describes the discard-late-reply contract | Still half-implemented: the arm is dead from the gateway side, the worker's `CancelCommand` path unreachable |
|
||||
| IPC-09 | Done | **Mostly** | Python: pinned grpcio-tools 1.80.0 + `Assert-GrpcioToolsVersion` (`clients/python/generate-proto.ps1:10,36-41`) + PATH-resolved python (:16-34); Go: `Resolve-Tool` PATH-first (`clients/go/generate-proto.ps1:9-31`); Java: `checkGeneratedClean` (`clients/java/zb-mom-ww-mxgateway-client/build.gradle:61-70`); CI java job verifies the generated tree (`ci.yml:120-121`) | Two guard holes found: the CI churn-revert masks real Java drift (IPC-24) and Go/Python committed bindings have no freshness guard at all — and are in fact stale at HEAD (IPC-25) |
|
||||
| IPC-10 | Open | **Confirmed open (softened)** | Validators still don't check sequence; but `gateway.md:328-330` now honestly states sequence is a diagnostic aid, not validated (the "annotate" half of the original recommendation). Proto comment still absent (`mxaccess_worker.proto:23` bare) | Acceptable state; see also IPC-31 (gateway stamp-vs-wire-order) |
|
||||
| IPC-11 | Open | **Confirmed open** | Strict equality check (`WorkerPipeSession.cs:221-226`); no min/max-range migration comment on `supported_protocol_version` (`mxaccess_worker.proto:42`) | No action was required; still true |
|
||||
| IPC-12 | Open | **Confirmed open (surface reduced)** | `Server/Workers/WorkerFrameWriter.cs` has no lock and no single-writer invariant note in its class doc (:8-11); all writes still funnel through the single write loop (`WorkerClient.cs:390-397`) | IPC-13's single `WriteAsync` per frame removed the two-write interleaving seam, but concurrent callers on the same `Stream` remain unsafe and undocumented |
|
||||
| IPC-13 | Done | **Yes** | Single serialization into one ArrayPool-rented prefixed buffer, one write (`Server/Workers/WorkerFrameWriter.cs:57-76`); wire format unchanged (LE uint32 + payload) | |
|
||||
| IPC-14 | Done | **Yes** | Pooled payload read, length-bounded `ParseFrom(payload, 0, length)`, return in `finally` (`Server/Workers/WorkerFrameReader.cs:51-79`) | Buffer never escapes; `ParseFrom` copies — lifetime sound |
|
||||
| IPC-15 | Done (as scoped) | **Yes** | Flush coalescing across the drained batch with written-and-flushed completion contract preserved (`Worker/Ipc/WorkerFrameWriter.cs:116-182`); multi-event envelope body intentionally not added (no `WorkerEventBatch` in the proto — grep 0); `gateway.md:887-890` correctly distinguishes shipped flush-coalescing from the deferred additive batching | |
|
||||
| IPC-16 | N/A | **Improved** | `gateway.md:331-333` documents the envelope `correlation_id` as authoritative with the inner reply as parity echo; proto comments still absent | |
|
||||
| IPC-17 | Done | **Yes** | `docs/ClientProtoGeneration.md:110,175` and `CLAUDE.md:83` use `clients/python/src/zb_mom_ww_mxgateway/generated`; grep finds no stale `src/mxgateway/generated` anywhere | |
|
||||
| IPC-18 | Preserve | **Preserved** | `Contracts/Generated/MxaccessWorker.cs` carries `MaxFrameBytes` (×15) — committed C# in sync with the proto; Rust vendored protos byte-identical; galaxy proto wire-identity held | Exception: Go/Python worker bindings — IPC-25 |
|
||||
| IPC-19 | Done | **Yes** | Regen-and-commit rule with net48/CS0246 rationale in `docs/Contracts.md:100-115` and in the csproj comment (`ZB.MOM.WW.MxGateway.Contracts.csproj:27-33`); enforced by `check-codegen.ps1` Check 2 (force-delete `Generated/*.cs`, rebuild, fail on `git status --porcelain` diff, lines 42-65) in CI | |
|
||||
| IPC-20 | Done | **Yes** | protoc pinned to 34.1 (`scripts/publish-client-proto-inputs.ps1:15`); regeneration refuses a non-pinned protoc (:161-163); `-Check` is version-tolerant by normalizing both sides through the *same* protoc with source info stripped (`New-SourceInfoFreeDescriptor`, :59-79, compare at :145-147); CI installs the pinned protoc (`ci.yml:43-47`) | Sound design; the same-binary normalization cancels version-specific encoding |
|
||||
| IPC-21 | Done | **Yes** | `gateway.md:341-362` shows the real seven-RPC service; the bidi `Session` sketch is now framed as "was considered as a possible long-term shape" (:368), outside the live service block | |
|
||||
| IPC-22 | N/A | **Unchanged** | Status-code-plus-prose model unchanged; `AcknowledgeAlarmReply.status` placeholder comment intact | Info; no action planned |
|
||||
|
||||
**Done claims that do not fully hold:** IPC-04 (residual byte-size failure mode, → IPC-23) and IPC-09 (guard coverage holes, → IPC-24/IPC-25). All other Done claims verified sound.
|
||||
|
||||
## New findings
|
||||
|
||||
### IPC-23 — Medium — `DrainEvents` bound is count-based only; a byte-heavy queue still builds a session-killing reply frame — Stability
|
||||
`Worker/Ipc/WorkerPipeSession.cs:537-562` caps the drain at `MaxDrainEventsPerReply` = 10 000 events (:26), but never sizes the reply. At the default negotiated frame max (16 MiB + 64 KiB), events averaging above ~1.7 KiB — exactly the array-heavy payloads this gateway exists for — overshoot `MaxMessageBytes`. The worker writer correctly rejects the oversized frame per-frame (`Worker/Ipc/WorkerFrameWriter.cs:250-255`, `IsPerFrameRejection` :192-198), so the *stream* survives, but the exception surfaces from `WriteControlReplyAsync` (`WorkerPipeSession.cs:481,485-496`) through `DispatchGatewayEnvelopeAsync` (:406) into `RunMessageLoopAsync` (:280) and out of `RunAsync` (:126) — the worker exits and the drained events are lost. This is the original S3 failure narrowed, not closed: a diagnostics command can still kill the session.
|
||||
**Recommendation:** size-aware packing in `CreateDrainEventsReply` (stop adding events when `reply.CalculateSize()` approaches the cap, report the truncation), or catch the per-frame rejection at the control-reply seam and answer with a protocol-status error reply instead of dying.
|
||||
|
||||
### IPC-24 — Medium — CI's unconditional churn-revert step masks real Java generated-code drift for message-level proto changes — Underdeveloped
|
||||
`.gitea/workflows/ci.yml:114-121` runs `git checkout -- .../MxaccessGateway.java` and `.../MxaccessWorker.java` unconditionally before the "Verify generated tree is clean" step. The Java bindings are single-file aggregates (`java_multiple_files` unset), so *any* message/field change to a proto lands exactly and only in those two files. A `.proto` edit committed without regenerating the Java client therefore passes CI — the freshly regenerated (correct) files are reverted to the stale committed ones, and the diff check sees a clean tree. The step's own comment scopes it to "when no .proto changed", but nothing enforces that condition. This partially neuters the IPC-09 `checkGeneratedClean` intent; only service-level changes (which touch the `*Grpc.java` stubs) would still be caught.
|
||||
**Recommendation:** gate the revert on the push/PR diff containing no `*.proto` change (e.g., `git diff --name-only $BASE..HEAD -- '*.proto'`), or diff the two aggregates modulo the known protobuf-runtime-version lines instead of discarding them wholesale.
|
||||
|
||||
### IPC-25 — Medium — Committed Go and Python worker bindings are stale at HEAD, and no guard covers them — Underdeveloped
|
||||
`clients/go/internal/generated/mxaccess_worker.pb.go` (last regenerated 2026-05-23, commit `397d3c5`) and `clients/python/src/zb_mom_ww_mxgateway/generated/mxaccess_worker_pb2.py` (2026-06-18, `e328758`) both predate the 2026-07-09 `max_frame_bytes` addition — grep finds zero occurrences in either, while the Java aggregate (regenerated 2026-07-10) has it. The freshness net closed by P1 covers `Contracts/Generated` (check-codegen Check 2), the descriptor (Check 1 + test), Rust vendored protos (Check 3), and Java (checkGeneratedClean) — but **nothing** compares the committed Go/Python bindings against the protos, and their unit tests don't exercise the worker types, so CI stays green. Functional impact today is nil (no language client consumes `GatewayHello`), but the published packages misrepresent the wire contract and the repo rule ("regenerate every generated client touched by the contract", CLAUDE.md verification matrix) was violated without any signal — the exact silent-drift class IPC-01/09 were meant to end.
|
||||
**Recommendation:** regenerate and commit both; add a Check 4 to `scripts/check-codegen.ps1` (or per-client steps in CI) that regenerates Go/Python bindings and fails on diff, with the same churn-tolerance handling the Python pin already provides.
|
||||
|
||||
### IPC-26 — Low — Worker frame writer: cancelling while waiting for the write lock leaves a ghost frame that is still written — Stability
|
||||
`Worker/Ipc/WorkerFrameWriter.cs:87-113`: the frame is enqueued (:88-98) *before* `_writeLock.WaitAsync(cancellationToken)` (:103). If the caller's token fires while waiting, `WriteAsync` throws `OperationCanceledException` but the `PendingFrame` stays queued — the next lock-holder (heartbeat, reply, event) writes it anyway, and its `Completion` resolves with no observer. Consequences: a write the caller observed as cancelled still reaches the wire; during shutdown, leftover cancelled event frames drain in the same batch as (behind) the `WorkerShutdownAck`, i.e. events can be emitted after the ack that is nominally the last frame; and if no writer ever follows, the frame lingers unwritten with an unobserved completion. Benign today (the gateway drops post-shutdown/unknown frames), but the "cancelled means not written" contract is silently violated.
|
||||
**Recommendation:** on cancellation, remove or tombstone the pending frame (e.g., `TrySetCanceled` and skip tombstoned frames in `DequeueNext`).
|
||||
|
||||
### IPC-27 — Low — The descriptor freshness test checks messages and fields only — enums, enum values, services/methods, and the Galaxy contract are blind spots — Underdeveloped
|
||||
`src/ZB.MOM.WW.MxGateway.Tests/Contracts/ClientProtoInputTests.cs:35-52` collects `{message}` and `{message}/{field}` symbols and enumerates only `MxaccessGatewayReflection.Descriptor` and `MxaccessWorkerReflection.Descriptor`. A new enum value (e.g., a new `MxCommandKind` case — the most likely future contract change), a new RPC on a service, or any change confined to `galaxy_repository.proto` would not redden the test. The script-side `-Check` in CI *does* catch all of these (full descriptor compare), so the gate as-a-whole holds — but `docs/ClientProtoGeneration.md:152-157` presents the test as the protoc-free primary guard, and on any runner without protoc it is the only one.
|
||||
**Recommendation:** extend the test to enum values, service methods, and the Galaxy file descriptor (all available via the same reflection surface).
|
||||
|
||||
### IPC-28 — Low — `docs/Grpc.md` exception-mapping prose omits the new `CommandTooLarge` → `ResourceExhausted` mapping — Conventions
|
||||
`Grpc/MxAccessGatewayService.cs:955` maps `WorkerClientErrorCode.CommandTooLarge` to `ResourceExhausted`, and the IPC-03 plan called for documenting that Invoke now returns `ResourceExhausted` for oversized payloads. `docs/Grpc.md:250` still enumerates only `CommandTimeout`/`GatewayShutdown`/`InvalidState`/`ProtocolViolation` with "unmapped codes fall through to `Unavailable`" — a reader concludes an oversized command surfaces as `Unavailable`. (`docs/GatewayConfiguration.md:120-129` does document the headroom rule and per-correlation failure.)
|
||||
**Recommendation:** add `CommandTooLarge → ResourceExhausted` to the mapping prose and the Invoke section.
|
||||
|
||||
### IPC-29 — Low — Worker writer's priority scheduling and write-time sequence stamping are undocumented in the frame-protocol doc — Conventions
|
||||
The worker writer is now a cooperative priority scheduler (control frames drain ahead of event frames) that stamps `Sequence` at the moment of writing under the lock (`Worker/Ipc/WorkerFrameWriter.cs:11-17,116-124,238-240`; `WorkerFrameWritePriority.cs`). These are protocol-relevant semantics: a per-frame rejection consumes a sequence number (stamped at :240 before the size check at :250 throws), so the wire sequence now has gaps after rejections, and event frames can be arbitrarily delayed behind a control backlog. `docs/WorkerFrameProtocol.md:1-43` still describes the frame layer as only "message boundaries, size limits, protobuf parsing, and envelope validation"; `gateway.md:328-330` covers the diagnostic-sequence half but not scheduling. Given the repo's docs-change-with-source rule, the component doc should carry this.
|
||||
**Recommendation:** add a short "Write scheduling and sequencing" section to `docs/WorkerFrameProtocol.md`.
|
||||
|
||||
### IPC-30 — Low — An oversized worker→gateway event frame is still session-fatal despite the per-frame rejection machinery — Stability
|
||||
`Worker/Ipc/WorkerPipeSession.cs:374-382`: the event drain loop awaits each event write; a `MessageTooLarge` per-frame rejection from the writer (an MXAccess array event serializing above the negotiated max) throws out of `RunEventDrainLoopAsync`, is rethrown by the message loop's `await eventDrainTask` (:294), and terminates `RunAsync` — the whole session dies for one unsendable event. Pre-existing behavior (not a regression), but the writer now *survives* the rejection and the session dies anyway one layer up, which wastes the new machinery. Handling it gracefully needs a policy decision (silently dropping an event conflicts with the delivery expectations behind "don't synthesize events"; a `WorkerFault` or a synthesized-loss marker are the honest options).
|
||||
**Recommendation:** decide and document the oversized-event policy; at minimum log the event identity before dying so the operator can find the offending tag.
|
||||
|
||||
### IPC-31 — Info — Gateway envelope sequence is stamped at creation, not at write, so wire order can disagree with sequence order — Stability
|
||||
`Server/Workers/WorkerClient.cs:1039` assigns `Sequence` via `Interlocked.Increment` inside `CreateEnvelope`; the envelope is then enqueued to the outbound channel. Two concurrent `InvokeAsync` callers can increment and enqueue in opposite orders, producing non-monotonic sequences on the wire. The worker fixed exactly this on its side (WRK-04: stamp under the write lock at the point of writing, `Worker/Ipc/WorkerFrameWriter.cs:238-240`); the gateway was not given the same treatment. Harmless — `gateway.md:328-330` declares sequence a diagnostic aid, never validated — but the two sides now implement different semantics for the same field.
|
||||
**Recommendation:** none required; if sequence ever becomes load-bearing, stamp in the gateway write loop.
|
||||
|
||||
### IPC-32 — Info — `check-codegen.ps1` check labels are wrong — Conventions
|
||||
`scripts/check-codegen.ps1:30,42,68` print "Check 1/2", "Check 2/2", then "Check 3/3" — the middle label predates the third check. Cosmetic only; CI log readers see a miscounted progression.
|
||||
**Recommendation:** relabel 1/3, 2/3, 3/3.
|
||||
|
||||
## Severity roll-up (new findings)
|
||||
|
||||
| Severity | Count | IDs |
|
||||
|----------|-------|-----|
|
||||
| High | 0 | — |
|
||||
| Medium | 3 | IPC-23, IPC-24, IPC-25 |
|
||||
| Low | 5 | IPC-26, IPC-27, IPC-28, IPC-29, IPC-30 |
|
||||
| Info | 2 | IPC-31, IPC-32 |
|
||||
|
||||
## Still good / preserved
|
||||
|
||||
The frame protocol core remains sound and got materially better since baseline: length validation before allocation on both sides (`Server/Workers/WorkerFrameReader.cs:36-49`, `Worker/Ipc/WorkerFrameReader.cs:36-49`), exact-read loops with typed `EndOfStream`, pooled buffers with correct lifetimes on both readers and the gateway writer (rented buffers never escape; `ParseFrom` copies; returns in `finally`), single-serialization single-write framing on both writers, and the new negotiated `max_frame_bytes` handshake with a zero-fallback for old peers, a 256 MiB sanity ceiling, and adoption sequenced safely before the message loop. The 64 KiB headroom invariant is enforced at startup with overflow-safe arithmetic, and the oversized-command path now fails one correlation instead of the session. Proto hygiene held: the only contract change since baseline is additive, reserved ranges and `UNSPECIFIED` zero enums are intact, `Contracts/Generated/` is in sync with the protos, the Rust vendored copies are byte-identical, and the descriptor set is fresh and doubly guarded (semantic test + normalized, protoc-pinned script check, both wired into CI). The doc layer at the contract boundary (`gateway.md` envelope section, seven-RPC surface, `Session` future-work framing, Python generated path, `Generated/` regen rule) now matches the shipped contract.
|
||||
|
||||
Windows-only residue for this domain: the worker-side behaviors verified statically here (negotiated-max adoption, priority scheduler, drain clamp, batch flush) are exercised only by `Worker.Tests -p:Platform=x86` on windev; the tracking log records them green there (356/0/11 on 2026-07-09), which this static review did not re-run.
|
||||
@@ -0,0 +1,98 @@
|
||||
# Security & Dashboard — Re-review (Post-Remediation)
|
||||
|
||||
**Date:** 2026-07-12 · **Commit:** `4f5371f` (`main`) · **Baseline:** `59856b8` (pre-remediation) · **Method:** static review on the macOS working tree (no builds, no runtime probes, no source modifications). Every claim cites a `path:line` read at HEAD. Prior findings are the `SEC-NN` IDs from `archreview/remediation/40-security-dashboard.md`; new findings continue from SEC-31.
|
||||
|
||||
## Prior-finding verification
|
||||
|
||||
| ID | Claimed | Verified? | Evidence (file:line at HEAD) | Notes |
|
||||
|----|---------|-----------|------------------------------|-------|
|
||||
| SEC-01 | Done | **Partial** | Defaults now `SpecialFolder`-derived: `Configuration/AuthenticationOptions.cs:16-19`, `Configuration/TlsOptions.cs:18-22`. Rooting enforced: `GatewayOptionsValidator.cs:110-113` (SqlitePath), `:442-445` (SelfSignedCertPath). Hygiene test: `src/ZB.MOM.WW.MxGateway.Tests/ProjectStructure/GatewayTreeHygieneTests.cs:12-27`. Stray tracked-tree DB deleted (no `*.db` under `src/` outside `bin/`). | Two residuals: (a) `IsRootedForAnyPlatform` (`GatewayOptionsValidator.cs:541-560`) deliberately passes Windows-rooted paths on Unix, so the shipped `appsettings.json` still materializes a literal `C:\ProgramData\...` file at runtime on macOS — a fresh stray auth DB dated **2026-07-09 (post-fix)** exists at `src/ZB.MOM.WW.MxGateway.Tests/bin/Debug/net10.0/C:\ProgramData\MxGateway\gateway-auth.db` (gitignored; excluded by the hygiene test's bin/obj filter, `GatewayTreeHygieneTests.cs:30-35`). (b) Galaxy `SnapshotCachePath` (`appsettings.json:80`) has no rooting validation — the Galaxy section is not covered by `GatewayOptionsValidator`. → **SEC-33** |
|
||||
| SEC-02 | Done | **Yes** | `Dashboard/DashboardAuthorizationHandler.cs:39-53`: both the `Mode==Disabled` and loopback bypasses fire only when `requirement.RequiredRoles.Contains(DashboardRoles.Viewer)`; `AdminOnly` is `[Admin]` only (`Dashboard/DashboardAuthorizationRequirement.cs:19-20`), so bypasses can never satisfy it. | Enforcement is at the policy layer as designed; forwarded-headers caveat is documented in the remarks (`:21-24`). |
|
||||
| SEC-03 | Done | **Yes** | `Dashboard/DashboardAuthenticationDefaults.cs:45,56` (plain + `__Host-` names); `Dashboard/DashboardServiceCollectionExtensions.cs:119-142`: explicit `CookieName` override wins, else `__Host-MxGatewayDashboard` only when `SecurePolicy==Always` (`RequireHttpsCookie` default true), else plain name. | Guard against applying `__Host-` without Secure is enforced (`:138-141`). Residual (unchanged SEC-23): an *explicit* `CookieName` starting `__Host-` combined with `RequireHttpsCookie=false` is still accepted with no validator diagnosis (`:134-137` applies it unconditionally). |
|
||||
| SEC-04 | Done | **Yes** | `Configuration/GatewayOptionsValidator.cs:314-318` (Production + `DisableLogin` → validation error, aborts startup); env threading via ctor `:23-27` and `Configuration/GatewayConfigurationServiceCollectionExtensions.cs:24` (TryAdd fallback = Development, real host wins). | Guard fires only for the exact `Production` environment name — see **SEC-35**. |
|
||||
| SEC-05 | Done | **Yes** | `Dashboard/HubTokenService.cs:37` (`TokenLifetime = 5 min`, rationale in comment `:30-36`); query-string/no-request-logging contract documented at `Dashboard/HubTokenAuthenticationHandler.cs:13-22`. | Tokens remain irrevocable within the 5-minute window and still travel in `?access_token=` (`HubTokenAuthenticationHandler.cs:69-71`) — the accepted design; jti revocation deferred until per-session ACLs. |
|
||||
| SEC-06 | Done | **Partial** | Production hard-stop verified: `GatewayOptionsValidator.cs:161-165` (`Transport==None` in Production → startup error). Docs: `docs/GatewayConfiguration.md:244-248` (env-var override `MxGateway__Ldap__ServiceAccountPassword` documented). | **The committed dev service-account password is still in the repo at `appsettings.json:29`** (`Ldap.ServiceAccountPassword`) and has not been rotated — the doc itself says it "should be rotated". The transport guard shipped; the credential-removal/rotation half of the remediation did not. → **SEC-36** |
|
||||
| SEC-07 | Done | **Yes** | `Security/Authorization/GatewayGrpcScopeResolver.cs:23` (`QueryActiveAlarmsRequest => GatewayScopes.EventsRead`); both tests now construct the real type (`Tests/Security/Authorization/GatewayGrpcAuthorizationInterceptorTests.cs:330,350`). | |
|
||||
| SEC-08 | Done | **Yes** | `Security/Authentication/CachingApiKeyVerifier.cs` (15 s success-only TTL cache keyed on SHA-256 of the presented token, `:96-120`; only successes cached `:110-117`); `Security/Authentication/CoalescingMarkApiKeyStore.cs:76-113` (≤1 `last_used` write/key/60 s); wired as decorators in `Security/Authentication/AuthStoreServiceCollectionExtensions.cs:89-98`; invalidation on dashboard revoke/rotate/delete at `Dashboard/DashboardApiKeyManagementService.cs:104,144,190`; per-call constraints JSON deserialize removed via blob cache (`Security/Authentication/GatewayApiKeyIdentityMapper.cs:22-45`). Tests exist (`Tests/Security/Authentication/CachingApiKeyVerifierTests.cs`). | New surface reviewed in depth — see SEC-34 (staleness/race, bounded) below. |
|
||||
| SEC-10 | Done | **Yes** | CLI: `--expires` parsed as relative `<N>d`/`<N>h` or absolute ISO-8601 with `AssumeUniversal|AdjustToUniversal` (`Security/Authentication/ApiKeyAdminCommandLineParser.cs:242-274`), threaded into `CreateKeyAsync` (`:85-117`). Dashboard: `DashboardApiKeySummary.cs:13` (`ExpiresUtc`), snapshot projection `Dashboard/DashboardSnapshotService.cs:277`, badge logic compares against `DateTimeOffset.UtcNow` with a 7-day "Expiring" warn window (`Dashboard/Components/Pages/ApiKeysPage.razor:474-497`; `StatusBadge.razor:12-13`). Verifier-side rejection is in the shared `ZB.MOM.WW.Auth.ApiKeys` 0.1.4 (documented `docs/Authentication.md:64-65`; not readable in this repo). | UTC semantics are correct end-to-end on the gateway side. Boundary note: `expiresAt <= now` shows Expired, and relative parse rejects signed values (`NumberStyles.None`). A just-expired key can still authenticate for ≤15 s via the verification cache — see SEC-34. |
|
||||
| SEC-11 | Done | **Yes, with new defects** | Login: fixed-window per-remote-IP limiter policy (`Dashboard/DashboardEndpointRouteBuilderExtensions.cs:27-41`), applied to POST `/auth/login` (`:83`), registered + 429 (`GatewayApplication.cs:111-126`), middleware in pipeline (`GatewayApplication.cs:46`), test `Tests/Gateway/Dashboard/DashboardLoginRateLimitTests.cs`. gRPC: `Security/Authorization/ApiKeyFailureLimiter.cs` checked **before** the store read (`GatewayGrpcAuthorizationInterceptor.cs:72-77`), failure recorded `:89`, reset on success `:97`; interceptor test asserts the short-circuit (`GatewayGrpcAuthorizationInterceptorTests.cs:395-402`). | The limiter exists and is enforced, but its key-id partitioning creates an unauthenticated lockout DoS (**SEC-31**) and its LRU eviction is flushable (**SEC-32**). |
|
||||
| SEC-12 | Done | **Yes** | `Dashboard/DashboardSessionAdminService.cs`: canonical `AuditEvent`s `dashboard-close-session`/`dashboard-kill-worker` (`:37-40`), written on Denied (`:65,147`), Success (`:89-96,171-178`), and every Failure arm (`:105,116,132,187,198,214`), category `SessionAdmin`, actor/remote/correlation captured (`:238-261`), via `IAuditWriter` (same store as API-key events). | Audit-event completeness is good: denied, not-found, faulted, and unexpected paths all emit. |
|
||||
| SEC-20 | Done | **Yes** | `Metrics/GatewayMetrics.cs:374` — `_heartbeatFailuresCounter.Add(1)` with no `session_id` tag (rationale comment `:370-373`). | The in-memory per-session map remains dashboard-only. |
|
||||
| SEC-25 | Done | **Yes** (mitigation, ACL still deferred as documented) | `Dashboard/Hubs/DashboardEventBroadcaster.cs:39,80-92`: when `ShowTagValues=false` (default, `appsettings.json:65`) a deep clone is redacted — top-level `MxEvent.value` (field 5, the only value carrier incl. buffered arrays, `Protos/mxaccess_gateway.proto:704-733`) plus alarm `CurrentValue`/`LimitValue`; source event never mutated (shared with gRPC/replay). `EventsHub.cs:42` keeps the `TODO(per-session-acl)` tied to roadmap item 12. | Redaction is complete for the wire shape at HEAD (verified against the proto: event bodies are discriminators; values ride in field 5 + the alarm body). |
|
||||
| SEC-30 | Done | **Yes** | `docs/Diagnostics.md:126` — explicit "Not yet implemented" callout; wiring deferred until SEC-13 lands. `GatewayLogRedactor` helpers still have no call sites. | |
|
||||
| SEC-09 | Done | **Yes** | `docs/GatewayDashboardDesign.md:424-427` (canonical `Administrator` role string clarified); live `appsettings.json:66-69` uses `Administrator`/`Viewer`. | |
|
||||
| SEC-22 | Done | **Yes** | `docs/Authentication.md` rewritten consumer-side: no `ApiKeyParser`/`SqliteApiKeyStore` references remain (grep-verified); verification flow documents the shared-library pipeline incl. expiry (`:64-65`) and the CLI table (`:219-220`). | |
|
||||
| SEC-13 | Not started | **Confirmed open** | `Diagnostics/GatewayLogRedactor.cs:12-16` still lists only `AuthenticateUser`/`WriteSecured`/`WriteSecured2`; `WriteSecuredBulk`/`WriteSecured2Bulk` exist as command kinds (`GatewayGrpcScopeResolver.cs:44-45`). | Still latent (no call sites). |
|
||||
| SEC-14 | Not started | **Confirmed open** | `GatewayApplication.cs:220-221` maps `MapZbHealth()`/`MapZbMetrics()` with no `RequireAuthorization`. | Leak severity reduced by SEC-20 (no session ids in exported metrics anymore). |
|
||||
| SEC-15 | Not started (accepted) | **Confirmed open** | GET `/logout` skips antiforgery, intentional (`Dashboard/DashboardEndpointRouteBuilderExtensions.cs:92-99`). | Accepted risk; unchanged. |
|
||||
| SEC-16 | Not started | **Confirmed open** | `Program.cs:37-39` maps `command.Pepper` → `MxGateway:ApiKeyPepper`. | |
|
||||
| SEC-17 | Not started | **Confirmed open** | `Security/Authorization/GatewayGrpcAuthorizationInterceptor.cs:22-48` overrides only unary + server-streaming. All current RPCs remain unary/server-streaming. | |
|
||||
| SEC-18 | Not started | **Confirmed open** | `Dashboard/DashboardApiKeyManagementService.cs:22` (`PepperUnavailableMarker = "pepper unavailable"` message match). | |
|
||||
| SEC-19 | Not started | **Confirmed open** | `Security/Audit/SqliteCanonicalAuditStore.cs:22,29` — `CREATE TABLE IF NOT EXISTS audit_event` still issued per write/read. | |
|
||||
| SEC-21 | Not started | **Confirmed open** | `Dashboard/DashboardSnapshotService.cs:142,163,256` — `RefreshApiKeySummariesAsync` still runs every publish tick regardless of audience. | |
|
||||
| SEC-23 | Not started | **Confirmed open** | `GatewayOptionsValidator.ValidateDashboard` (`:307-352`) gained the SEC-04 production guard but still ignores `AutoLoginUser`, `RequireHttpsCookie`, and `CookieName` (no `__Host-`-without-Secure consistency check). | |
|
||||
| SEC-24 | Not started | **Confirmed open** | `Configuration/GatewayConfigurationProvider.cs:58-59` — effective dashboard config still projects only `Enabled`/`AllowAnonymousLocalhost`; no `DisableLogin`/`AutoLoginUser`/`RequireHttpsCookie`/`CookieName`. | |
|
||||
| SEC-26 | Not started | **Confirmed open** | No retention/prune path in `Security/Audit/SqliteCanonicalAuditStore.cs` (grep: no DELETE/retention). | |
|
||||
| SEC-27 | Not started | **Confirmed open** | `Dashboard/DashboardSnapshotService.cs:17,96` — `GatewayStatus` still the hardcoded `"Healthy"` constant. | |
|
||||
|
||||
## New findings
|
||||
|
||||
### SEC-31 · Medium (Security) — API-key lockout DoS: the gRPC failure limiter partitions on an attacker-controlled key id and blocks *before* verification
|
||||
|
||||
**Files:** `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcAuthorizationInterceptor.cs:72-77,115-136`; `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs:65-114`; `src/ZB.MOM.WW.MxGateway.Server/Configuration/SecurityOptions.cs:51-57`.
|
||||
|
||||
`ResolvePeerKey` extracts the key id from the **unauthenticated** presented token (`parts[1]` of `mxgw_<id>_<secret>`, interceptor `:127-132`) and `IsBlocked` short-circuits with `ResourceExhausted` *before* `VerifyAsync` runs (`:72-77`). The success-reset (`:97`) is only reachable after a verification — which a blocked peer never gets. Consequence: any network peer that knows a key id can send 10 garbage-secret requests per 60-second window (defaults, `SecurityOptions.cs:51,57`) and deny that key indefinitely; the legitimate client presenting the *correct* secret is rejected before its credential is ever checked. Key ids are not secrets: they are embedded in every token, displayed to every dashboard Viewer, and pushed to anonymous-loopback hub clients in the snapshot's API-key inventory. This inverts the NAT-caveat rationale (the comment at `:113-114` optimizes for not locking out co-located clients, but hands remote attackers a per-key kill switch). The dashboard login limiter does not have this defect (partitioned on remote IP, `DashboardEndpointRouteBuilderExtensions.cs:31`).
|
||||
|
||||
**Recommendation:** partition on the (transport peer, key id) pair — `context.Peer` is already available — so a remote attacker only locks out *their own* address's attempts against the key; or, if key-id-scoped blocking is retained for NAT reasons, allow a trickle of verifications while blocked (e.g. one probe per few seconds) so a legitimate correct secret can still authenticate and reset the counter.
|
||||
|
||||
### SEC-32 · Low (Security) — Failure-limiter LRU eviction is flushable by spraying unique peer keys, weakening the brute-force bound
|
||||
|
||||
**Files:** `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs:124-148`; `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcAuthorizationInterceptor.cs:127-132`.
|
||||
|
||||
The tracked-peer map evicts the least-recently-active entry once it exceeds `ApiKeyFailureTrackedPeers` (default 4096). Because the partition key is derived from attacker-chosen token text (any `a_b_c`-shaped garbage mints a fresh `key:b` partition; the prefix is not even checked against `mxgw`), a guesser can interleave real guesses with ~4096 unique throwaway tokens to evict its own (or the SEC-31 victim's) `PeerState` and reset the window, resuming guessing past the intended 10-per-minute bound. The flush costs ~4096 requests each cycle, so the limiter still slows the attack materially — hence Low — but the documented guarantee ("a spray of unique peer keys cannot grow memory without limit", which is true, silently trades away the *blocking* guarantee).
|
||||
|
||||
**Recommendation:** never evict an entry that is currently blocked (or whose window still holds failures ≥ limit); prefer evicting entries whose windows have fully expired. Combine with the SEC-31 re-keying so partitions are bound to transport peers, which caps how many partitions one peer can mint.
|
||||
|
||||
### SEC-33 · Low (Security/Stability) — The "rooted for any platform" acceptance re-opens the SEC-01 runtime failure on non-Windows hosts; a fresh stray auth DB exists in the tree (build output)
|
||||
|
||||
**Files:** `src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs:541-560`; `src/ZB.MOM.WW.MxGateway.Server/appsettings.json:17,80`; `src/ZB.MOM.WW.MxGateway.Tests/ProjectStructure/GatewayTreeHygieneTests.cs:30-35`.
|
||||
|
||||
`IsRootedForAnyPlatform` deliberately passes Windows drive-qualified/UNC paths on Unix so the shipped `appsettings.json` validates cross-platform. But the validator runs in the same process that then *uses* the path: on a Unix host, `C:\ProgramData\MxGateway\gateway-auth.db` contains no Unix separators, SQLite treats it as one relative filename, and the credential store lands in the CWD — exactly the original SEC-01 mechanism, now with a validator error message that promises it "never lands in the launch working directory" (`:112`). Concrete evidence at HEAD: `src/ZB.MOM.WW.MxGateway.Tests/bin/Debug/net10.0/C:\ProgramData\MxGateway\gateway-auth.db` dated 2026-07-09 — created *after* the fix landed, by a test/tool run resolving the Windows path relative to the test bin dir. It is gitignored and invisible to the hygiene test (bin/obj excluded), so the guard only protects the tracked tree, not the runtime behavior. Additionally, Galaxy `SnapshotCachePath` (`appsettings.json:80`) is bound by the shared GalaxyRepository package and gets no rooting validation at all.
|
||||
|
||||
**Recommendation:** treat foreign-platform-rooted paths as a *startup error on the host that cannot use them* (i.e. check `Path.IsPathRooted` for the running OS at runtime, keeping the any-platform acceptance only for tooling that validates configs offline), or resolve per-OS defaults in config composition; extend rooting validation to Galaxy `SnapshotCachePath`.
|
||||
|
||||
### SEC-34 · Low (Security) — Verification cache: expired/out-of-band-revoked keys stay valid up to TTL, and an in-flight verification can re-populate the cache after `Invalidate`
|
||||
|
||||
**Files:** `src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/CachingApiKeyVerifier.cs:96-120,123-137`; `src/ZB.MOM.WW.MxGateway.Server/Configuration/SecurityOptions.cs:21`.
|
||||
|
||||
Three bounded staleness windows in the new cache: (1) a key revoked via the **CLI** (separate process — its cache is not the gateway's) keeps authenticating for up to 15 s, which the remarks document; (2) a key whose `ExpiresUtc` passes while its verification is cached also keeps working ≤15 s past expiry — expiry is enforced by the inner library verifier, which a cache hit never reaches, and this case is *not* mentioned in the documented backstop rationale; (3) race: a `VerifyAsync` that passed the inner verifier just before a dashboard revoke can `_cache.Set` *after* `Invalidate(keyId)` ran (`:108-117` vs `:130-136` — no epoch/version check), re-caching the revoked identity for a full TTL despite the "gateway-initiated mutations take effect immediately" contract (`:11-13`). All three are bounded by the 15 s TTL (worst case ~30 s for the race), so exposure is small; the cache-key design itself is sound (SHA-256 of the full token, success-only caching, constant-time compare stays in the library on every miss, no plaintext secrets in memory).
|
||||
|
||||
**Recommendation:** document the expiry-past-TTL window alongside the revocation backstop; for the race, stamp cache entries with a per-key generation counter bumped by `Invalidate` and discard `Set`s from an older generation (or simply accept and document it — at a 15 s TTL the risk is minimal).
|
||||
|
||||
### SEC-35 · Info (Conventions) — Production hard-stops key on the exact `Production` environment name
|
||||
|
||||
**Files:** `src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs:23-27,161-165,314-318`; `src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayConfigurationServiceCollectionExtensions.cs:24,38-51`.
|
||||
|
||||
The `DisableLogin` and plaintext-LDAP guards fire only when `IHostEnvironment.IsProduction()` — i.e. `ASPNETCORE_ENVIRONMENT` unset (defaults to Production, so the NSSM-deployed hosts are covered) or exactly `Production`. A host launched as `Staging`, `Prod`, or any custom name silently disarms both guards while remaining a real deployment. The `FallbackHostEnvironment` (Development) is registered via TryAdd and only wins in bare test/tooling containers, which is correct. Acceptable as designed, but worth stating in `docs/GatewayConfiguration.md`: the guards are environment-name-based, and non-`Production` names carry the permissive dev posture.
|
||||
|
||||
**Recommendation:** document the environment-name contract; optionally treat "not Development" as production-like for these two guards.
|
||||
|
||||
### SEC-36 · Low (Security) — The committed dev LDAP service-account password remains in `appsettings.json` and was not rotated (SEC-06 residual)
|
||||
|
||||
**Files:** `src/ZB.MOM.WW.MxGateway.Server/appsettings.json:29`; `docs/GatewayConfiguration.md:248`.
|
||||
|
||||
The SEC-06 remediation shipped the production transport hard-stop and the env-var override documentation, but the credential itself (`Ldap.ServiceAccountPassword`) is still committed at HEAD and — per the doc's own wording — still awaiting rotation. As long as the shared GLAuth instance honors this credential, the repo discloses a live directory service account. (Not reproducing the value here; see the cited line.)
|
||||
|
||||
**Recommendation:** rotate the GLAuth service-account credential (source of truth `scadaproj/infra/glauth/`), replace the committed value with a placeholder or drop the key entirely (the validator already fails a blank password only when LDAP is enabled — dev can supply it via user-secrets/env), and note the rotation in `glauth.md`.
|
||||
|
||||
## Severity roll-up (new findings)
|
||||
|
||||
| Severity | Count | IDs |
|
||||
|----------|-------|-----|
|
||||
| Medium | 1 | SEC-31 |
|
||||
| Low | 4 | SEC-32, SEC-33, SEC-34, SEC-36 |
|
||||
| Info | 1 | SEC-35 |
|
||||
|
||||
## Still good / preserved
|
||||
|
||||
The load-bearing controls from SEC-28 all survive at HEAD: fail-closed unrecognized-request → `admin` (`GatewayGrpcScopeResolver.cs:29`); opaque auth failures with the stage never disclosed (`GatewayGrpcAuthorizationInterceptor.cs:82-93`, and the new `ResourceExhausted` throttle message reveals nothing about secret validity); constant-time compare stays inside the shared verifier on every cache miss, with only SHA-256 token digests held in memory (`CachingApiKeyVerifier.cs:36-39,166-168`); `SanitizeReturnUrl` still blocks open redirects (`DashboardEndpointRouteBuilderExtensions.cs:237-248`); single generic login-failure message; antiforgery on POST login/logout (`:145,181`); the self-signed PFX generation still hardens permissions before writing key bytes and now uses unique temp names to avoid concurrent-writer collisions (`Security/Tls/SelfSignedCertificateProvider.cs:163-176`); anonymous-loopback hub-token minting is harmless (a principal with no name/id produces a token `Validate` rejects, `HubTokenService.cs:90-93`); no logging call site emitting passwords, secrets, peppers, or credentials was found in the changed files; and the UI-stack rule (local Bootstrap only) still holds. The dashboard Close/Kill service-layer Admin re-checks remain in place beneath the now-correct policy layer (`DashboardSessionAdminService.cs:49-55`), giving genuine defense-in-depth.
|
||||
@@ -0,0 +1,152 @@
|
||||
# Language Clients — Architecture Review (Re-run)
|
||||
|
||||
**Date:** 2026-07-12 · **Commit:** `4f5371f` (main) · **Baseline:** `59856b8` (pre-remediation) · **Method:** static review on macOS (no builds, no tests run)
|
||||
|
||||
Scope: `clients/dotnet` (lib + CLI + tests), `clients/go` (mxgw-go), `clients/java`, `clients/python`, `clients/rust` (lib + CLI + build.rs + vendored protos), cross-client parity, `docs/ClientPackaging.md`, client READMEs, `scripts/pack-clients.ps1`. Generated directories checked for hygiene/freshness only. Prior findings tracked as CLI-01..CLI-34 in `archreview/remediation/50-clients.md` and `archreview/remediation/00-tracking.md`.
|
||||
|
||||
## Prior-finding verification
|
||||
|
||||
| ID | Claimed status | Verified? | Evidence (file:line at HEAD) | Notes |
|
||||
|----|----------------|-----------|------------------------------|-------|
|
||||
| CLI-01 | Done | **Yes** | `clients/go/mxgateway/session.go:1085-1103` (reserve check `len(results) >= eventBufferSize`, cancel + non-blocking terminal send), `:47-56` (`eventBufferSize`/`eventBufferReservedSlots`), `clients/go/mxgateway/errors.go:73` (`ErrSlowConsumer`), test `clients/go/mxgateway/client_session_test.go:150-185` | Sound: sole-producer invariant means the 17th slot is only ever used by the terminal error; doc comments on `Events`/`EventsAfter` updated. Nit: if `Recv` returns a real terminal error while the 16 data slots are full, it is reported as `ErrSlowConsumer` instead (see CLI-44). |
|
||||
| CLI-02 | Done | **Yes (code); docs half missing** | `clients/rust/build.rs:18-26` (repo-path-first, vendored fallback), `clients/rust/Cargo.toml:20` (`include = [... "protos/*.proto" ...]`), `clients/rust/protos/*.proto` byte-identical to `src/ZB.MOM.WW.MxGateway.Contracts/Protos/*` (diff clean), `scripts/pack-clients.ps1:190-212` (`--no-verify` removed from both `cargo package` and `cargo publish`), drift guard `scripts/check-codegen.ps1:68-82` | Core fix verified. But no doc anywhere describes the vendored layout — `clients/rust/README.md:21-22` still says build.rs reads only `../../src/.../Protos`, and `docs/ClientPackaging.md` never mentions `clients/rust/protos/` (CLI-42). |
|
||||
| CLI-03 | Done | **Yes, with a caveat** | `clients/rust/src/error.rs:324-333` (`ensure_mxaccess_success`, `hresult < 0`), `:104-107` (`Error::MxAccess(Box<MxAccessError>)`), wired into every `invoke` at `clients/rust/src/client.rs:296-297` (raw path preserved as `invoke_raw`); tests `error.rs:410-452` | Rust correctly adopted `hresult < 0` from the start. Caveat: the status-array check branches on `success == 0`, contradicting the proto's own "branch on category" instruction (CLI-37). |
|
||||
| CLI-04 | Done | **Yes (library surface, all five)** | .NET `clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxGatewaySession.cs:891` (AdviseSupervisory), `:935` (AddBufferedItem), `:988` (SetBufferedUpdateInterval), `:1033`/`:1076` (Suspend/Activate), `:1128`/`:1200` (WriteSecured/2), `:1276` (AuthenticateUser), `:1332` (ArchestrAUserToId). Go `clients/go/mxgateway/session.go:711-996`. Java `clients/java/.../client/MxGatewaySession.java:720-988`. Python `clients/python/src/zb_mom_ww_mxgateway/session.py:595-813`. Rust `clients/rust/src/session.rs:264-925`. Per-client tests exist (e.g. `session_parity_helpers_test.go:15-199`, `test_typed_command_helpers.py`, `MxGatewayClientSessionTests.java:579-689`, `MxGatewayClientSessionTests.cs:570+`, `client_behavior.rs:572+`) | Phase 1 + Phase 2 both landed in every client. Divergences inside the new surface: HRESULT semantics differ per language (CLI-08 open → CLI-38), redaction strength differs (CLI-40), malformed-reply fallback differs (CLI-41), CLI subcommand coverage and flag names differ (CLI-45, parity note). Parity contract (WriteSecured native failure surfaced unchanged) is explicitly documented and tested in each client. |
|
||||
| CLI-30 | Done | **Yes** | Rust `clients/rust/src/session.rs:145-151` (`unregister`); .NET `MxGatewaySession.cs:857` (`UnregisterAsync`) — Go/Java/Python already had it | Matrix cell closed in all five. |
|
||||
| CLI-12 | Done | **Yes (named targets); residue elsewhere** | `clients/java/README.md`, `clients/java/JavaClientDesign.md`, `docs/ClientPackaging.md:193` all say Java 17 now (grep "Java 21" clean in those files) | Residue: `docs/style-guides/JavaStyleGuide.md:8` still says "Java 21 preferred" — style guides are authoritative per CLAUDE.md (CLI-43). Historical plan docs (`docs/plans/*`, `docs/ImplementationPlanClients.md:312`) also retain "Java 21" but are archival. |
|
||||
| CLI-15 | Done | **Yes (library); CLIs uneven** | .NET `MxEventStreamItem.cs` + `MxEventStreamExtensions.cs` + `MxGatewaySession.cs:1423` (`StreamEventItemsAsync`); Go `session.go:29-60` (`EventResult.ReplayGap`/`IsReplayGap`), `:1040-1042` (sentinel promoted, `Event` cleared); Rust `client.rs` `EventItem::{Event,ReplayGap}` + `from_event`; Python `events.py:11-63` (`ReplayGap` dataclass) + `session.py:816-870` (`stream_events` yields the union, `_surface_replay_gaps` propagates `aclose`); Java `MxEventStreamItem.java` + `MxEventStream.java:127` (`nextItem()`). Tests in all five (e.g. `client_behavior.rs:167-212`, `test_replay_gap.py`, `MxGatewayClientSessionTests.cs:232-270`, `MxGatewayClientSessionTests.java:546-547`, Go `client_session_test.go`). Docs: `docs/ClientLibrariesDesign.md:77-87`, `docs/CrossLanguageSmokeMatrix.md:33-39`, all five READMEs mention the gap | Uniform resume contract (`oldest_available_sequence - 1`) documented everywhere; sentinel never synthesized or swallowed by any library. **However** two CLIs mishandle it: `mxgw-py stream-events` crashes on a gap (CLI-35) and `mxgw-go stream-events` destroys it (CLI-36). Also note Java/.NET keep the typed path opt-in — the pre-existing raw iterators (`MxEventStream.next()`, `StreamEventsAsync`) still deliver the sentinel as a raw event (documented, `MxEventStream.java:28-38`). |
|
||||
| CLI-16 | Done | **Yes** | `docs/ClientPackaging.md:51-52` (`.slnx`), `:160` (`src/zb_mom_ww_mxgateway/generated`), `:187` (`python -m zb_mom_ww_mxgateway_cli`), `:193` (Java 17 + correct package names) | Verified against `clients/python/pyproject.toml` and the actual `.slnx`. |
|
||||
| CLI-18 | Done | **Yes** | `clients/dotnet/ZB.MOM.WW.MxGateway.Client/ZB.MOM.WW.MxGateway.Client.csproj:22` (`<Version>0.1.2</Version>`) | See CLI-39: 0.1.2 is the *already-published* version. |
|
||||
| CLI-21 | Done | **Yes (constant); guard not implemented** | `clients/go/mxgateway/version.go:5-7` (`ClientVersion = "0.1.2"`, comment "keep in sync with the module tag") | The remediation design's second half — a tag-time consistency check in `scripts/tag-go-module.ps1` — was not implemented (grep for `ClientVersion`/`version.go` in the script: no hits). Drift can recur silently. |
|
||||
| CLI-26 | Done | **Yes (literal)** | `clients/python/src/zb_mom_ww_mxgateway/version.py:3` (`__version__ = "0.1.2"`) matches `pyproject.toml:9` | Implemented as a second literal, not the recommended `importlib.metadata` derivation; CLI tests (`test_cli.py:213`) assert self-consistency against `__version__`, not against `pyproject.toml`, so the original drift mode remains possible. |
|
||||
| CLI-29 | Done | **Yes** | `clients/rust/src/version.rs:6-8` (`CLIENT_VERSION: &str = env!("CARGO_PKG_VERSION")`) | Cannot drift by construction — the strongest of the four version fixes. |
|
||||
| CLI-05 | Not started | **Still open** | `MxGatewaySession.cs:19` (`internal MxGatewaySession(`); no `AttachSession` anywhere in the .NET client | .NET remains the only client that cannot re-attach a typed session wrapper — now more consequential because `ReplayGap` handling shipped and reconnect is a first-class flow. |
|
||||
| CLI-06 | Not started | **Still open** | `MxGatewaySession.cs:1435-1439` (`DisposeAsync` → unbounded `CloseAsync()`) | Unchanged. |
|
||||
| CLI-07 | Not started | **Still open** | `MxGatewayClient.cs:311` (`timeout.CancelAfter(Options.DefaultCallTimeout)` still caps the whole pipeline at one attempt's budget) | Unchanged. |
|
||||
| CLI-08 | Not started | **Still open — and now embedded in the new typed surface** | .NET `MxCommandReplyExtensions.cs:34` (`Hresult != 0`), Go `errors.go:187` (`GetHresult() != 0`), Java `MxGatewayErrors.java:50` (`getHresult() != 0`); Python `errors.py:133` and Rust `error.rs:325` use the correct `< 0` | Every new CLI-04 helper in .NET/Go/Java inherits `!= 0`; the same `S_FALSE`-style reply now errors in three clients and passes in two. `docs/ClientLibrariesDesign.md:118-119` incorrectly claims all five run `< 0` (CLI-38). |
|
||||
| CLI-09 | Not started | **Still open** | `clients/go/mxgateway/errors.go` — no `AuthenticationError`/`AuthorizationError`/sentinels (grep clean) | Unchanged. |
|
||||
| CLI-10 | Not started | **Still open** | `clients/go/mxgateway/client.go:64` (`grpc.WithBlock()`), `:68` (`grpc.DialContext`) | Unchanged. |
|
||||
| CLI-11 | Not started | **Still open** | `clients/go/cmd/mxgw-go/main.go` — no `require-certificate-validation` flag (grep clean) | Unchanged; new `write-secured`/`authenticate-user` subcommands also run skip-verify under `--tls` without CA. |
|
||||
| CLI-13 | Not started | **Still open** | `clients/java/.../client/MxGatewayClient.java:248` (`new MxEventStream(16)`), overflow still cancels | Unchanged. |
|
||||
| CLI-14 | Not started | **Still open (partially improved)** | `clients/python/src/zb_mom_ww_mxgateway/options.py:135-154` — TOFU pin + `localhost` SNI default, still no runtime warning; README now documents trust-on-first-use and `--require-certificate-validation` (`clients/python/README.md:382`, `:422`) | The doc half improved; the warning/log half not done. |
|
||||
| CLI-17 | Not started | **Still open** | .NET `MxGatewayClient.cs:367-370` accept-all callback; Go `client.go` `InsecureSkipVerify`; Java `InsecureTrustManagerFactory`; Python TOFU; Rust strict | Divergence unchanged; no warnings added anywhere. |
|
||||
| CLI-19 | Not started | **Still open** | `Properties/AssemblyInfo.cs:3` and `ZB.MOM.WW.MxGateway.Client.csproj:37` both declare `InternalsVisibleTo` | Unchanged. |
|
||||
| CLI-20 | Not started | **Still open** | `MxGatewayClient.cs:367-370` — accept-all callback installed silently | Unchanged. |
|
||||
| CLI-22 | Not started | **Still open** | `clients/go/mxgateway/session.go:1126-1131` (`newCorrelationID` returns `""` on `rand.Read` error) | Unchanged; every new typed helper routes through it. |
|
||||
| CLI-23 | Not started | **Still open** | `session.go:433-441` (`WriteBulk` short-circuits `len==0`) vs `:289-309` (`AddItemBulk` sends empty to the wire) | Both now reject `nil` explicitly, but the empty-slice asymmetry persists. |
|
||||
| CLI-24 | Not started | **Incidentally FIXED** | `clients/java/.../client/MxEventStream.java:25-28` — "Single consumer. This stream is not safe to drain from more than one thread." | Landed as part of the CLI-15 Javadoc work. Tracking should be updated to Done. |
|
||||
| CLI-25 | Not started | **Still open** | `MxGatewayClient.java:347-351` (`close()` calls `shutdown()` without awaiting) | Unchanged. |
|
||||
| CLI-27 | Not started | **Still open** | `clients/python/src/zb_mom_ww_mxgateway/session.py:40-48` (no lock; repeated close synthesizes a local `CloseSessionReply`) | Unchanged. |
|
||||
| CLI-28 | Not started | **Still open** | `session.py:913` (`from .client import GatewayClient # noqa: E402` still at the bottom) | Unchanged. |
|
||||
| CLI-31 | Not started | **Still open — worse** | `clients/rust/crates/mxgw-cli/src/main.rs` is now 2,889 lines (was 2,699) | The three new subcommands grew the monolith. |
|
||||
| CLI-32 | Not started | **Still open** | No bulk cap in `MxGatewaySession.cs` or `MxGatewaySession.java` (grep for 1000/MAX_BULK clean); Go/Python/Rust caps intact (`session.go:19`, `session.py:11`, `session.rs:29`) | Unchanged. |
|
||||
| CLI-33 | Not started | **Still open** | `docs/ClientBehaviorFixtures.md` — no backpressure/slow-consumer section (grep clean) | The Go behavior it would document has changed (CLI-01), making the doc gap slightly more visible; behavior is at least documented in Go doc comments and README. |
|
||||
| CLI-34 | Not started | **Resolved (covered)** | Root `.gitignore:87` (`.pytest_cache/`), `:103` (`**/build/`); `git check-ignore` confirms both `clients/python/build` and `clients/python/.pytest_cache` are ignored | No accidental-commit risk. Tracking can be closed. |
|
||||
|
||||
**Done claims that don't fully hold:** none of the Done claims is false at the library level, but three carry material caveats: (1) **CLI-15** — the Python and Go *CLIs* were not updated for the typed gap and now misbehave on it (new findings CLI-35/CLI-36); (2) **CLI-02** — the documentation half (vendored-proto layout in `clients/rust/README.md` / `docs/ClientPackaging.md`) was skipped; (3) the tracking change-log entry for CLI-04 ("each runs `hresult < 0` + `MxStatusProxy` validation") and `docs/ClientLibrariesDesign.md:118-119` overstate — .NET/Go/Java still validate `!= 0` (CLI-08 is open, as tracked). CLI-21 and CLI-26 are fixed as constants but without the drift guards their designs specified.
|
||||
|
||||
## New findings
|
||||
|
||||
### CLI-35 — Python CLI `stream-events` crashes on a ReplayGap
|
||||
**Severity:** Medium · **Category:** correctness / CLI
|
||||
**Files:** `clients/python/src/zb_mom_ww_mxgateway_cli/commands.py:1098-1106` (`_stream_events`), `:1627-1632` (`_message_dict`); `clients/python/src/zb_mom_ww_mxgateway/session.py:816-870`
|
||||
`Session.stream_events()` now yields `pb.MxEvent | ReplayGap` (the CLI-15 change), but the CLI's `_stream_events` feeds every collected item into `_message_dict`, which calls `google.protobuf.json_format.MessageToDict`. `ReplayGap` is a plain dataclass (`events.py:11`), not a proto message, so `MessageToDict` raises and the command exits with an error after having consumed the stream — precisely on the resume-after-outage path where an operator most needs the tool. No test covers it (`grep -i replay clients/python/tests/test_cli.py` → nothing).
|
||||
**Recommendation:** branch on `isinstance(item, ReplayGap)` in `_stream_events` and emit a distinct JSON object (mirror the Rust CLI's `{"replayGap": {...}}` row, `mxgw-cli/src/main.rs:1020-1035`); add a CLI test that scripts a gap.
|
||||
|
||||
### CLI-36 — Go CLI `stream-events` silently destroys the ReplayGap signal
|
||||
**Severity:** Medium · **Category:** correctness / CLI
|
||||
**Files:** `clients/go/cmd/mxgw-go/main.go:969-983`; `clients/go/mxgateway/session.go:1040-1042`
|
||||
The library deliberately clears `EventResult.Event` on a gap (`session.go:1041`, "so consumers never process the sentinel as a data change"), but the CLI loop only checks `result.Err` and then formats `result.Event`: JSON mode marshals a nil `*MxEvent` (empty object), text mode prints `0 MX_EVENT_FAMILY_UNSPECIFIED`. The gap's cursors (`requested_after_sequence`, `oldest_available_sequence`) — the exact data an operator needs to resume — are discarded. Contrast: the Rust CLI prints a dedicated `REPLAY_GAP` row; the .NET and Java CLIs pass the raw sentinel through proto-JSON so `replayGap` is at least visible.
|
||||
**Recommendation:** add an `if result.IsReplayGap()` branch printing the two sequences (text) / a `replayGap` JSON object; add a CLI test.
|
||||
|
||||
### CLI-37 — Status-array validation branches on `success`, contradicting the contract's own "branch on category" rule; .NET is the lone divergent (and lone conformant) client
|
||||
**Severity:** Medium · **Category:** correctness / cross-client parity
|
||||
**Files:** `src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_gateway.proto:982-992`; Rust `clients/rust/src/error.rs:326` (`status.success == 0`); Go `clients/go/mxgateway/status.go:4-6` (`GetSuccess() != 0`); Java `MxStatuses.java:26`; Python `errors.py:141`; .NET `MxStatusProxyExtensions.cs:11-17` (`Success != 0 && Category is MxStatusCategory.Ok`)
|
||||
The proto comment (pre-dating remediation) states `success` "is NOT a boolean … carried verbatim for diagnostics; the authoritative success/failure of the operation is `category` (MX_STATUS_CATEGORY_OK marks success) … Clients should branch on `category`, not on a specific `success` value." Four clients — including the brand-new Rust `ensure_mxaccess_success` written for CLI-03 — branch only on `success`; .NET requires both `success != 0` *and* `category == Ok`. A reply with `success = 1, category = COMMUNICATION_ERROR` passes Go/Java/Python/Rust and throws in .NET; a reply with `success = 0, category = OK` does the reverse in .NET. Either the proto comment is wrong (then fix the comment and .NET) or the four clients are (then fix them) — today the wire contract and four of five implementations disagree.
|
||||
**Recommendation:** decide the authoritative field once (the proto says `category`), align all five `IsSuccess`-equivalents, and add a shared behavior-fixture case (`success` and `category` disagreeing) so the suites lock it in. Coordinate with CLI-08 since it is the same validation function in every client.
|
||||
|
||||
### CLI-38 — Shared design doc claims `hresult < 0` validation that three clients don't perform
|
||||
**Severity:** Medium · **Category:** documentation drift (CLAUDE.md same-commit rule)
|
||||
**Files:** `docs/ClientLibrariesDesign.md:118-119` ("runs the same MXAccess-level reply validation (HRESULT `< 0` + per-item `MxStatusProxy`)"); actual: `MxCommandReplyExtensions.cs:34`, `errors.go:187`, `MxGatewayErrors.java:50` all `!= 0`
|
||||
The Typed Command Parity section (added with CLI-04) documents COM-correct `< 0` semantics as if uniform, but only Python and Rust implement them; the 00-tracking change log (2026-07-09 Wave E2 entry) repeats the same claim. Until CLI-08 lands, the load-bearing design doc describes behavior that .NET/Go/Java do not have, and a positive success HRESULT (e.g. `S_FALSE`) on any *new* helper (WriteSecured, AuthenticateUser, …) errors in three languages and succeeds in two.
|
||||
**Recommendation:** either land CLI-08 (three one-line changes + tests, already designed) or correct the doc to state the current divergence. Landing CLI-08 is strictly better and closes CLI-38 for free.
|
||||
|
||||
### CLI-39 — Version constants aligned to the already-published 0.1.2 despite breaking API changes since that publish
|
||||
**Severity:** Medium · **Category:** packaging / release hygiene
|
||||
**Files:** `clients/rust/Cargo.toml:3` (0.1.2), `clients/python/pyproject.toml:9` + `version.py:3` (0.1.2), `clients/go/mxgateway/version.go:7` (0.1.2), `clients/dotnet/.../ZB.MOM.WW.MxGateway.Client.csproj:22` (0.1.2)
|
||||
0.1.2 was published to Gitea before this remediation (per `scripts/pack-clients.ps1` history). Since then the public API changed incompatibly in at least two clients: Rust's `EventStream` item type changed from `Result<MxEvent, Error>` to `Result<EventItem, Error>` (`clients/rust/src/client.rs:139-141`) and `Error` gained a variant; Python's `stream_events` now yields `MxEvent | ReplayGap` (`session.py:816-820`), breaking consumers that assume proto messages. The version-alignment fixes (CLI-18/21/26) set the source to the *old* released number instead of bumping. Consequence: the next `cargo publish` / `pip` publish of the current tree either collides with the existing 0.1.2 artifact (registries reject duplicate versions) or, if forced, ships a different API under an identical version string.
|
||||
**Recommendation:** bump all five clients to 0.1.3 (or 0.2.0 for Rust/Python given the breaking stream-surface change; Java already sits at 0.2.0) before the next `pack-clients.ps1` run, and add the version-vs-registry check to the pack script.
|
||||
|
||||
### CLI-40 — Credential-redaction seam is uniform in name only; Rust/Java/.NET cannot scrub an echoed password
|
||||
**Severity:** Low · **Category:** security / cross-client parity
|
||||
**Files:** Go `clients/go/mxgateway/errors.go:16-66` + `session.go:745`, `:836` (exact-secret `redactSecrets` on WriteSecured value + AuthenticateUser password); Python `session.py:573-592` + `:875-909` (exact-secret `_invoke_redacted`); Rust `clients/rust/src/error.rs:363-375` (`redact_credentials` scrubs only whitespace-delimited tokens starting `mxgw_` or equal to `bearer`); Java `MxGatewaySecrets.java:44-57` (same pattern-only scrub); .NET — no library-level scrub at all (exception text embeds `status.DiagnosticText` verbatim via `MxStatusProxyExtensions.ToDiagnosticSummary`; only the CLI redacts, `MxGatewayCliSecretRedactor.cs:14-32`)
|
||||
The design doc (`docs/ClientLibrariesDesign.md:123-127`) and the Rust helper docs (`session.rs:876-883`: diagnostics "are scrubbed by the client's credential-redaction seam") claim credentials can never reach exception text. That holds by *construction* everywhere (exceptions carry reply-derived text, never the request), but only Go and Python defend against the gateway/MXAccess echoing the credential back in `diagnostic_text`; Rust/Java scrub only API-key-shaped tokens and .NET scrubs nothing in the library. The per-client tests pass because the fakes don't echo.
|
||||
**Recommendation:** either downgrade the doc claim to "requests are never embedded in errors" or port Go's exact-secret scrub to Rust (`MxAccessError::Display` already funnels through `redact_credentials` — extend it to take known secrets), Java, and .NET. Add one shared fixture where the scripted reply's `diagnostic_text` contains the credential.
|
||||
|
||||
### CLI-41 — Malformed-reply fallback for `AuthenticateUser`/`ArchestrAUserToId`/`AddBufferedItem` differs per language (silent 0 vs typed error vs NRE)
|
||||
**Severity:** Low · **Category:** cross-client parity / robustness
|
||||
**Files:** Go `session.go:807-819` (falls back to `GetReturnValue().GetInt32Value()` → silent `0` when both absent); Python `session.py:707` (`reply.authenticate_user.user_id` → proto3 default `0` when payload absent, no fallback, no error); Java `MxGatewaySession.java:868-880` (payload else `getReturnValue().getInt32Value()` → `0`); .NET `MxGatewaySession.cs:1289` (`reply.AuthenticateUser?.UserId ?? reply.ReturnValue.Int32Value` — `NullReferenceException` when both are absent, since `ReturnValue` is a nullable message); Rust `session.rs` helpers `authenticate_user_id`/`archestra_user_id` return `Error::MalformedReply` (`session.rs:1073-1089`), while `add_buffered_item_handle` *does* fall back to `return_value` (`:1057-1071`) — inconsistent even within Rust
|
||||
A gateway reply that omits the typed payload yields: user id `0` (Go/Java, and Python even when `return_value` is present), an NRE (.NET), or a typed error (Rust). Getting `0` back from `AuthenticateUser` is the worst mode — callers feed it into `WriteSecured` as a real user id.
|
||||
**Recommendation:** pick one contract (Rust's `MalformedReply`-style error is the safest) and apply it to all five; at minimum make Python check `HasField`/`WhichOneof` and .NET null-check `ReturnValue`.
|
||||
|
||||
### CLI-42 — CLI-02's documentation half missing: vendored Rust protos undescribed anywhere
|
||||
**Severity:** Low · **Category:** documentation drift
|
||||
**Files:** `clients/rust/README.md:21-22` (still: "build.rs reads the `.proto` files from `../../src/ZB.MOM.WW.MxGateway.Contracts/Protos`" — no mention of the `clients/rust/protos/` fallback); `docs/ClientPackaging.md` (no "vendor"/`protos/` mention; grep `-i vendored docs/*.md` → zero hits repo-wide)
|
||||
The vendored copies are load-bearing published build inputs with a refresh obligation on every `.proto` change (enforced only by `scripts/check-codegen.ps1` Check 3 and a `build.rs` comment). CLAUDE.md's docs-in-same-commit rule was not met for CLI-02.
|
||||
**Recommendation:** one paragraph in the Rust README's generation section + the Rust section of `docs/ClientPackaging.md` describing the repo-path-first/vendored-fallback resolution and the refresh rule.
|
||||
|
||||
### CLI-43 — Java style guide still prescribes "Java 21 preferred" after the 17 retarget
|
||||
**Severity:** Low · **Category:** documentation drift
|
||||
**Files:** `docs/style-guides/JavaStyleGuide.md:8`
|
||||
CLAUDE.md declares `docs/style-guides/` authoritative; the guide contradicts the shipped `toolchain 17` / `options.release = 17` build and the corrected CLI-12 docs. (Historical plan docs `docs/plans/2026-05-28-*.md:9`, `docs/plans/2026-06-16-*.md:9`, `docs/ImplementationPlanClients.md:312` also say 21, but are archival.)
|
||||
**Recommendation:** update the style guide line to Java 17 (Ignition 8.3 target), optionally noting 21+ compatibility.
|
||||
|
||||
### CLI-44 — Go event goroutine can mislabel a genuine terminal stream error as `ErrSlowConsumer`
|
||||
**Severity:** Low · **Category:** correctness (edge case)
|
||||
**Files:** `clients/go/mxgateway/session.go:1051-1057` (Recv-error path reuses `sendEventResult`) + `:1085-1100` (overflow branch)
|
||||
When `stream.Recv()` returns a real terminal error while the 16 data slots are full, `sendEventResult` hits the overflow branch first and enqueues `ErrSlowConsumer` instead of the actual `GatewayError{Err: err}` — the consumer gets a loud terminal error (good) with the wrong root cause (mildly misleading; the real gRPC status is lost). Not silent loss, so Low.
|
||||
**Recommendation:** in the Recv-error path, bypass the overflow substitution (e.g. a `terminal bool` parameter that uses the reserved slot directly with the caller's error).
|
||||
|
||||
### CLI-45 — New CLI credential flags diverge: env-var names, empty-password handling, and missing-password behavior differ per language
|
||||
**Severity:** Low · **Category:** cross-client parity / CLI usability
|
||||
**Files:** Go `cmd/mxgw-go/main.go:441-443` (default env `MXGATEWAY_VERIFY_PASSWORD`; empty resolved password sent to the wire); Java `MxGatewayCli.java:1140-1160` (same env name; `resolvedPassword = ""` fallback sent as empty credential); Rust `mxgw-cli/src/main.rs:143-152` (same env name; missing → `Error::InvalidArgument`); Python `commands.py:833-848` (no default env name — must pass `--password-env`; missing → `click.UsageError`); .NET `MxGatewayClientCli.cs:355-388` (different env name `MXGATEWAY_VERIFY_USER_PASSWORD` and flag `--verify-user-password`; missing → `ArgumentException`)
|
||||
The same operator workflow (`export MXGATEWAY_VERIFY_PASSWORD=… && <cli> authenticate-user …`) works in Go/Java/Rust, needs an explicit flag in Python, and needs a *different* variable in .NET; Go and Java silently authenticate with an empty password where the other three fail fast. Subcommand coverage also diverges: .NET CLI exposes all nine new commands (unregister/buffered/suspend/activate/write-secured2/archestra-user-to-id), Rust adds unregister + the two credential commands, Go/Python/Java add only `write-secured` + `authenticate-user`.
|
||||
**Recommendation:** standardize on one env-var name (and add it as an alias where it differs), fail fast on empty passwords in Go/Java, and either level up the CLI subcommand sets or note the deltas in `docs/CrossLanguageSmokeMatrix.md`.
|
||||
|
||||
## Severity roll-up (new findings)
|
||||
|
||||
| Severity | Count | IDs |
|
||||
|---|---|---|
|
||||
| High | 0 | — |
|
||||
| Medium | 5 | CLI-35, CLI-36, CLI-37, CLI-38, CLI-39 |
|
||||
| Low | 6 | CLI-40, CLI-41, CLI-42, CLI-43, CLI-44, CLI-45 |
|
||||
|
||||
## Cross-client parity note (updated)
|
||||
|
||||
The two headline parity gaps from the prior review are closed at the library level: **typed single-item command parity** (CLI-04, incl. the full buffered/suspend/activate family and `Unregister` everywhere) and the **typed `ReplayGap` signal** (CLI-15) now exist in all five clients with tests, and the Go overflow path is loud (CLI-01). What remains divergent:
|
||||
|
||||
| Behavior | .NET | Go | Java | Python | Rust |
|
||||
|---|---|---|---|---|---|
|
||||
| HRESULT failure test (CLI-08) | `!= 0` | `!= 0` | `!= 0` | `< 0` ✓ | `< 0` ✓ |
|
||||
| Status-entry failure test (CLI-37) | `success==0` **or** `category != Ok` | `success==0` | `success==0` | `success==0` | `success==0` |
|
||||
| ReplayGap in default stream surface | opt-in (`StreamEventItemsAsync`) | typed by default (`EventResult`) | opt-in (`nextItem()`) | typed by default | typed by default (`EventItem`) |
|
||||
| CLI renders ReplayGap | raw proto JSON (visible) | **lost** (CLI-36) | raw proto JSON (visible) | **crash** (CLI-35) | dedicated row ✓ |
|
||||
| Credential scrub of surfaced errors (CLI-40) | none (lib) / exact (CLI) | exact-secret | `mxgw_`/`bearer` pattern | exact-secret | `mxgw_`/`bearer` pattern |
|
||||
| AuthenticateUser malformed-reply result (CLI-41) | NRE possible | silent 0 | silent 0 | silent 0 | typed error ✓ |
|
||||
| Session re-attach to existing id (CLI-05) | **No** | Yes | Yes | Yes | Yes |
|
||||
| Client-side 1000-item bulk cap (CLI-32) | No | Yes | No | Yes | Yes |
|
||||
| Event backpressure (CLI-01/13/33) | unbuffered | 16+1, loud `ErrSlowConsumer` ✓ | 16, cancel+error | unbuffered | unbuffered |
|
||||
| TLS default (CLI-17) | accept-all | accept-all | accept-all | TOFU pin | strict pin |
|
||||
| Typed auth errors (CLI-09) | Yes | **No** | Yes | Yes | Yes |
|
||||
| Version constant vs published artifact (CLI-39) | 0.1.2 = already-shipped | 0.1.2 = already-shipped | 0.2.0 | 0.1.2 = already-shipped | 0.1.2 = already-shipped |
|
||||
| Credential CLI env var (CLI-45) | `MXGATEWAY_VERIFY_USER_PASSWORD` | `MXGATEWAY_VERIFY_PASSWORD` | `MXGATEWAY_VERIFY_PASSWORD` | none (explicit flag) | `MXGATEWAY_VERIFY_PASSWORD` |
|
||||
|
||||
The resume contract itself (`after_worker_sequence = oldest_available_sequence - 1`) is uniform and consistently documented in all five clients, both proto comments, the design doc, and the smoke matrix — that part of the epic is genuinely converged.
|
||||
|
||||
## Still good / preserved
|
||||
|
||||
- The remediation work is high quality where it landed: the Go reserved-slot design is provably safe (single producer), the Rust vendored-proto fallback keeps in-repo edits live while making the tarball self-sufficient, `cargo package` verification is restored, and `check-codegen.ps1` Check 3 guards vendored-proto drift.
|
||||
- MXAccess parity discipline held under pressure: every new secured/auth helper documents and tests that native failures (WriteSecured before authenticate/advise, failed authentication) surface unchanged; no client synthesizes or swallows events, and the ReplayGap sentinel is forwarded, never invented (`EventItem::from_event`, `_surface_replay_gaps`, `MxEventStreamItem`, Go's cleared-`Event` conversion all preserve this).
|
||||
- Generated-code hygiene remains clean: vendored Rust protos byte-identical to Contracts; the Java `MxaccessWorker.java` regen matches a real `mxaccess_worker.proto` change (+6 lines) rather than spurious churn; the tracked Java protoset descriptor was refreshed in step; a `checkGeneratedClean` Gradle guard (build.gradle:68+) now catches stale Java codegen; Go/Python `generate-proto.ps1` scripts are now PATH-portable with pinned-version assertions instead of hard-coded per-machine paths.
|
||||
- The non-doc .NET library diff outside the new features is pure XML-doc completion (verified: no behavioral lines).
|
||||
- Test investment matches the surface: ~1,500 new test lines across the five suites, including per-client credential-absence assertions, replay-gap fixtures, the Go slow-consumer test, and Rust ensure_mxaccess_success unit tests.
|
||||
@@ -0,0 +1,84 @@
|
||||
# Testing, Documentation & Underdeveloped Areas — Re-Review
|
||||
|
||||
- **Date:** 2026-07-12
|
||||
- **Commit:** `4f5371f` (`main`, "Merge branch 'fix/archreview-p2' into main (P2 tier: completeness & polish)")
|
||||
- **Baseline:** pre-remediation `59856b8`; prior report `archreview/60-testing-docs-gaps.md` (2026-07-08)
|
||||
- **Method:** macOS static review — reading only, no builds, no test runs. All paths relative to `/Users/dohertj2/Desktop/MxAccessGateway`.
|
||||
|
||||
## Prior-finding verification
|
||||
|
||||
| ID | Claimed status | Verified? | Evidence (file:line) | Notes |
|
||||
|----|----------------|-----------|----------------------|-------|
|
||||
| TST-01 | Done | **Yes** | `src/ZB.MOM.WW.MxGateway.Tests/Gateway/GatewayEndToEndReconnectReplayTests.cs:52` (no-gap resume), `:151` (stale-cursor `ReplayGap`) | Real e2e: drives the actual gRPC `StreamEvents` path via `FakeWorkerHarness`, gated deterministic event emission, full detach (cancel + await so the lease disposes), reconnect with `AfterWorkerSequence`, exact-sequence/order/no-dup assertions plus sentinel shape (`RequestedAfterSequence`/`OldestAvailableSequence`, empty body, family Unspecified) at `:214-231`. Client consumers exist in all five clients (e.g. `clients/dotnet/.../MxEventStreamItem.cs`, `clients/python/tests/test_replay_gap.py`, Java `MxEventStreamItem.java`). Not tautological — it exercises the replay ring across a detach, not a mock of it. |
|
||||
| TST-02 | Done | **Yes** | `src/ZB.MOM.WW.MxGateway.Server/Grpc/EventStreamService.cs:57-62` (ordinal `OwnerKeyId` vs caller-key check → `PermissionDenied` before attach); owner recorded at `Sessions/GatewaySession.cs:197,242` | Tested both ways: `Tests/Gateway/Grpc/EventStreamServiceTests.cs:50-57` (owner match streams) and `:73-89` (mismatch → `PermissionDenied`, asserted **before** any event yields). Documented: `docs/Sessions.md:220`, CLAUDE.md:124. |
|
||||
| TST-03 | Done | **Yes, with a large residual gap** | `.gitea/workflows/ci.yml:17-121` (`portable` + `java` jobs); codegen guard wired at `:60-62` → `scripts/check-codegen.ps1` | CI exists and per the tracking log ran green on main. But the `windows` and `live-mxaccess` jobs were **removed** (`ci.yml:123-131`, commit `abb0930`) because act_runner host-mode on Windows is broken — see new finding TST-25. Codegen freshness is genuinely wired (descriptor set, `Contracts/Generated` diff, vendored Rust protos — `check-codegen.ps1:4-16`). |
|
||||
| TST-04 | Done | **Yes** | `oldtasks.md:27-34` (Phase 4 scoped as TST-15, Phase 5 "DEFERRED, not planned"), `:65-93` (per-phase status incl. "`EnableOrphanReattach` (Task 26) does not exist — do not reference"); CLAUDE.md:79 reconnect paragraph states clients consume `ReplayGap` and reattach is deferred | Governance decision recorded in the tracking source and its mirror; the CLAUDE.md invariant ("gateway restart does not reattach") stands. |
|
||||
| TST-08 | Done | **Yes** | CLAUDE.md:91 — filtered-run guidance reframed as speed-only: "the suite exits cleanly (verified on macOS and the Windows dev box — 0 surviving `testhost`/worker processes)" | De-staled as claimed; tracking log records the windev 3×-run evidence and the separately-fixed Windows temp-file-lock flakes (commit `11a716a`). |
|
||||
| TST-11 | Done | **Yes, with caveat** | `src/Directory.Build.props:19` (`<Version>0.1.2</Version>`), `:24-33` (git short SHA → `SourceRevisionId`/InformationalVersion, guarded for git-less builds); Contracts `0.1.2` (`Contracts.csproj:10`) | .NET side single-sourced and SHA-stamped; Python/Rust/Go/.NET client constants aligned to 0.1.2 (CLI-18/21/26/29 per tracking). Residual: Java remains `0.2.0` (`clients/java/build.gradle:16`) — convergence explicitly deferred as a release decision, so the drift is now *documented*, not accidental. |
|
||||
| TST-12 | Done | **Yes** | CLAUDE.md:79 — "detach-grace and replay retention are **on** by default (`DetachGraceSeconds=30`, `ReplayBufferCapacity=1024`, `ReplayRetentionSeconds=300`)" | Matches code: `Configuration/SessionOptions.cs:46`, `Configuration/EventOptions.cs:21,29`. |
|
||||
| TST-13 | Done | **Yes** | `gateway.md:305-312` — envelope sketch replaced with proto-as-source-of-truth pointer ("do not hand-copy… an inlined copy drifts"), correct `string correlation_id`; `gateway.md:776` — "one active client event subscriber per session **by default**" | All three stale spots corrected. |
|
||||
| TST-23 | Done | **Yes** | `gateway.md:365-372` — "### Future work: bidirectional `Session` (not implemented)" | "Best long-term shape" framing gone. |
|
||||
| TST-05 | Not started | **Confirmed open — now worse** | `WorkerLiveMxAccessSmokeTests.cs` still 8 `[LiveMxAccessFact]`s, opt-in only; `ci.yml` contains no `MXGATEWAY_RUN_LIVE_MXACCESS_TESTS` anywhere | The planned scheduled live job was removed with the Windows jobs (`ci.yml:123-131`), so live coverage regressed from "planned nightly" back to "run by memory". Folded into new finding TST-25. |
|
||||
| TST-06 | Not started | **Confirmed open** | `src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardLiveDataService.cs` exists; no `DashboardLiveDataServiceTests` in `Tests/Gateway/Dashboard/` (directory listing has 18 classes, none for it) | `EventsHub`/`AlarmsHubPublisher` hub methods likewise still untested (new `DashboardEventBroadcasterTests` covers the mirror redaction, not the live-data session lifecycle). |
|
||||
| TST-07 | Not started | **Confirmed open (lines moved)** | `Tests/Gateway/Workers/WorkerClientTests.cs:545` (`Task.Delay(150ms)` then negative assertion); `Tests/Gateway/Sessions/SessionManagerTests.cs:402` (`Task.Delay(50)` race) | Same latent flakes; now more relevant because they run on shared CI runners on every push. |
|
||||
| TST-09 | Not started | **Confirmed open** | `Server/GatewayApplication.cs:75-76` — still exactly one `AuthStoreHealthCheck` | No Galaxy SQL / LDAP / worker-executable / alarm-monitor checks. |
|
||||
| TST-10 | Not started | **Confirmed open** | No `docs/Deployment.md` (`ls docs/` — no deploy doc); no deploy script under `scripts/` | Deploy knowledge still lives only in operator memory notes. |
|
||||
| TST-14 | Not started | **Confirmed open** | Repo root still holds `MxAccessGateway-docs-{issues,fixed,final}.md`, `MxGatewayClient-docs-{issues,fixed}.md`, `oldtasks.md`, `stillpending.md` (root `ls`) | `oldtasks.md` did gain a legitimate new role as the epic-governance record (TST-04), so "delete when epic closes" needs re-deciding; the five untracked `*-docs-*.md` files remain pure clutter. |
|
||||
| TST-15 | Not started (design done) | **Confirmed open** | `Server/Dashboard/Hubs/EventsHub.cs:42` — `TODO(per-session-acl)` still present | Design now exists: `docs/plans/2026-07-10-dashboard-session-acl-tst15.md`. Mitigation shipped meanwhile: SEC-25 value redaction in the hub mirror (see TST-27). |
|
||||
| TST-16 | Not started | **Partially resolved — see TST-27** | `Server/Dashboard/Hubs/DashboardEventBroadcaster.cs:16,29` — `ShowTagValues` now gates value redaction of the SignalR mirror (SEC-25) | The flag is no longer fully dead, but it still does not gate `/browse` live-value display, and the config doc still calls it "Reserved" — fresh drift (TST-27). |
|
||||
| TST-17 | Not started | **Confirmed open** | `Worker/MxAccess/WnWrapAlarmConsumer.cs:~264-275` — `_ = ackOperatorDomain; _ = ackOperatorFullName;` then 6-arg call; no diagnostic surfaced in the ack reply | Silent drop unchanged (comment block explains the −55 stub, but callers still can't see the degrade). |
|
||||
| TST-18 | Not started | **Confirmed open** | No `*HostedService*` test files under `Tests/Gateway/Sessions/` or `Tests/Gateway/Workers/` | Sweep cores remain covered indirectly (`SessionManagerTests.cs:985,1015` now also cover faulted-reap). |
|
||||
| TST-19 | Not started | **Confirmed open** | `Tests/Gateway/Workers/Fakes/FakeWorkerHarness.cs` — no lockstep note / `WorkerPipeSession` pointer (grep empty); no README in `Fakes/` | Risk is higher now that the real-worker suite has no scheduled run (TST-25). |
|
||||
| TST-20 | Not started | **Confirmed open** | `scripts/run-client-e2e-tests.ps1:46-48` — `DrainEveryTags = 15` workaround intact | Advise-without-consumer sharp edge still un-addressed as product feedback. |
|
||||
| TST-21 | Not started | **Confirmed open** | `Server/appsettings.json:10` — daily rolling file sink, relative path, no size cap / retained-count | Unchanged. |
|
||||
| TST-22 | Not started | **Confirmed open** | `docs/GatewayConfiguration.md` shape block (~:12-95) still lacks `DisableLogin`/`AutoLoginUser`/`CookieName`/`Tls`/`Alarms:Fallback` (grep of the block returns none; tables at `:181-188,:313+` document them) | The block did gain new keys (`FaultedGraceSeconds:41`, `ShowTagValues:60`) so it is maintained — the omissions are just still omitted. |
|
||||
| TST-24 | Not started (deferred, CI-gated) | **Confirmed open** | Only Java has an in-process gateway harness (`clients/java/.../cli/InProcessGatewayHarness.java`); other four clients still unit-test against mocks (e.g. `clients/python/tests/test_replay_gap.py`) | Deferral on TST-03 no longer blocks it — CI is live. |
|
||||
|
||||
**Done claims that failed verification: none.** All nine Done claims hold with file:line evidence. Two carry material caveats: TST-03 (CI exists but the Windows/live half of the matrix was subsequently removed) and TST-11 (Java version convergence deferred).
|
||||
|
||||
## New findings
|
||||
|
||||
### TST-25 — The entire Windows-only test surface is now unguarded by any automation — **High** · CI/coverage
|
||||
|
||||
- **Files:** `.gitea/workflows/ci.yml:123-131` (removal note, commit `abb0930`); `src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWritePriority.cs` (new); `src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerFrameProtocolTests.cs:353-395` (priority-ordering tests); `src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs` (8 live facts)
|
||||
- **Description:** The `windows` and `live-mxaccess` CI jobs were removed because Gitea act_runner v0.6.1 host-mode is broken on Windows and a gated job with no runner wedges every run in "queued". The consequence is that the x86/net48 Worker build, all of `Worker.Tests`, and the live-MXAccess suite have **zero** automated execution — and this is not the pre-remediation status quo, because the remediation itself added new Windows-only mechanisms that are now tested only by hand: the `WorkerFrameWritePriority` control-vs-event write scheduling (WRK-01/WRK-07 machinery — its ordering tests exist and are meaningful, but they compile only on Windows), the worker-side `max_frame_bytes` adoption, the WnWrap/subtag alarm consumers, and STA runtime changes (`StaRuntime.cs` modified since baseline). The prior review's TST-05 concern ("live paths verified by memory") now also applies to a whole tier of *unit* tests. The portable job's `check-codegen.ps1` Generated-diff check does substitute for the net48 CS0246 guard (good), but nothing substitutes for running the worker tests.
|
||||
- **Recommendation:** Restore Windows coverage by a route that doesn't depend on act_runner: a scheduled CI step (or standalone cron on the Mac/windev) that drives the existing windev worktree process over SSH — `dotnet build -p:Platform=x86` + `dotnet test Worker.Tests` — and reports failures loudly; the remote-PS `-EncodedCommand` recipe already exists in operator memory. Until then, record a mandatory manual cadence in `docs/GatewayTesting.md` (per-merge for worker-touching changes, weekly otherwise) instead of leaving it "when someone remembers".
|
||||
|
||||
### TST-26 — docs/GatewayTesting.md and check-codegen.ps1 describe CI jobs that no longer exist — **Medium** · documentation currency
|
||||
|
||||
- **Files:** `docs/GatewayTesting.md:426-431`; `scripts/check-codegen.ps1:17`; `.gitea/workflows/ci.yml:12-14`
|
||||
- **Description:** `docs/GatewayTesting.md` still documents the **`windows`** job ("the only host that builds the x86/net48 Worker… the definitive guard for the 'regenerate and commit Generated/' rule") and the **`live-mxaccess`** scheduled job — both deleted from `ci.yml` in `abb0930`, which touched only the workflow file. This violates the repo's own docs-change-in-same-commit rule and actively misleads: a reader concludes the worker build is CI-guarded when it is not, and the "definitive guard" claim is doubly wrong (the surviving guard for the Generated-commit rule is the portable job's diff check, not a Windows compile). Same stale claim inside `check-codegen.ps1:17` ("guarded by the Windows CI job, not here") and in `ci.yml:12-14`, whose nightly cron comment still says "Nightly live-MXAccess smoke (windev)" though the schedule now only re-runs `portable`+`java`.
|
||||
- **Recommendation:** One small commit: rewrite the GatewayTesting.md CI section to the two-job reality plus the manual windev process (and link the ci.yml:123-131 rationale), fix the check-codegen.ps1 comment, and either repurpose or delete the cron (a nightly portable re-run has some drift-detection value — if kept, say so).
|
||||
|
||||
### TST-27 — `ShowTagValues` config doc still says "Reserved" after SEC-25 made the flag live — **Medium** · documentation currency
|
||||
|
||||
- **Files:** `docs/GatewayConfiguration.md:185` ("Reserved display control for tag values"); `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardEventBroadcaster.cs:16-29` (flag read and enforced); `docs/GatewayDashboardDesign.md:170` (current, documents the redaction)
|
||||
- **Description:** SEC-25 wired `Dashboard:ShowTagValues` into `DashboardEventBroadcaster`: when `false` (default), tag values are blanked from the deep-cloned event before it is mirrored to SignalR. The flag is therefore no longer dead — but the authoritative config reference still labels it "Reserved", so an operator consulting the options table concludes toggling it does nothing, when it actually controls whether tag values leak to every dashboard hub subscriber. This is remediation-created drift: the dashboard design doc was updated in the same change, the config doc was not. (Residual of TST-16 remains separately: the flag still gates nothing in the `/browse` live-value path.)
|
||||
- **Recommendation:** Update the `:185` row to describe the mirror-redaction behavior (and its security relevance given the missing per-session ACL, TST-15); note the `/browse` gap or close it when TST-16 is decided.
|
||||
|
||||
### TST-28 — Gateway-side `max_frame_bytes` handshake field has no test in the portable suite — **Low** · test coverage
|
||||
|
||||
- **Files:** `src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs:988` (`MaxFrameBytes = (uint)_connection.FrameOptions.MaxMessageBytes` in `GatewayHello`); no reference to `MaxFrameBytes` anywhere under `src/ZB.MOM.WW.MxGateway.Tests` (grep empty)
|
||||
- **Description:** The frame-size negotiation added since baseline is asserted only on the worker side (adoption tests in `Worker.Tests`, Windows-only per TST-25). Nothing in the CI-run gateway suite asserts the gateway actually populates the field from its configured `MaxMessageBytes` — a regression to sending `0` (which the worker treats as "older gateway, use default") would pass every automated test while silently reverting the negotiated limit.
|
||||
- **Recommendation:** One fake-worker assertion: capture the `GatewayHello` in `FakeWorkerHarness` and assert `MaxFrameBytes` equals the configured `Worker.MaxMessageBytes` (both default and overridden).
|
||||
|
||||
### TST-29 — `oldtasks.md` retirement decision needs refresh now that it carries the epic-closure record — **Low** · repo hygiene
|
||||
|
||||
- **Files:** `oldtasks.md:27-34,65-93`; repo root (five untracked `*-docs-*.md` files still present)
|
||||
- **Description:** TST-14's plan was "keep `oldtasks.md` only until the epic resumes, then delete", but TST-04 made it the durable governance record for Phase-4-scoped / Phase-5-deferred — deleting it now would orphan the only statement that `EnableOrphanReattach` must not be referenced. Meanwhile the mechanical half of TST-14 (deleting the user's five untracked, gitignored `*-docs-*.md` working files, 15k+ lines) remains undone and keeps tripping greps.
|
||||
- **Recommendation:** Fold the Phase-5-deferred statement into `docs/DesignDecisions.md` (where invariant-adjacent decisions live), then retire `oldtasks.md`; delete the untracked docs-review artifacts (zero repo impact).
|
||||
|
||||
## Severity roll-up (new findings)
|
||||
|
||||
| Severity | Count | IDs |
|
||||
|---|---|---|
|
||||
| High | 1 | TST-25 |
|
||||
| Medium | 2 | TST-26, TST-27 |
|
||||
| Low | 2 | TST-28, TST-29 |
|
||||
|
||||
## Still good / preserved
|
||||
|
||||
- The reconnect/replay e2e test (TST-01) is the strongest new test in the tree: real gRPC path, deterministic gated emission, detach-survival of the ring, exact-sequence and sentinel-shape assertions — a model for future e2e work.
|
||||
- The new remediation machinery is genuinely tested, not tautologically: `CachingApiKeyVerifierTests` (9 facts, incl. `CoalescingMarkApiKeyStore` window behavior at `:106,:125` with an injected clock), `ApiKeyFailureLimiter` exercised through `GatewayGrpcAuthorizationInterceptorTests:363-445` (asserts the verifier is **not** called once tripped, and that success resets), faulted-session reaping via `SessionManagerTests.cs:985` (default reap) and `:1015` (grace window), `DashboardEventBroadcasterTests` (redaction), `DashboardLoginRateLimitTests`, plus a new repo-hygiene guard (`Tests/ProjectStructure/GatewayTreeHygieneTests.cs`) that fails if a stray SQLite DB materializes under `src/`.
|
||||
- Documentation currency of the load-bearing docs is high where the remediation touched them: `docs/GatewayConfiguration.md` fully documents the new `Security:*` block (`:278-284`) and `FaultedGraceSeconds` (`:141`) matching `SecurityOptions.cs`/`SessionOptions.cs`; `docs/Sessions.md:220` documents owner-scoped attach precisely as implemented; `gateway.md`'s envelope section now points at the proto instead of inlining it; `docs/WorkerFrameProtocol.md` covers the `max_frame_bytes` handshake. The two drift spots found (TST-26, TST-27) are the exceptions, not the pattern.
|
||||
- The codegen-freshness guard (`scripts/check-codegen.ps1`) is real defense-in-depth: descriptor set, `Contracts/Generated` diff, and vendored Rust protos all checked on every push, directly covering this repo's most-repeated historical failure class — and the tracking log shows CI caught five latent defects on its first green run.
|
||||
@@ -0,0 +1,163 @@
|
||||
# MxAccessGateway — Remediation Tracking (2026-07-12 review)
|
||||
|
||||
Master progress tracker for the 2026-07-12 follow-up architecture review. Generated 2026-07-13.
|
||||
|
||||
Source review: [`../00-overall.md`](../00-overall.md) · Per-domain remediation designs are linked below and hold the full **Finding / Impact / Design / Implementation / Verification** for every entry here. The first-cycle tracker ([`../../remediation/00-tracking.md`](../../remediation/00-tracking.md)) remains the record for the original 153 findings; this document tracks only the 47 new IDs (GWC-24+, WRK-21+, IPC-23+, SEC-31+, CLI-35+, TST-25+) plus the old-tracker actions listed at the end.
|
||||
|
||||
## How to use this document
|
||||
|
||||
- Each finding has a stable ID that never changes. Cite it in commits, branches, and PRs (e.g. `fix(GWC-25): empty-ring ReplayGap sentinel`).
|
||||
- The **Status** column is the single source of truth for progress. Update it in the same change that lands the fix.
|
||||
- Do the work in **roadmap-tier order** (P0 → P1 → P2), respecting the `Dep` column and the cross-cutting clusters below — several fixes are one change set across two domains and must land together.
|
||||
- When a fix lands: flip Status to `Done` and, per the repo rule, update the affected docs in the same commit.
|
||||
|
||||
**Status legend:** `Not started` · `In progress` · `In review` · `Done` · `Won't fix` (record why in the domain doc) · `N/A` (informational / decision recorded, no action).
|
||||
|
||||
## Severity roll-up
|
||||
|
||||
| Domain | Doc | High | Medium | Low | Info | Total |
|
||||
|---|---|:-:|:-:|:-:|:-:|:-:|
|
||||
| Gateway core | [10-gateway-core.md](10-gateway-core.md) | — | 2 | 4 | 1 | 7 |
|
||||
| Worker | [20-worker.md](20-worker.md) | — | 1 | 7 | — | 8 |
|
||||
| Contracts & IPC | [30-contracts-ipc.md](30-contracts-ipc.md) | — | 3 | 5 | 2 | 10 |
|
||||
| Security & dashboard | [40-security-dashboard.md](40-security-dashboard.md) | — | 1 | 4 | 1 | 6 |
|
||||
| Clients | [50-clients.md](50-clients.md) | — | 5 | 6 | — | 11 |
|
||||
| Testing, docs & gaps | [60-testing-docs-gaps.md](60-testing-docs-gaps.md) | 1 | 2 | 3 | — | 6 |
|
||||
| **Total** | | **1** | **14** | **29** | **4** | **48** |
|
||||
|
||||
## Roadmap-tier roll-up
|
||||
|
||||
| Tier | Meaning | Count | Findings |
|
||||
|---|---|:-:|---|
|
||||
| **P0** | Correctness & safety — all small-to-medium | 10 | GWC-25, WRK-21, IPC-23, IPC-24, IPC-25, IPC-30, SEC-31, SEC-32, CLI-35, CLI-36 |
|
||||
| **P1** | Process & hardening | 12 | GWC-24, WRK-26, SEC-33, SEC-36, CLI-37, CLI-38, CLI-39, CLI-42, CLI-45, TST-25, TST-26, TST-27 |
|
||||
| **P2** | Completeness & polish | 10 | GWC-26, GWC-27, GWC-28, WRK-25, IPC-26, IPC-27, SEC-34, TST-28, TST-29, TST-30 |
|
||||
| **—** | Not individually in the roadmap (Lows/Infos) | 16 | GWC-29/30, WRK-22/23/24/27/28, IPC-28/29/31/32, SEC-35, CLI-40/41/43/44 |
|
||||
|
||||
### P0 — do first (10)
|
||||
|
||||
Sequenced by cluster; a cluster is one change set.
|
||||
|
||||
| ID | Sev | Eff | Dep | Status | Title |
|
||||
|---|---|:-:|---|---|---|
|
||||
| GWC-25 | Medium | S | CLI-35/36 (coord) | Not started | Empty-ring ReplayGap sentinel carries `oldest_available_sequence = 0`, dead-streaming a compliant client |
|
||||
| CLI-35 | Medium | S | GWC-25 (coord) | Not started | Python CLI `stream-events` crashes on a ReplayGap |
|
||||
| CLI-36 | Medium | S | GWC-25 (coord) | Not started | Go CLI `stream-events` silently destroys the ReplayGap signal |
|
||||
| WRK-21 | Medium | M | owns IPC-23 fix; WRK-28 same batch | Not started | DrainEvents bound is count-based only; oversized reply kills the session and loses the drained events |
|
||||
| IPC-23 | Medium | S | WRK-21 | Not started | DrainEvents contract requirements (reply fits negotiated max, no event loss, drain-until-empty) + proto-comment/doc wave |
|
||||
| IPC-30 | Low | M | WRK-21 (same batch) | Not started | Oversized event frame stays session-fatal by design, but the death becomes structured (fault frame + logged identity) |
|
||||
| SEC-31 | Medium | M | — | Not started | Failure limiter partitions on attacker-controlled key id and blocks before verification (lockout DoS) |
|
||||
| SEC-32 | Low | S | SEC-31 | Not started | Failure-limiter LRU flushable by junk-token spray; token prefix never validated |
|
||||
| IPC-24 | Medium | S | — | Not started | CI's unconditional Java churn-revert masks real drift |
|
||||
| IPC-25 | Medium | M | — | Not started | Regenerate stale Go/Python worker bindings + add binding-freshness guard (Check 4) to check-codegen.ps1 |
|
||||
|
||||
## Finding registers by domain
|
||||
|
||||
Full design + implementation for each row lives in the linked domain doc under its ID heading.
|
||||
|
||||
### Gateway core — [10-gateway-core.md](10-gateway-core.md)
|
||||
|
||||
| ID | Sev | Tier | Eff | Dep | Status | Title |
|
||||
|---|---|:-:|:-:|---|---|---|
|
||||
| GWC-24 | Medium | P1 | M | GWC-21 (coord, old tracker) | Not started | Unbounded event staging channel: sustained slow drain grows memory silently and invisibly |
|
||||
| GWC-25 | Medium | P0 | S | CLI-35/36 (coord) | Not started | Empty-ring ReplayGap sentinel carries `oldest_available_sequence = 0` |
|
||||
| GWC-26 | Low | P2 | M | GWC-27 | Not started | Alarm monitor attaches its subscriber after SubscribeAlarms; window transitions bypass the feed |
|
||||
| GWC-27 | Low | P2 | S | GWC-26 | Not started | `AttachInternalEventSubscriber` bypasses the readiness gate; premature attach poisons the distributor |
|
||||
| GWC-28 | Low | P2 | S | GWC-10 (coord, old tracker) | Not started | Gateway→worker envelope `sequence` stamped at creation, not at write |
|
||||
| GWC-29 | Low | — | S | — | Not started | `Invoke` deep-clones the entire request only to discard the cloned command |
|
||||
| GWC-30 | Info | — | S | — | Not started | Frame reader allocates a fresh 4-byte length-prefix array per frame |
|
||||
|
||||
### Worker — [20-worker.md](20-worker.md)
|
||||
|
||||
| ID | Sev | Tier | Eff | Dep | Status | Title |
|
||||
|---|---|:-:|:-:|---|---|---|
|
||||
| WRK-21 | Medium | P0 | M | IPC-23 (fix owned here); WRK-28 | Not started | DrainEvents bound count-based only; oversized reply kills session and loses drained events |
|
||||
| WRK-22 | Low | — | S | IPC-26 (fix owned here) | Not started | Cancelled `WriteAsync` leaves its frame queued; it is still written later |
|
||||
| WRK-23 | Low | — | S | WRK-21 | Not started | Rejected frames consume sequence numbers, producing wire gaps |
|
||||
| WRK-24 | Low | — | S | — | Not started | `AdoptNegotiatedMaxMessageBytes` has no lower-bound sanity check |
|
||||
| WRK-25 | Low | P2 | S | WRK-22 (shared seam) | Not started | WRK-12 flush coalescing never engages on the event hot path |
|
||||
| WRK-26 | Low | P1 | S | WRK-23 (soft); discharges IPC-29 | Not started | Write-priority and overflow doc drift from the WRK-07 change |
|
||||
| WRK-27 | Low | — | S | — | Not started | Alarm poll bypasses the watchdog's in-flight suppression (15 s vs 75 s) |
|
||||
| WRK-28 | Low | — | S | WRK-21 (same batch) | Not started | 10,000 drain cap is a duplicated magic constant |
|
||||
|
||||
### Contracts & IPC — [30-contracts-ipc.md](30-contracts-ipc.md)
|
||||
|
||||
| ID | Sev | Tier | Eff | Dep | Status | Title |
|
||||
|---|---|:-:|:-:|---|---|---|
|
||||
| IPC-23 | Medium | P0 | S | WRK-21 (mechanics) | Not started | DrainEvents byte-blindness — contract requirements + proto-comment/doc wave |
|
||||
| IPC-24 | Medium | P0 | S | — | Not started | CI's unconditional Java churn-revert masks real drift |
|
||||
| IPC-25 | Medium | P0 | M | — | Not started | Stale Go/Python worker bindings: regenerate (pinned toolchains) + check-codegen Check 4 |
|
||||
| IPC-26 | Low | P2 | S | WRK-22 (mechanics) | Not started | Cancelled write leaves ghost frame — cancelled means never written |
|
||||
| IPC-27 | Low | P2 | S | — | Not started | Descriptor-freshness test blind to enums/services/galaxy descriptor |
|
||||
| IPC-28 | Low | — | S | — | Not started | docs/Grpc.md missing CommandTooLarge → ResourceExhausted mapping |
|
||||
| IPC-29 | Low | — | S | WRK-26 (discharged by) | Not started | WorkerFrameProtocol.md missing write-scheduling/sequencing section |
|
||||
| IPC-30 | Low | P0 | M | WRK-21 (same batch) | Not started | Oversized event frame: keep session-fatal, make the death structured |
|
||||
| IPC-31 | Info | — | — | — | N/A | Gateway creation-time sequence stamping accepted; diagnostic-only, decision recorded |
|
||||
| IPC-32 | Info | — | S | IPC-25 (folded in) | Not started | check-codegen banner relabel 1/4…4/4 |
|
||||
|
||||
### Security & dashboard — [40-security-dashboard.md](40-security-dashboard.md)
|
||||
|
||||
| ID | Sev | Tier | Eff | Dep | Status | Title |
|
||||
|---|---|:-:|:-:|---|---|---|
|
||||
| SEC-31 | Medium | P0 | M | — | Not started | Failure limiter: composite (peer, key-id) partitions + cross-peer aggregate with probe admission |
|
||||
| SEC-32 | Low | P0 | S | SEC-31 | Not started | Limiter LRU flushable by junk-token spray; validate token shape, cap per-peer partitions |
|
||||
| SEC-33 | Low | P1 | M | old SEC-23 (co-locate) | Not started | Host-meaningful path rooting; drop Windows literals from appsettings; validate Galaxy `SnapshotCachePath` |
|
||||
| SEC-34 | Low | P2 | S | — | Not started | Verification cache: expiry outlives TTL; `Invalidate` races in-flight repopulation |
|
||||
| SEC-35 | Info | — | S | — | N/A | Production hard-stops key on exact `Production` environment name (doc-only) |
|
||||
| SEC-36 | Low | P1 | M | cross-repo `scadaproj/infra/glauth` | Not started | Committed dev LDAP service-account password: rotate, remove, move dev channel to user-secrets |
|
||||
|
||||
### Clients — [50-clients.md](50-clients.md)
|
||||
|
||||
| ID | Sev | Tier | Eff | Dep | Status | Title |
|
||||
|---|---|:-:|:-:|---|---|---|
|
||||
| CLI-35 | Medium | P0 | S | GWC-25 (coord) | Not started | Python CLI `stream-events` crashes on a ReplayGap |
|
||||
| CLI-36 | Medium | P0 | S | GWC-25 (coord) | Not started | Go CLI `stream-events` silently destroys the ReplayGap signal |
|
||||
| CLI-37 | Medium | P1 | M | CLI-38 (co-land) | Not started | Status-array validation must branch on `category` per the proto contract (4-vs-1 divergence) |
|
||||
| CLI-38 | Medium | P1 | S | — | Not started | Align .NET/Go/Java on `hresult < 0` — lands old CLI-08, cures design-doc drift |
|
||||
| CLI-39 | Medium | P1 | S | CLI-35..38, CLI-45 (land last) | Not started | Bump client versions off published 0.1.2 (converge on 0.2.0); registry-collision guard in pack-clients.ps1 |
|
||||
| CLI-40 | Low | — | M | — | Not started | Port the exact-secret credential scrub to Rust/Java/.NET |
|
||||
| CLI-41 | Low | — | M | — | Not started | Uniform malformed-reply contract for AuthenticateUser/ArchestrAUserToId/AddBufferedItem |
|
||||
| CLI-42 | Low | P1 | S | — | Not started | Document the vendored Rust proto layout (CLI-02's missing doc half) |
|
||||
| CLI-43 | Low | — | S | — | Not started | Java style guide still prescribes "Java 21 preferred" |
|
||||
| CLI-44 | Low | — | S | — | Not started | Go event goroutine can mislabel a genuine terminal error as `ErrSlowConsumer` |
|
||||
| CLI-45 | Low | P1 | M | — | Not started | Standardize CLI credential env-var names; fail fast on missing/empty passwords |
|
||||
|
||||
### Testing, docs & gaps — [60-testing-docs-gaps.md](60-testing-docs-gaps.md)
|
||||
|
||||
| ID | Sev | Tier | Eff | Dep | Status | Title |
|
||||
|---|---|:-:|:-:|---|---|---|
|
||||
| TST-25 | High | P1 | M | unlocks old TST-05, TST-24 | Done | Windows/x86 test tier has zero automation — SSH-driven windev CI job |
|
||||
| TST-26 | Medium | P1 | S | TST-25 (same commit) | Done | Docs/scripts describe removed CI jobs; Generated/-guard reattributed to check-codegen |
|
||||
| TST-27 | Medium | P1 | S | — | Not started | `ShowTagValues` config row still says "Reserved" after SEC-25 made the flag live |
|
||||
| TST-28 | Low | P2 | S | relates IPC-02 (old) | Not started | Gateway-side `max_frame_bytes` handshake untested in the CI-run suite |
|
||||
| TST-29 | Low | P2 | S | — | Not started | Retire `oldtasks.md` (fold Phase-5 governance into DesignDecisions.md); delete root artifacts |
|
||||
| TST-30 | Low | P2 | M | — | Not started | Single shared Gitea runner is a CI throughput/availability bottleneck (cross-repo contention, no run cancel/delete API) |
|
||||
|
||||
## Cross-cutting clusters
|
||||
|
||||
Sequence these together rather than piecemeal — several are one change set spanning two domains:
|
||||
|
||||
- **Drain / oversized-frame cluster (P0):** WRK-21 + IPC-23 + IPC-30, with WRK-28 and WRK-23 in the same batch. One worker change set in `WorkerPipeSession.cs`/`WorkerFrameWriter.cs`; WRK-21 owns the byte-budgeted drain and must satisfy IPC-23's requirements R1–R3; IPC-30's structured-fault seam lands in the same drain loop. IPC-23's proto-comment edits trigger the full regen fan-out (Generated/, Rust vendored, Go/Java, descriptor set) — land with WRK-21 so the wave happens once. One windev x86 verification run for the cluster.
|
||||
- **ReplayGap end-to-end (P0):** GWC-25 (server sentinel arithmetic) + CLI-35 (Python CLI) + CLI-36 (Go CLI). Independently landable; the e2e resume walk validates only when all three are in.
|
||||
- **Auth limiter (P0):** SEC-31 + SEC-32 — same component, one change set, one test suite.
|
||||
- **Codegen freshness (P0):** IPC-24 + IPC-25 (+ IPC-32 folded in). Both edit `.gitea/workflows/ci.yml` — coordinate the branch with TST-25, which touches the same file.
|
||||
- **Windows-tier automation (P1):** TST-25 + TST-26 (same commit). Unlocks old TST-05/TST-24 and provides CI evidence for every windev-verified cluster above; until it lands, record windev runs in this tracker's change log.
|
||||
- **Client conformance + release train (P1):** CLI-37 + CLI-38 co-land (one conformance commit; closes old CLI-08), then CLI-45, with CLI-40/41 fixtures as follow-ups; **CLI-39 lands last** so published 0.2.0 carries the conformant behavior. Shared fixtures under `clients/proto/fixtures/behavior/`; update CrossLanguageSmokeMatrix.md/ClientLibrariesDesign.md same-commit. Do not republish regenerated bindings (IPC-25) before CLI-39 resolves.
|
||||
- **Doc-drift batch (P1):** TST-27 + WRK-26 (discharges IPC-29) + CLI-42 + IPC-28 + SEC-35's doc note — one sweep commit is fine.
|
||||
- **Backpressure follow-on (P1):** GWC-24, coordinating with still-open old GWC-21 (`EventChannelFullModeTimeout` configurability).
|
||||
|
||||
## Old-tracker actions ([`../../remediation/00-tracking.md`](../../remediation/00-tracking.md))
|
||||
|
||||
- Close **CLI-24** and **CLI-34** as `Done` (incidentally fixed; evidence in [../50-clients.md](../50-clients.md)).
|
||||
- When CLI-38 lands, close old **CLI-08** with a pointer here.
|
||||
- When WRK-26 lands, its doc section also discharges the WorkerFrameProtocol gap; when TST-25 lands, revisit old **TST-05** (scheduled live smoke) and **TST-24** (client wire tests), which it unlocks.
|
||||
|
||||
## Change log
|
||||
|
||||
| Date | Change |
|
||||
|---|---|
|
||||
| 2026-07-13 | Initial tracking doc generated from the six domain remediation designs. All 47 findings `Not started` (IPC-31, SEC-35 `N/A`). |
|
||||
| 2026-07-13 | TST-25/TST-26 → `In progress` (branch `fix/tst-25-windev-ci`). Added `scripts/ci/{windev-worker-ci.ps1,run-windev-ci.sh,windev.known_hosts}`, `windows-x86` (per-push) + `nightly-windev` (scheduled) jobs in `ci.yml`, and the TST-26 doc/comment fixes (GatewayTesting.md, Contracts.md, check-codegen.ps1). Mechanism hand-verified on windev: `build`→0, bogus-SHA→nonzero (lock released), `test`→356 passed/0 failed in ~50s (per-push stays `test`, no demotion), and run-windev-ci.sh SSH+EncodedCommand exit-code propagation confirmed. |
|
||||
| 2026-07-13 | Operator bring-up complete: dedicated CI ed25519 key installed in windev `administrators_authorized_keys` (authorized into `dohertj2`, which owns the working MXAccess/toolchain env — a fresh OS account would break the build; the key is independently revocable), Gitea secrets `WINDEV_SSH_KEY`/`WINDEV_SSH_KNOWN_HOSTS` + variable `WINDEV_SSH_USER=dohertj2` stored, runner→`10.100.0.48:22` egress verified on the `traefik` net, issue-write confirmed. **TST-25/TST-26 → `Done`:** credentialed `windows-x86` ran GREEN on `d769244` (Gitea run #37) — Linux runner SSHed windev, checked out the SHA in `C:\build\mxaccessgw-ci` under lock, ran the x86 Worker build + `Worker.Tests`, exit 0; `nightly-windev` correctly skipped on the push event. Branch merged to `main`. Follow-ups (old tracker): revisit **TST-05** (scheduled live smoke — now covered by `nightly-windev`) and **TST-24** (client wire tests) which this unlocks. |
|
||||
| 2026-07-13 | Ran the TST-25 acceptance checks (scripts/ci/README.md) — they caught **two real CI defects, both fixed** on `fix/tst-25-ci-key-log-leak`: (1) **CI SSH key leaked in cleartext** in the `windows-x86` step env echo (Gitea's line-oriented masker missed the multiline PEM) — rotated the CI key on windev (old pubkey revoked), stored the key **base64-encoded** so the masker redacts it to `***` (confirmed on run #38), taught `run-windev-ci.sh` to decode, dropped the redundant public known-hosts secret from the job env; (2) **bootstrap lock race** — `run-windev-ci.sh`'s pre-hand-off `git fetch`/`checkout` ran outside the worktree lock, so concurrent runs collided on `.git/index.lock`; the bootstrap now holds the lock (ps1 re-uses it via `MXGW_CI_LOCK_HELD`), retest confirmed clean serialization. Also **deflaked** `SessionManagerTests` fail-fast timing assertions (absolute `<100ms` wall-clock bound flaked under CI load; now anchored to the configured timeout / dropped for the zero-timeout case). Checks passed: unreachable-host fast-fail (exit 255/15s), deliberate-red propagation (Worker.Tests failure → exit 1), lock concurrency (2nd run waits), no-key-in-logs (masked). Merge target `df7e20d` verified GREEN via the local windev path (Worker build + 356 tests); merged to `main` `19cbf7b`. Check 6 (forced-failure nightly issue): issue endpoint+token proven live at bring-up (#124); in-CI forced-failure probe abandoned to shared-runner congestion (residual `if: failure()` gating is standard Actions). |
|
||||
| 2026-07-13 | New finding **TST-30** (`Low`/`P2`) added — surfaced during TST-25 acceptance: CI runs on a single shared `gitea-runner` (`maxParallel=1`, co-located `10.100.0.35`) interleaved with `dohertj2/lmxopcua`, and Gitea 1.26 exposes no run cancel/delete, so queue latency is unbounded under cross-repo contention and the runner is a single point of failure. Design: add a second/labelled runner + document the no-cancel reality and the `run-windev-ci.sh` bypass. Roll-ups updated (Testing Low 2→3, total 47→48; P2 9→10). |
|
||||
@@ -0,0 +1,171 @@
|
||||
# Gateway Server Core — Remediation Design & Implementation (2026-07-12 cycle)
|
||||
|
||||
Source review: [10-gateway-core.md](../10-gateway-core.md) · Roadmap: [00-overall.md](../00-overall.md) · Generated: 2026-07-13
|
||||
|
||||
This document turns the 2026-07-12 re-review's **new** Gateway Server Core findings (GWC-24 through GWC-30) into buildable remediation entries. Every cited `path:line` was re-verified against `main` at `4f5371f`. Tiers follow the overall roadmap: GWC-25 is P0.1 (grouped with the client-side CLI-35/36 ReplayGap handling), GWC-24 is P1.6, and GWC-26/27/28 sit in the P2 batch; GWC-29/30 are untiered polish. Prior-cycle findings still open (GWC-05, 09, 10, 11, 12, 13, 16–22) are tracked in `archreview/remediation/10-gateway-core.md` and are not re-planned here, but two of them (GWC-10, GWC-21) are named below where a new fix should coordinate with them.
|
||||
|
||||
## Finding index
|
||||
|
||||
| ID | Sev | Tier | Eff | Dep | Status | Title |
|
||||
|----|-----|------|-----|-----|--------|-------|
|
||||
| GWC-24 | Medium | P1 | M | GWC-21 (coord) | Not started | Unbounded event staging channel: sustained slow drain grows memory silently and invisibly |
|
||||
| GWC-25 | Medium | P0 | S | CLI-35/36 (coord) | Not started | Empty-ring ReplayGap sentinel carries `oldest_available_sequence = 0`, dead-streaming a compliant client |
|
||||
| GWC-26 | Low | P2 | M | GWC-27 | Not started | Alarm monitor attaches its subscriber after SubscribeAlarms; window transitions bypass the feed, missed Acknowledge never repaired |
|
||||
| GWC-27 | Low | P2 | S | GWC-26 | Not started | `AttachInternalEventSubscriber` bypasses the readiness gate; premature attach poisons the distributor permanently |
|
||||
| GWC-28 | Low | P2 | S | GWC-10 (coord) | Not started | Gateway→worker envelope `sequence` stamped at creation, not at write — non-monotonic on the wire under concurrent invokes |
|
||||
| GWC-29 | Low | — | S | — | Not started | `Invoke` deep-clones the entire request only to discard the cloned command |
|
||||
| GWC-30 | Info | — | S | — | Not started | Frame reader allocates a fresh 4-byte length-prefix array per frame |
|
||||
|
||||
Dependency notes: GWC-26 and GWC-27 both change the internal-subscriber attach path (`GatewaySession.AttachInternalEventSubscriber` and its `SessionManager`/alarm-monitor callers) — land GWC-27's readiness gate first (or in the same commit), then GWC-26's reorder, so the reordered monitor attach is proven against the gate. GWC-24 is the direct successor of the prior cycle's GWC-04 backpressure fix and raises the value of the still-open GWC-21 (making `EventChannelFullModeTimeout` configurable); GWC-28 is the gateway half of the worker's WRK-04 fix and must be coordinated with the still-open GWC-10 if inbound sequence enforcement is ever added. GWC-25 is server-complete on its own, but the end-to-end reconnect story also needs the client-domain CLI-35/36 fixes (Python CLI crashes on the sentinel, Go CLI destroys it).
|
||||
|
||||
---
|
||||
|
||||
## GWC-24 — Unbounded event staging channel: sustained slow drain grows memory silently and invisibly `Medium` · `P1`
|
||||
|
||||
**Finding.** The GWC-04 remediation decoupled the read loop from event backpressure by staging events into `_eventStaging`, an **unbounded** channel (`Workers/WorkerClient.cs:93-100`). `StageWorkerEvent`'s `TryWrite` therefore always succeeds (`:565-573`), and the sustained-overflow `ProtocolViolation` fault fires only when a *single* timed `WriteAsync` against the bounded `_events` exceeds `EventChannelFullModeTimeout` (default 5 s, `:610-647`). The queue-depth gauge counts only `_events` — `_eventQueueDepth` is incremented in `EnqueueWorkerEventAsync` (`:616`, `:627`) and decremented in `ReadEventsCoreAsync` (`:289`), so staged-but-unqueued events are invisible to `SetWorkerEventQueueDepth`. The field comment (`:28-33`) claims staging "only fills during the bounded EventChannelFullModeTimeout window", which is true only for a full consumer stall, not for a consumer that drains slower than the worker produces while each individual write still completes inside the window.
|
||||
|
||||
**Impact.** Before GWC-04, a full `_events` blocked the read loop, which propagated backpressure through the pipe to the worker's own bounded queue — gateway memory per session was structurally bounded end-to-end. Now a slow-but-live distributor pump (each `WriteAsync` completing in, say, 4 s while the worker emits faster) grows `_eventStaging` without bound, without any fault, and without any metric showing it. The repo's fail-fast backpressure invariant currently rests on pump liveness (the pump's own fan-out is non-blocking `TryWrite`, `Sessions/SessionEventDistributor.cs:582-594`) rather than on channel bounds — unlikely to bite, but no longer structurally impossible, and undiagnosable when it does.
|
||||
|
||||
**Design.** Do **both** halves of the review's recommendation — restore a structural bound *and* make the backlog observable:
|
||||
|
||||
1. **Bound the staging channel** at `2 × EventChannelCapacity` (derived from the existing option; no new configuration key). Total gateway-side buffering per session becomes `3 × MxGateway:Events:QueueCapacity` (bounded consumer channel + 2× staging), which absorbs the same bursts the unbounded design absorbed in practice while capping the pathological case. Create it with `FullMode = BoundedChannelFullMode.Wait` so `TryWrite` returns `false` when full (the same Wait+TryWrite overflow-detection idiom the distributor documents at `SessionEventDistributor.cs:354-361`).
|
||||
2. **Fault immediately on a staging `TryWrite` miss.** A full staging channel means `EventWriteLoopAsync` has been saturated against a full `_events` for however long the worker took to emit `2 × capacity` further events — that *is* the sustained-slow-drain signal, with no timer needed. `StageWorkerEvent` treats a `false` return as `SetFaulted(ProtocolViolation, …)` unless the client is already terminal (a completed channel also returns `false` during shutdown — guard with the existing `IsTerminalState()` so the shutdown path stays a silent drop, as documented today). `SetFaulted` is non-blocking, so calling it from `DispatchEnvelope` on the read loop preserves the GWC-04 guarantee that the read loop never awaits behind events. Keep the existing `EventChannelFullModeTimeout` timed-write fault in `EnqueueWorkerEventAsync` unchanged — it catches the full-stall case earlier than the staging bound would.
|
||||
3. **Unify the queue-depth gauge over both channels.** Move the `Interlocked.Increment(ref _eventQueueDepth)` + `SetWorkerEventQueueDepth` from `EnqueueWorkerEventAsync` (`:616-617`, `:627-628`) to `StageWorkerEvent` (on successful `TryWrite`). The decrement already happens at consumer read (`ReadEventsCoreAsync:289`), so the single counter then reports *total undelivered events* (staged + queued) with no second counter and no extra hot-path work. Record `_metrics?.QueueOverflow("worker-event-staging")` on the staging fault so it is distinguishable from the existing `"worker-events"` overflow label.
|
||||
|
||||
Rejected alternatives: (a) unbounded channel + a total-depth ceiling check — same effect but needs a second configured ceiling and a watermark comparison per event; the bounded channel gives the ceiling for free and fails at exactly the point the invariant is violated. (b) fault-on-sustained-growth (timestamp when the backlog first appears, fault when it persists past a window) — more machinery and a weaker guarantee, since memory still grows throughout the window. Fail-fast is the repo's documented backpressure policy (`docs/DesignDecisions.md`); a loud session fault with a worker kill is the correct outcome, not silent buffering.
|
||||
|
||||
Coordinate with (do not block on) open GWC-21: if `EventChannelFullModeTimeout` is made configurable in the same pass, the 2× staging multiplier is the natural companion knob; until then the derived constant needs no validator change.
|
||||
|
||||
**Implementation.**
|
||||
- `Workers/WorkerClient.cs` constructor (`:93-100`): replace `Channel.CreateUnbounded<WorkerEvent>` with `Channel.CreateBounded<WorkerEvent>(new BoundedChannelOptions(checked(2 * _options.EventChannelCapacity)) { SingleReader = true, SingleWriter = true, FullMode = BoundedChannelFullMode.Wait, AllowSynchronousContinuations = false })`. Rewrite the `_eventStaging` field comment (`:28-33`) — it must state the bound, the overflow fault, and that the depth gauge covers staged + queued.
|
||||
- `StageWorkerEvent` (`:565-573`): on `TryWrite` success, `Interlocked.Increment(ref _eventQueueDepth)` + `_metrics?.SetWorkerEventQueueDepth(...)`; on failure, `if (IsTerminalState()) return;` then `_metrics?.QueueOverflow("worker-event-staging")` and `SetFaulted(WorkerClientErrorCode.ProtocolViolation, …)` with a message naming the staging capacity (`2 × EventChannelCapacity`), the current depth, and the actionable fix (attach/unblock the `StreamEvents` consumer or raise `MxGateway:Events:QueueCapacity`). Update the XML doc, which currently asserts `TryWrite` always succeeds.
|
||||
- `EnqueueWorkerEventAsync` (`:610-647`): remove the two depth increments (now counted at staging); keep the timed `WriteAsync`, both fault paths, and the `Volatile.Read` depth in the overflow message.
|
||||
- `ReadEventsCoreAsync` (`:284-293`): unchanged — the decrement site is already correct for the unified counter.
|
||||
- Docs, same commit: `docs/MxAccessWorkerInstanceDesign.md` / `docs/GatewayProcessDesign.md` wherever the GWC-04 read-loop/event-writer decoupling is described (state the staging bound and the two fault modes); `docs/GatewayConfiguration.md` under `MxGateway:Events:QueueCapacity` (total per-session gateway buffering is 3× the value; overflow of either bound faults the session); the metrics doc if it defines the worker event queue-depth gauge (now includes staged events).
|
||||
- Tests (`Gateway/Workers/WorkerClientTests.cs`), new:
|
||||
- `StagingChannelOverflowFaultsWorkerWithoutWaitingForFullModeTimeout` — small `EventChannelCapacity` (e.g. 4) via `WorkerClientOptions`, no event consumer attached, long `EventChannelFullModeTimeout` (e.g. 5 min so the timed-write fault cannot be the trigger); push `> 3 × capacity` events through the fake pipe; assert the client faults `ProtocolViolation` with the staging message, and that a command reply interleaved before the fault still completed (read loop never stalled).
|
||||
- `WorkerEventQueueDepthGaugeCountsStagedEvents` — no consumer, push N events (N below the staging bound but above `EventChannelCapacity`); assert the gauge reports N (staged + queued); attach the consumer, drain, assert the gauge returns to 0.
|
||||
- Existing GWC-04 tests (reply-not-stalled-behind-events, sustained-overflow fault) must stay green.
|
||||
|
||||
**Verification.** `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx`; `dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~WorkerClientTests"`. Gateway-side only — no worker change, no windev/x86 needed.
|
||||
|
||||
---
|
||||
|
||||
## GWC-25 — Empty-ring ReplayGap sentinel carries `oldest_available_sequence = 0`, dead-streaming a compliant client `Medium` · `P0`
|
||||
|
||||
**Finding.** `RegisterWithReplay`'s empty-ring branch sets `oldestAvailableSequence = 0` even when `gap == true` (`Sessions/SessionEventDistributor.cs:463-467`), and `EventStreamService` emits that verbatim in the ReplayGap sentinel (`Grpc/EventStreamService.cs:133-139`, `:210-222`). The proto contract (`src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_gateway.proto`, `ReplayGap.oldest_available_sequence`) instructs a client that wants a gapless resume to send `after_worker_sequence = oldest_available_sequence − 1`. With `oldest = 0`, a uint64 client computes `2^64−1`; on the next resume `RegisterWithReplay` replays nothing and reports no gap (the `afterSequence == ulong.MaxValue` wrap guard at `:471`/`:757` suppresses it), and the live filter `mxEvent.WorkerSequence <= afterWorkerSequence` (`EventStreamService.cs:179`) then drops **every** live event — a silently dead stream. Reachable by default: `ReplayRetentionSeconds = 300` age-evicts inside `RegisterWithReplay` via `EvictAged()` (`:458`), and `ReplayBufferCapacity = 0` with a behind cursor hits the same branch. All five clients shipped the `oldest − 1` formula (CLI-15).
|
||||
|
||||
**Impact.** A client that reconnects after ≥ 5 minutes detached (or against a retention-disabled gateway) and follows the documented resume formula gets a permanently event-less stream with no error — the exact resilience feature (detach-grace + replay, on by default) fails in its headline scenario.
|
||||
|
||||
**Design.** In the empty-ring-with-gap branch, emit `oldest_available_sequence = _highestSequenceSeen + 1` — the next sequence that can possibly be delivered. The client's `oldest − 1` then equals `_highestSequenceSeen`: the follow-up resume replays nothing, reports no gap, and the live filter passes every event newer than the highest already seen. Nothing is lost relative to today (the evicted events are unrecoverable either way — the sentinel's whole job is to say "re-snapshot"), and no client changes. Keep `0` for the no-gap case, where the field is documented as meaningless and never emitted. `TryGetReplayFrom` (`:735-774`) does not surface an oldest-available value, so it needs no change. Rejected alternative: define `0` as a special "nothing retained" case in the proto and handle it in all five clients — five client releases plus a version bump (CLI-39 is already blocking publishes) versus a one-line server fix that keeps the single resume formula universal.
|
||||
|
||||
The proto comment currently states "`oldest_available_sequence` itself IS still retained", which becomes false in the empty-ring case — per the docs-with-source rule, amend the field comment in the same commit to define the empty-ring value ("when nothing is retained, this is the next sequence that can be delivered — `highest observed + 1` — and the `oldest − 1` resume formula remains valid; the interval evicted is unchanged"). This is a comment-only proto change (no descriptor delta), but the repo's codegen rules still apply — see the steps.
|
||||
|
||||
**Implementation.**
|
||||
- `Sessions/SessionEventDistributor.cs:463-467`: replace `oldestAvailableSequence = 0;` with `oldestAvailableSequence = gap ? _highestSequenceSeen + 1 : 0;` plus a comment explaining the `oldest − 1` client formula this must keep valid (cite this finding).
|
||||
- `src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_gateway.proto` (`ReplayGap.oldest_available_sequence`, ~line 759): append the empty-ring sentence above. Then regenerate per repo rules: delete `src/ZB.MOM.WW.MxGateway.Contracts/Generated/*.cs`, `dotnet build src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj`, and **commit `Generated/`** (net48 worker builds break otherwise). Sync the vendored client copies of the proto byte-identical (`clients/*/`); a comment-only edit changes no descriptor, so: Python `*_pb2*` output is unchanged (comments are not embedded — regenerate with the pinned grpcio-tools only if the files actually differ), Go/C#/Rust generated doc comments will churn — regenerate those per each client README, and revert spurious Java aggregate-file churn if no message-level delta appears (per the established Java convention).
|
||||
- `docs/Sessions.md` (~lines 228-234, ReplayGap section): document the empty-ring sentinel value and that `after_worker_sequence = oldest_available_sequence − 1` is the universal resume formula in both the retained and fully-evicted cases.
|
||||
- Tests, new:
|
||||
- `Gateway/Sessions/SessionEventDistributorTests.cs` — `RegisterWithReplayReportsNextDeliverableSequenceWhenRingEmptiedByAge`: FakeTimeProvider, capacity > 0, short retention; pump events up to highest sequence H; advance the clock past retention; `RegisterWithReplay(afterSequence: 1, …)` → asserts `gap == true`, `oldestAvailableSequence == H + 1`, `replayedEvents` empty, `liveResumeSequence == 1`.
|
||||
- `RegisterWithReplayReportsNextDeliverableSequenceWithRetentionDisabled`: capacity-0 internal ctor overload, events seen to H, resume behind → `gap == true`, `oldestAvailableSequence == H + 1`.
|
||||
- `ResumeUsingSentinelFormulaAfterEmptyRingGapDeliversLiveEvents`: after the gap above, re-register with `afterSequence = (H + 1) − 1 = H` → no gap, no replay, and a newly pumped event `H + 1` is delivered on the live channel — the direct regression test for the dead-stream failure.
|
||||
- `Gateway/GatewayEndToEndReconnectReplayTests.cs` — `ReconnectAfterFullAgeEvictionResumesWithSentinelFormula`: fake-worker e2e; stream, detach, evict everything by advancing time past `ReplayRetentionSeconds`, reconnect with the last-seen watermark, apply `oldest − 1` from the received sentinel on the follow-up resume, assert new live events arrive.
|
||||
|
||||
Cross-domain: the client-side halves — Python CLI crashing on the sentinel (CLI-35) and the Go CLI destroying it (CLI-36) — are planned in the clients remediation doc; this server fix is independent and safe to land first (roadmap P0.1 groups all three).
|
||||
|
||||
**Verification.** `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx` (includes the contracts regen); `dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~SessionEventDistributorTests"` and `--filter "FullyQualifiedName~GatewayEndToEndReconnectReplay"`. If the proto comment lands: the committed `Generated/` keeps the net48 worker buildable, but an actual x86 worker build/test run needs windev — flag as the standard windev verification (comment-only change, so functional risk is nil).
|
||||
|
||||
---
|
||||
|
||||
## GWC-26 — Alarm monitor attaches its subscriber after SubscribeAlarms; window transitions bypass the feed, missed Acknowledge never repaired `Low` · `P2`
|
||||
|
||||
**Finding.** `RunMonitorAsync` invokes `SubscribeAlarmsAsync` and the first `ReconcileAsync` (`Alarms/GatewayAlarmMonitor.cs:213-214`) *before* the internal distributor subscriber is attached inside `_sessionManager.ReadAlarmEventsAsync` (`:232-234` → `Sessions/SessionManager.cs:195-208` → `GatewaySession.AttachInternalEventSubscriber`, `Sessions/GatewaySession.cs:553-562`). The distributor pump has been running since `MarkReady` started the dashboard mirror (`GatewaySession.cs:445-449`, `:621`), and a late subscriber misses previously fanned events by design (`SessionEventDistributor.cs:579-582`). Raise/Clear lost in the window are repaired by the next reconcile, but `ApplyReconcile` broadcasts only presence deltas (`GatewayAlarmMonitor.cs:514-553`): an Acknowledge that lands in the window updates `_alarms` silently on the snapshot replace (`:547-551`) with no `Broadcast`, so live `StreamAlarms` subscribers show the alarm unacked until it clears.
|
||||
|
||||
**Impact.** A one-shot window of a few hundred ms (two worker command round-trips) per monitor lifecycle in which alarm transitions reach only the dashboard mirror; a missed Acknowledge is never broadcast to alarm-feed subscribers. Same defect class as the fixed GWC-01, shrunk from permanent to transient.
|
||||
|
||||
**Design.** Two independent halves:
|
||||
|
||||
1. **Attach before subscribing.** The monitor already holds the `GatewaySession` instance returned by `OpenSessionAsync` (`:203-208`), so take the internal lease directly — `session.AttachInternalEventSubscriber()` — *before* `SubscribeAlarmsAsync`, and enumerate `lease.Reader.ReadAllAsync(...)` after reconcile. Transitions arriving during subscribe + reconcile simply buffer in the lease's bounded channel (`MxGateway:Events:QueueCapacity`, default 1024 — ample for a two-round-trip window); if it ever overflowed, the internal subscriber is merely disconnected (never faults the session, per the `isInternal` contract), the enumeration ends, and the monitor's supervisor loop restarts — acceptable and loud. Processing buffered transitions *after* `ApplyReconcile` is order-safe: `ApplyTransition` handles alarms already present in the reconciled snapshot. With the monitor calling the session directly, `ISessionManager.ReadAlarmEventsAsync` loses its only caller — remove it rather than leave dead public surface (mirrors the GWC-19 precedent). Rejected alternative: buffer inside `ReadAlarmEventsAsync` by splitting attach/enumerate on the manager — same effect, but keeps an indirection whose only job was to hide the session object the monitor already has.
|
||||
2. **Repair acked-state in reconcile.** In `ApplyReconcile`, for references present in both `_alarms` and the incoming snapshot, broadcast an `Acknowledge` feed transition when the state changed to acked (`existing.CurrentState != incoming.CurrentState && incoming.CurrentState == AlarmConditionState.ActiveAcked`), via the existing `TransitionFromSnapshot(incoming, AlarmTransitionKind.Acknowledge)`. This is defense-in-depth for any future missed window (worker restart, overflow disconnect), not just this one. Invariant note: `ApplyReconcile` already broadcasts snapshot-derived Raise/Clear transitions on the **alarm feed** (`AlarmFeedMessage`, the StreamAlarms/dashboard surface — see the method comment "broadcasting a synthetic transition for any alarm the live stream missed"); extending that established feed-level repair to acked-state deltas does not touch the gRPC `StreamEvents` path and therefore does not violate the "never synthesize events" rule, which governs `MxEvent` emission.
|
||||
|
||||
**Implementation.**
|
||||
- `Alarms/GatewayAlarmMonitor.cs` `RunMonitorAsync` (`:211-250`): after `OpenSessionAsync`/`_session` publication, `using IEventSubscriberLease alarmLease = session.AttachInternalEventSubscriber();` **before** `SubscribeAlarmsAsync(...)`; replace the `_sessionManager.ReadAlarmEventsAsync(...)` enumeration with `await foreach (MxEvent mxEvent in alarmLease.Reader.ReadAllAsync(linked.Token))`; update the `:228-231` comment to state the attach-before-subscribe ordering and why.
|
||||
- `Sessions/SessionManager.cs:195-208` and `ISessionManager`: remove `ReadAlarmEventsAsync` (re-verify zero remaining callers with a grep at implementation time; repoint any test that used it at `GatewaySession.AttachInternalEventSubscriber`).
|
||||
- `Alarms/GatewayAlarmMonitor.cs` `ApplyReconcile` (`:514-553`): before the snapshot replace, add the both-present acked-delta loop described above.
|
||||
- Docs, same commit: `docs/Sessions.md`'s note that the alarm monitor attaches as an internal distributor subscriber (added by the GWC-01 fix) — extend with "attached before SubscribeAlarms so no transition window is missed"; `gateway.md`'s alarm-monitor section if it describes the startup ordering.
|
||||
- Tests:
|
||||
- `GatewayAlarmMonitorAttachOrderTests` (new, or extend the existing alarm monitor tests in `Alarms/`): fake worker emits an `Acknowledge` (and a `Raise`) transition immediately after the `SubscribeAlarms` reply and before the first reconcile reply; assert a `StreamAlarms` subscriber observes both — the direct regression test for the window.
|
||||
- `ApplyReconcileBroadcastsAcknowledgeDelta` (unit): seed the cache with an `Active` alarm, reconcile with the same reference `ActiveAcked`; assert exactly one `Acknowledge` feed message and no Raise/Clear.
|
||||
- Existing alarm monitor + `SessionManagerTests` suites stay green after the `ReadAlarmEventsAsync` removal.
|
||||
|
||||
**Verification.** `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx`; `dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~GatewayAlarmMonitor"` and `--filter "FullyQualifiedName~SessionManagerTests"`. Depends on GWC-27 landing first (or together) so the earlier attach point runs against the readiness gate.
|
||||
|
||||
---
|
||||
|
||||
## GWC-27 — `AttachInternalEventSubscriber` bypasses the readiness gate; premature attach poisons the distributor permanently `Low` · `P2`
|
||||
|
||||
**Finding.** `GatewaySession.AttachInternalEventSubscriber` (`Sessions/GatewaySession.cs:553-562`) runs `EnsureDistributorCreated` → `Register(isInternal: true)` → `StartPumpIfRequested` with no state check, unlike `AttachEventSubscriber`, which requires `_state == Ready && _workerClient?.State == Ready` under `_syncRoot` (`:889-896`). `SessionManager.ReadAlarmEventsAsync` (`Sessions/SessionManager.cs:195-208`) exposes the unchecked path publicly. If invoked before Ready, the pump's source (`MapWorkerEventsAsync` → `GetReadyWorkerClientAsync`, `:740-749`, `:1910`) throws `SessionNotReady`; `PumpAsync` completes all subscribers with the error and latches `_completed` (`SessionEventDistributor.cs:603-612`, `:662-680`). `_eventDistributorStarted` is never reset (`GatewaySession.cs:527-531`) and the distributor is replaced only in `DisposeAsync`, so every later registration — including gRPC `StreamEvents` — receives an immediately-completed channel carrying the stale error: a Ready session with permanently dead event streaming. Unreachable today (the alarm monitor attaches only after `OpenSessionAsync` returns Ready), but one future call site away and failing in the worst way (silent, permanent, per-session).
|
||||
|
||||
**Impact.** Latent lifecycle hazard: any refactor or new caller that attaches internally before Ready silently kills event streaming for that session's whole lifetime, with no fault and no log pointing at the cause.
|
||||
|
||||
**Design.** Mirror `AttachEventSubscriber`'s readiness gate at the top of `AttachInternalEventSubscriber`: under `_syncRoot`, throw `SessionManagerException(SessionManagerErrorCode.SessionNotReady, …)` when `_state != SessionState.Ready || _workerClient?.State != WorkerClientState.Ready`, *before* `EnsureDistributorCreated` — so a premature attach fails loudly and never constructs/starts the distributor. Rejected alternative: make the distributor resettable after a pump fault — a much larger lifecycle change (replay ring, `_completed` latch, dashboard mirror lease all assume one distributor per session) that would also mask caller bugs instead of surfacing them. `StartDashboardMirror` keeps its direct `distributor.Register(isInternal: true)` path — it is called from `MarkReady` under its own state guard (`:604`) and is not a public entry point.
|
||||
|
||||
**Implementation.**
|
||||
- `Sessions/GatewaySession.cs:553-562`: add the `_syncRoot` readiness check (reuse the exact exception code and message shape from `AttachEventSubscriber:891-896`); then proceed with the existing `EnsureDistributorCreated`/`Register`/`StartPumpIfRequested` sequence outside the lock (matching `AttachEventSubscriber`'s lock discipline — the check is under the lock, the distributor calls are not). Extend the XML remarks to state the gate.
|
||||
- `Sessions/SessionManager.cs:195-208`: no separate check needed — the path inherits the gate via the session call, and GWC-26 removes the method entirely.
|
||||
- Tests (`Gateway/Sessions/GatewaySessionTests.cs`), new `AttachInternalEventSubscriberBeforeReadyThrowsAndDoesNotPoisonDistributor`: on a not-yet-Ready session, assert `SessionManagerException`/`SessionNotReady`; then drive the session to Ready and assert `AttachEventSubscriber` still yields a live stream (the key assertion: the distributor was never created/started by the failed attach).
|
||||
|
||||
**Verification.** `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx`; `dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~GatewaySessionTests"`. Land before (or with) GWC-26.
|
||||
|
||||
---
|
||||
|
||||
## GWC-28 — Gateway→worker envelope `sequence` stamped at creation, not at write `Low` · `P2`
|
||||
|
||||
**Finding.** `CreateEnvelope` stamps `Sequence = (ulong)Interlocked.Increment(ref _nextSequence)` at envelope construction (`Workers/WorkerClient.cs:1031-1044`); envelopes are then enqueued to `_outboundEnvelopes` (`EnqueueAsync:957-972`) and written by the single `WriteLoopAsync` (`:390-409`); `WorkerFrameWriter` does not re-stamp. Two concurrent `InvokeAsync` callers can stamp 1 and 2 but enqueue in the order 2, 1 — non-monotonic sequences on the pipe, violating `gateway.md`'s "monotonic per sender" contract. This is exactly what WRK-04 fixed on the worker side (stamp at the point of writing, under the writer's serialization); the gateway half was not given the same treatment. Benign today — neither side validates inbound sequence (GWC-10/IPC-10 open) — but it would start faulting sessions the moment worker-side validation is added.
|
||||
|
||||
**Impact.** Contract violation lying in wait: adding the specified sequence enforcement on the worker side (the mirror of GWC-10) would immediately fault healthy sessions under concurrent command load.
|
||||
|
||||
**Design.** Stamp at write, matching WRK-04: `WriteLoopAsync` assigns `envelope.Sequence` immediately before `_writer.WriteAsync`. The write loop is the channel's single consumer (`SingleReader = true`, `:74`), so writes are naturally serialized — the counter becomes a plain `ulong` incremented only on the write loop, and the `Interlocked` goes away. Every outbound envelope flows through `_outboundEnvelopes` (hello `:150`, commands `:223`, shutdown `:317`), so coverage is total. Rejected alternative: re-stamp inside `WorkerFrameWriter` — the frame writer is a pure framing layer shared with the `FakeWorkerHarness` and tests; giving it protocol state would leak envelope semantics into the wire layer on both sides.
|
||||
|
||||
**Implementation.**
|
||||
- `Workers/WorkerClient.cs:1031-1044` (`CreateEnvelope`): remove the `Sequence` assignment; add a comment that the sequence is stamped in `WriteLoopAsync` at the point of writing (cite WRK-04 parity and this finding).
|
||||
- `Workers/WorkerClient.cs:390-409` (`WriteLoopAsync`): before `_writer.WriteAsync(envelope, …)`, `envelope.Sequence = ++_nextSequence;` — change the `_nextSequence` field (`:37`) from `long` + `Interlocked` to a plain `ulong` touched only by the write loop, and update its usage accordingly.
|
||||
- Docs, same commit: the `gateway.md` envelope-sequence paragraph (the "monotonic per sender" contract, ~line 314) — note both sides stamp at write under their single write path; cross-reference open GWC-10 for the (still absent) inbound enforcement.
|
||||
- Tests (`Gateway/Workers/WorkerClientTests.cs`), new `ConcurrentInvokesEmitStrictlyIncreasingSequencesOnTheWire`: fake pipe; fire N (e.g. 32) parallel `InvokeAsync` calls; capture the frames the "worker" side receives; assert the `Sequence` values are strictly increasing in wire order. Existing handshake/shutdown tests confirm hello/shutdown envelopes still carry sequences.
|
||||
|
||||
**Verification.** `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx`; `dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~WorkerClientTests"` and `--filter "FullyQualifiedName~WorkerFrameProtocolTests"`. Gateway-side only; the worker's own WRK-04 stamping is untouched.
|
||||
|
||||
---
|
||||
|
||||
## GWC-29 — `Invoke` deep-clones the entire request only to discard the cloned command `Low` · `—`
|
||||
|
||||
**Finding.** `MxAccessGatewayService.Invoke` runs `MxCommandRequest invokeRequest = request.Clone(); invokeRequest.Command = commandToInvoke;` (`Grpc/MxAccessGatewayService.cs:119-120`) — a deep clone of the whole request *including its command payload* (potentially a large bulk-write graph), whose cloned command is immediately overwritten and discarded. `MapCommand` then performs the one clone actually needed (`request.Command.Clone()`, `Grpc/MxAccessGrpcMapper.cs:27-37`). Net cost: one full wasted command deep-clone per `Invoke` — the same cost class GWC-07/IPC-05 eliminated on the event and envelope paths.
|
||||
|
||||
**Impact.** Avoidable allocation and copy proportional to command size on every command, worst for exactly the bulk writes that are largest.
|
||||
|
||||
**Design.** Skip the request clone entirely: add a `MapCommand(MxCommand command)` overload that builds the `WorkerCommand` from the command alone (`Command = command.Clone()`, `EnqueueTimestamp` as today) — `MapCommand` reads nothing else from the request. `Invoke` passes `commandToInvoke` directly. Keep the existing `MapCommand(MxCommandRequest)` overload delegating to the new one so other callers and tests are untouched. The single remaining clone is still required and must stay: `commandToInvoke` may be the caller-owned `request.Command` (gRPC-owned graph) and is also read *after* the invoke by `session.TrackCommandReply(commandToInvoke, …)` (`MxAccessGatewayService.cs:132`), so ownership transfer (à la GWC-07) is not safe here — the clone in `MapCommand` is what keeps `WorkerClient.CreateCommandEnvelope`'s documented no-aliasing invariant (`Workers/WorkerClient.cs:1000-1004`) true. That comment stays accurate unchanged.
|
||||
|
||||
**Implementation.**
|
||||
- `Grpc/MxAccessGrpcMapper.cs`: add `public WorkerCommand MapCommand(MxCommand command)` (null-check, clone, timestamp); retarget `MapCommand(MxCommandRequest request)` to validate and delegate.
|
||||
- `Grpc/MxAccessGatewayService.cs:119-121`: delete the `request.Clone()` and the `invokeRequest` local; `WorkerCommand workerCommand = mapper.MapCommand(commandToInvoke);`.
|
||||
- Tests: `Gateway/Grpc/MxAccessGrpcMapperTests.cs` — new `MapCommandFromCommandClonesPayload` (mutating the returned `WorkerCommand.Command` does not affect the input command; `EnqueueTimestamp` populated; both overloads produce equal results). Existing `MxAccessGatewayServiceTests` and `MxAccessGatewayServiceConstraintTests` must stay green — the constraint path (`bulkConstraintPlan.Command` flowing through, denied-merge on the reply) is the behavior most worth re-running.
|
||||
- Docs: none (internal hot-path change; no contract or config surface).
|
||||
|
||||
**Verification.** `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx`; `dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~MxAccessGrpcMapperTests"` and `--filter "FullyQualifiedName~MxAccessGatewayService"`.
|
||||
|
||||
---
|
||||
|
||||
## GWC-30 — Frame reader allocates a fresh 4-byte length-prefix array per frame `Info` · `—`
|
||||
|
||||
**Finding.** `WorkerFrameReader.ReadAsync` allocates `new byte[sizeof(uint)]` per frame (`Workers/WorkerFrameReader.cs:33`); the GWC-08 pass pooled the payload buffer but left the prefix. Gen-0, 4 bytes — negligible; recorded so the next hot-path pass knows it was seen.
|
||||
|
||||
**Impact.** One tiny short-lived allocation per inbound frame. No functional risk.
|
||||
|
||||
**Design.** A reusable per-reader scratch field: `private readonly byte[] _lengthPrefix = new byte[sizeof(uint)];`. The reader is single-consumer by construction — exactly one read loop per `WorkerClient` calls `ReadAsync`, never concurrently (handshake reads happen before the read loop starts) — so a per-instance buffer is safe; document that `ReadAsync` is not reentrant. Rejected: pooling 4 bytes via `ArrayPool` — rent/return overhead exceeds the allocation it saves.
|
||||
|
||||
**Implementation.**
|
||||
- `Workers/WorkerFrameReader.cs`: add the `_lengthPrefix` field, use it at `:33-36`, and add a one-line class remark that `ReadAsync` must not be invoked concurrently (single-consumer contract).
|
||||
- Tests: existing `Gateway/Workers/WorkerFrameProtocolTests.cs` round-trips cover it; add (if absent) a sequential multi-frame read on one reader instance asserting all frames parse — guarding the shared-scratch reuse.
|
||||
- Docs: none.
|
||||
|
||||
**Verification.** `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx`; `dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~WorkerFrameProtocolTests"`.
|
||||
@@ -0,0 +1,539 @@
|
||||
# Worker Process — Remediation Design & Implementation (2026-07-12 cycle)
|
||||
|
||||
Source review: [20-worker.md](../20-worker.md) · Roadmap: [00-overall.md](../00-overall.md) · Generated: 2026-07-13 · Baseline: `main` @ `4f5371f`
|
||||
|
||||
This cycle's findings are concentrated in the new frame-write scheduler and the DrainEvents bound
|
||||
that the prior remediation added — the scheduler is fundamentally sound, and every fix below is an
|
||||
edge-hardening of it rather than a redesign. One finding is P0 (WRK-21, shared with IPC-23: a
|
||||
diagnostics command can still kill the session); the rest are Lows plus two residuals of prior Done
|
||||
items (WRK-25, WRK-26). All fixes preserve MXAccess parity (no MXAccess behavior changes — these
|
||||
are all IPC/liveness-layer changes), preserve the control-before-event write-priority guarantee,
|
||||
and use net48-safe constructs only (plain constructors and get-only properties, no init-only
|
||||
members, no positional records). The worker builds and tests only on the Windows x86 host
|
||||
(windev); verification commands assume that host.
|
||||
|
||||
## Finding register
|
||||
|
||||
| ID | Sev | Tier | Eff | Dep | Status | Title |
|
||||
|----|-----|------|-----|-----|--------|-------|
|
||||
| WRK-21 | Medium | P0 | M | IPC-23 (same defect, fix owned here); WRK-28 (same lines) | Not started | DrainEvents bound is count-based only; an oversized reply still kills the session and loses the drained events |
|
||||
| WRK-22 | Low | — | S | IPC-26 (same defect, fix owned here) | Not started | Cancelled `WriteAsync` leaves its frame queued; it is still written later |
|
||||
| WRK-23 | Low | — | S | WRK-21 (rejection path becomes backstop-only) | Not started | Rejected frames consume sequence numbers, producing wire gaps |
|
||||
| WRK-24 | Low | — | S | — | Not started | `AdoptNegotiatedMaxMessageBytes` has no lower-bound sanity check |
|
||||
| WRK-25 | Low | P2 | S | WRK-22 (both touch enqueue/dequeue) | Not started | WRK-12 flush coalescing never engages on the event hot path |
|
||||
| WRK-26 | Low | P1 | S | WRK-23 (soft — sequence prose); discharges IPC-29 | Not started | Write-priority and overflow doc drift from the WRK-07 change |
|
||||
| WRK-27 | Low | — | S | — | Not started | Alarm poll bypasses the watchdog's in-flight suppression (15 s vs 75 s) |
|
||||
| WRK-28 | Low | — | S | WRK-21 (land in the same commit cluster) | Not started | 10,000 drain cap is a duplicated magic constant with a comment-only sync contract |
|
||||
|
||||
---
|
||||
|
||||
## WRK-21 — DrainEvents bound is count-based only; an oversized reply still kills the session and loses the drained events `Medium` · `P0`
|
||||
|
||||
**Finding.** `CreateDrainEventsReply` clamps only the event *count* to `MaxDrainEventsPerReply`
|
||||
(10,000) (`src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs:545-557`); the reply is written
|
||||
via `WriteControlReplyAsync` (:481, :485-496) with no catch. A reply whose serialized size exceeds
|
||||
the negotiated `MaxMessageBytes` throws `WorkerFrameProtocolException(MessageTooLarge)` from
|
||||
`WorkerFrameWriter.WriteFrameAsync` (`Ipc/WorkerFrameWriter.cs:250-255`) — correctly a *per-frame*
|
||||
rejection at the writer, but `HandleControlCommandAsync`/`DispatchGatewayEnvelopeAsync` propagate
|
||||
it out of `RunMessageLoopAsync`, so the session dies and `WorkerApplication.cs:110-119` exits
|
||||
`ProtocolViolation` (6). The 10,000 drained events were already removed from `MxAccessEventQueue`
|
||||
by `runtimeSession.DrainEvents(maxEvents)` (:551) and are gone. 10,000 events at ~1.7 KiB each
|
||||
(large string/array `MxValue`s) exceed the default 16 MiB + 64 KiB frame max, so the comment's
|
||||
stated goal ("cannot pack the whole queue into one session-killing reply frame", :21-26) is not met
|
||||
for byte-heavy events. **This is the same defect as IPC-23; this entry owns the single fix** (the
|
||||
IPC remediation register should point here). Roadmap P0 item 2: byte-aware DrainEvents reply
|
||||
capping such that **no diagnostics command can be session-fatal**.
|
||||
|
||||
**Impact.** A `DrainEvents max_events=0` diagnostic against a byte-heavy queue — the array-heavy
|
||||
payloads this gateway exists for — terminates a healthy session (exit 6, gateway kills the worker)
|
||||
*and* silently destroys the drained events. The exact "session-killing reply frame" the prior fix
|
||||
(IPC-04/WRK slice) was added to prevent, narrowed but not closed.
|
||||
|
||||
**Design.** Two layers, both required by the P0 acceptance criterion:
|
||||
|
||||
1. **Primary — byte-aware drain that never removes an event it cannot ship.** The reply must be
|
||||
sized *while draining*, and the size decision must happen inside the queue's lock so events are
|
||||
only dequeued once they are known to fit. Add a byte-budgeted overload to `MxAccessEventQueue`:
|
||||
drain from the head while `count < maxEvents` **and** the accumulated serialized size plus the
|
||||
next event's size stays within a byte budget; an event that does not fit is left at the head for
|
||||
the next call. The budget is `_options.MaxMessageBytes` (the negotiated value, read at
|
||||
reply-build time) minus a fixed headroom constant covering the envelope/reply wrapper overhead
|
||||
(64 KiB — the same envelope-overhead reserve rationale `docs/WorkerFrameProtocol.md` already
|
||||
documents for the frame max itself). Per-event cost is `WorkerEvent.CalculateSize() + 8` — the
|
||||
`WorkerEvent` wrapper slightly overestimates the packed `MxEvent`, and +8 conservatively covers
|
||||
the repeated-field tag and length varint, so the estimate errs strictly on the safe side.
|
||||
Truncation is reported in the reply's existing `DiagnosticMessage` field ("N events returned,
|
||||
M remain; repeat DrainEvents for the rest"), so **no proto change** is needed.
|
||||
|
||||
Degenerate case: a single head event whose serialized size alone exceeds the budget. It is
|
||||
**not** drained (draining it would either build an oversized reply or lose it); the reply
|
||||
returns whatever fit before it (possibly zero events) with a `DiagnosticMessage` naming the
|
||||
blocked event's `WorkerSequence` so the operator can find the offending tag. Such an event
|
||||
cannot be shipped on the streaming path either — that policy question is IPC-30 and stays out
|
||||
of scope here; the WRK-21 guarantee is only that DrainEvents never dies and never loses events.
|
||||
|
||||
2. **Backstop — the control-reply write seam must not be session-fatal.** Even with capping (a
|
||||
future command, a sizing bug), catch `WorkerFrameProtocolException` with
|
||||
`ErrorCode == MessageTooLarge` at the two reply-write seams and answer the correlation with a
|
||||
small error reply instead of unwinding the session: `HandleControlCommandAsync`'s
|
||||
`WriteControlReplyAsync` call (:481) and `ProcessCommandAsync`'s reply write (:630-638), which
|
||||
today converts *any* write failure into `_state = Faulted` + `MxaccessCommandFailed` — i.e. an
|
||||
oversized STA command reply also faults the whole session. The error reply uses
|
||||
`ProtocolStatusCode.InvalidRequest` with an explanatory message ("reply exceeded the negotiated
|
||||
frame maximum; retry with a smaller request").
|
||||
|
||||
Rejected alternatives: (a) adding `truncated`/`remaining_count` fields to `DrainEventsReply` — a
|
||||
contracts change that regenerates and ripples through all five language clients for a diagnostic
|
||||
command; `DiagnosticMessage` carries the same information at zero contract cost. (b) Catch-only
|
||||
(no byte capping) — survives the session but still loses the already-drained events, failing the
|
||||
"no event loss" half. (c) Draining into the reply and measuring `reply.CalculateSize()`
|
||||
periodically — protobuf C# does not memoize sizes, making that O(n²), and events over the line
|
||||
would already be dequeued. (d) A new `ProtocolStatusCode.ReplyTooLarge` enum value — proto change
|
||||
for a should-be-unreachable backstop; rejected for the same ripple reason as (a).
|
||||
|
||||
Parity note: DrainEvents is a gateway-protocol diagnostic, not an MXAccess call — no MXAccess
|
||||
behavior changes. Control-before-event priority is untouched (the reply stays a Control frame).
|
||||
|
||||
**Implementation.**
|
||||
1. `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/WorkerEventDrainResult.cs` (new): sealed class, plain
|
||||
constructor + get-only properties (net48-safe): `IReadOnlyList<WorkerEvent> Events`,
|
||||
`bool TruncatedBySize`, `int RemainingCount`, `ulong OversizedHeadSequence` (0 = none).
|
||||
2. `MxAccess/MxAccessEventQueue.cs`: add
|
||||
`public WorkerEventDrainResult Drain(uint maxEvents, int maxTotalBytes)` — under `syncRoot`,
|
||||
loop `events.Peek()`, compute `workerEvent.CalculateSize() + 8`, dequeue while it fits the
|
||||
remaining budget and `drained.Count < maxEvents` (0 = existing "all" semantics for the count
|
||||
dimension); on a head event that alone exceeds `maxTotalBytes`, stop and record its
|
||||
`WorkerSequence` into `OversizedHeadSequence`; `RemainingCount = events.Count` on exit.
|
||||
3. `MxAccess/IWorkerRuntimeSession.cs`: add
|
||||
`WorkerEventDrainResult DrainEvents(uint maxEvents, int maxTotalBytes);` (no default interface
|
||||
method — the worker is net48, which has no DIM runtime support). Implement in
|
||||
`MxAccess/MxAccessStaSession.cs` (delegate to the queue) and in every test fake that implements
|
||||
the interface (the fake runtime session in `Worker.Tests/Ipc/WorkerPipeSessionTests.cs`).
|
||||
4. `Ipc/WorkerPipeSession.cs`:
|
||||
- add `private const int DrainReplyFrameHeadroomBytes = 64 * 1024;` next to
|
||||
`MaxDrainEventsPerReply` (:26) with a comment tying it to the envelope-overhead reserve;
|
||||
- `CreateDrainEventsReply` (:537-562): call the byte-budgeted overload with
|
||||
`_options.MaxMessageBytes - DrainReplyFrameHeadroomBytes`; when `TruncatedBySize`, set
|
||||
`reply.DiagnosticMessage` with returned/remaining counts and, when
|
||||
`OversizedHeadSequence != 0`, the blocked sequence;
|
||||
- `HandleControlCommandAsync` (:481): wrap `WriteControlReplyAsync(reply, ...)` in
|
||||
`try/catch (WorkerFrameProtocolException ex) when (ex.ErrorCode == WorkerFrameProtocolErrorCode.MessageTooLarge)`
|
||||
→ log `WorkerControlReplyTooLarge` (correlation id, kind, payload size from the exception
|
||||
message) and write a fallback `MxCommandReply` (same correlation id/kind,
|
||||
`ProtocolStatusCode.InvalidRequest`, explanatory message) via `WriteControlReplyAsync`;
|
||||
- `ProcessCommandAsync` (:630-638): put the same catch around the reply write specifically
|
||||
(inner try), writing the fallback error reply for the correlation instead of falling into the
|
||||
generic catch that sets `_state = WorkerState.Faulted`.
|
||||
5. Fold WRK-28 into this commit cluster (same lines — see below).
|
||||
6. Docs same commit: `docs/MxAccessWorkerInstanceDesign.md` — the control-command/DrainEvents
|
||||
prose gains the byte cap, the truncation `DiagnosticMessage` contract, and the
|
||||
oversized-head-event behavior; note in the Outbound Queues section that no control reply is
|
||||
session-fatal on size. `docs/WorkerFrameProtocol.md` — one sentence in the size-limits section:
|
||||
the session pre-sizes DrainEvents replies below the negotiated max, and `MessageTooLarge` on a
|
||||
reply write is answered with an error reply, not session teardown.
|
||||
|
||||
**Verification.** On windev:
|
||||
`dotnet build src/ZB.MOM.WW.MxGateway.Worker/ZB.MOM.WW.MxGateway.Worker.csproj -p:Platform=x86`
|
||||
then
|
||||
`dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86 --filter "FullyQualifiedName~WorkerPipeSessionTests|FullyQualifiedName~MxAccessEventQueueTests"`.
|
||||
New tests:
|
||||
- `WorkerPipeSessionTests.DrainEvents_ByteHeavyQueue_ReplyIsBoundedAndSessionSurvives` — the
|
||||
review's named repro: enqueue 10,000 events carrying large string values (≥ ~1.7 KiB each) into
|
||||
a real `MxAccessEventQueue` behind the fake runtime, send `DrainEvents max_events=0`; assert the
|
||||
reply envelope's serialized size ≤ `MaxMessageBytes`, `DiagnosticMessage` reports truncation,
|
||||
and the session still answers a subsequent `Ping` (no fault frame, `RunAsync` still running).
|
||||
- `WorkerPipeSessionTests.DrainEvents_RepeatedCalls_RecoverAllEventsWithoutLoss` — issue
|
||||
DrainEvents until empty; assert the union of replies is exactly the 10,000 enqueued events in
|
||||
order (split/bounded replies, zero loss).
|
||||
- `WorkerPipeSessionTests.ControlReplyTooLarge_WritesErrorReplyInsteadOfDying` — shrink the
|
||||
negotiated max via `WorkerFrameProtocolOptions` so the reply write throws `MessageTooLarge`;
|
||||
assert a fallback `InvalidRequest` reply with the same correlation id is written and the session
|
||||
survives.
|
||||
- `WorkerPipeSessionTests.CommandReplyTooLarge_WritesErrorReplyInsteadOfFaulting` — same via the
|
||||
STA command path (`ProcessCommandAsync`); assert no `WorkerFault`, `_state` stays `Ready`.
|
||||
- `MxAccessEventQueueTests.Drain_ByteBudget_StopsBeforeBudgetAndLeavesRemainderQueued` — order
|
||||
preserved, `RemainingCount` exact, undrained events still present.
|
||||
- `MxAccessEventQueueTests.Drain_ByteBudget_OversizedHead_DrainsNothingAndReportsHeadSequence`.
|
||||
|
||||
---
|
||||
|
||||
## WRK-22 — Cancelled `WriteAsync` leaves its frame queued; it is still written later `Low` · `—`
|
||||
|
||||
**Finding.** `WorkerFrameWriter.WriteAsync` enqueues the `PendingFrame` before the cancellable
|
||||
lock wait (`Ipc/WorkerFrameWriter.cs:87-103`); if `_writeLock.WaitAsync(cancellationToken)`
|
||||
throws, nothing removes the frame, so the next lock-holder writes it. Concrete scenario: at
|
||||
shutdown the event-drain loop's token is cancelled (`Ipc/WorkerPipeSession.cs:298-304`) while its
|
||||
event frame waits for the lock; the shutdown-ack write then drains queues — ack first (Control),
|
||||
then the orphaned event — so an event frame is emitted *after* `WorkerShutdownAck`, and a write
|
||||
reported as cancelled to its caller still reaches the wire. **Same defect as IPC-26** (the IPC
|
||||
review describes this writer); this entry owns the single fix.
|
||||
|
||||
**Impact.** Benign today (the gateway drops post-shutdown/unknown frames), but the "cancelled
|
||||
means not written" contract is silently violated, an unobserved completion lingers, and any future
|
||||
gateway-side strictness about frames after the ack turns this into a protocol violation.
|
||||
|
||||
**Design.** Tombstone on cancellation, claim on dequeue. `Queue<T>` has no random removal, so the
|
||||
standard fix is: the cancelled caller marks its frame cancelled under `_gate`; `DequeueNext` skips
|
||||
completed (cancelled) frames and atomically marks the frame it returns as *claimed*. A frame
|
||||
already claimed by a draining lock-holder is mid-write and can no longer be recalled — the
|
||||
documented contract becomes "cancellation guarantees the frame is not written *if it has not yet
|
||||
been claimed*; a claimed frame may still reach the wire." That residual window is unavoidable
|
||||
without blocking the canceller behind the very write it is abandoning (rejected: it inverts the
|
||||
point of cancellation). Rejected alternative: switching to a `LinkedList` for O(1) removal —
|
||||
tombstoning is simpler, allocation-free, and the queues are short-lived.
|
||||
|
||||
**Implementation.**
|
||||
1. `Ipc/WorkerFrameWriter.cs`, nested `PendingFrame`: add `public bool Claimed;` (plain field,
|
||||
mutated only under `_gate`).
|
||||
2. `WriteAsync(envelope, priority, ct)` (:100-113): wrap `await _writeLock.WaitAsync(cancellationToken)`
|
||||
in `try/catch (OperationCanceledException)` → `lock (_gate) { if (!frame.Claimed) { frame.Completion.TrySetCanceled(cancellationToken); } }`
|
||||
then rethrow. (If already claimed, the frame will be written; the caller still observes the
|
||||
cancellation — documented in the method's XML doc.)
|
||||
3. `DequeueNext` (:200-216): under `_gate`, loop-dequeue past frames whose
|
||||
`Completion.Task.IsCanceled`; set `Claimed = true` on the frame actually returned.
|
||||
`FailAllQueued`/`FailFrames` already use `TrySetException`, which no-ops on cancelled frames.
|
||||
4. Docs same commit: the "Write scheduling and sequencing" section of
|
||||
`docs/WorkerFrameProtocol.md` (created by WRK-26 — if WRK-26 has not landed yet, carry the
|
||||
cancellation paragraph in this commit and let WRK-26 absorb it) states the
|
||||
cancelled-means-not-written-unless-claimed contract.
|
||||
|
||||
**Verification.** On windev:
|
||||
`dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86 --filter "FullyQualifiedName~WorkerFrameProtocolTests"`.
|
||||
New tests:
|
||||
- `WorkerFrameProtocolTests.WriteAsync_CancelledWhileWaitingForLock_FrameIsNeverWritten` — block
|
||||
the stream (gated fake stream) so writer A holds `_writeLock` mid-write; enqueue an event write
|
||||
with a CTS; cancel; assert `OperationCanceledException`; unblock A; write a control frame;
|
||||
assert the captured wire bytes contain A's frame and the control frame only — the cancelled
|
||||
event envelope never appears.
|
||||
- `WorkerFrameProtocolTests.WriteAsync_CancelledEventFrame_DoesNotTrailShutdownAck` — the review's
|
||||
shutdown scenario: cancelled event frame queued, then a Control (ack-shaped) write drains;
|
||||
assert the event frame is skipped and the ack is the last frame on the wire.
|
||||
|
||||
---
|
||||
|
||||
## WRK-23 — Rejected frames consume sequence numbers, producing wire gaps `Low` · `—`
|
||||
|
||||
**Finding.** `WorkerFrameWriter.WriteFrameAsync` stamps `envelope.Sequence = unchecked(++_nextSequence)`
|
||||
(`Ipc/WorkerFrameWriter.cs:240`) *before* the empty-payload and `MessageTooLarge` checks
|
||||
(:243-255), so a per-frame rejection leaves a hole in the on-wire sequence. `gateway.md:328` calls
|
||||
`sequence` a monotonic *diagnostic* counter, so nothing breaks — but the existing test
|
||||
`WorkerFrameProtocolTests.WriteAsync_UnderConcurrentCalls_StampsGapFreeMonotonicSequence`
|
||||
documents a stronger gap-free expectation than the code guarantees once a rejection occurs.
|
||||
|
||||
**Impact.** Diagnostics only: an operator correlating wire captures sees phantom gaps after any
|
||||
rejection and may chase a lost-frame bug that does not exist; the test asserts a guarantee the
|
||||
code does not hold.
|
||||
|
||||
**Design.** Peek-stamp, validate, commit. The sequence value participates in `CalculateSize()`
|
||||
(varint width), so it cannot simply be stamped after the size checks — the checked size would not
|
||||
match the written payload. Instead compute a *candidate* (`_nextSequence + 1`), stamp it, run the
|
||||
size/empty checks against the stamped envelope, and only commit `_nextSequence = candidate`
|
||||
immediately before the stream write. On a per-frame rejection the counter is untouched, so the
|
||||
next accepted frame reuses the number and the wire stays contiguous. Safe because `_nextSequence`
|
||||
is only ever touched by the lock-holder (`:44-48`). The rejected envelope retains its tentative
|
||||
sequence — a caller-visible mutation on a failed envelope that already exists today. Rejected
|
||||
alternative: decrement on rejection (`--_nextSequence`) — equivalent effect but leaves a window
|
||||
where the counter was observably advanced; the candidate/commit form has no intermediate state.
|
||||
Dependency note: after WRK-21 lands, `MessageTooLarge` at the writer becomes a rare backstop, so
|
||||
this fix is what makes the existing gap-free test's guarantee genuinely true rather than
|
||||
usually-true.
|
||||
|
||||
**Implementation.**
|
||||
1. `Ipc/WorkerFrameWriter.cs`, `WriteFrameAsync` (:234-267): keep
|
||||
`WorkerEnvelopeValidator.Validate` first; replace the stamp with
|
||||
`ulong candidate = unchecked(_nextSequence + 1); envelope.Sequence = candidate;`; after the
|
||||
empty-payload and `MaxMessageBytes` checks pass, `_nextSequence = candidate;` immediately
|
||||
before the buffer build/`WriteTo`. Update the :45-48 comment.
|
||||
2. Docs same commit: none required beyond WRK-26's scheduling section, which should state
|
||||
"rejected frames do not consume sequence numbers" once this lands (see WRK-26 ordering note).
|
||||
|
||||
**Verification.** On windev:
|
||||
`dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86 --filter "FullyQualifiedName~WorkerFrameProtocolTests"`.
|
||||
New test:
|
||||
- `WorkerFrameProtocolTests.WriteAsync_PerFrameRejection_DoesNotConsumeSequence` — write an
|
||||
accepted frame, attempt an oversized frame (assert `MessageTooLarge` rejection), write another
|
||||
accepted frame; assert the two on-wire frames carry sequences 1 and 2.
|
||||
The existing `WriteAsync_UnderConcurrentCalls_StampsGapFreeMonotonicSequence` must still pass.
|
||||
|
||||
---
|
||||
|
||||
## WRK-24 — `AdoptNegotiatedMaxMessageBytes` has no lower-bound sanity check `Low` · `—`
|
||||
|
||||
**Finding.** `Ipc/WorkerFrameProtocolOptions.cs:124-140` — 0 keeps the default and >256 MiB
|
||||
throws, but any value from 1 byte up is adopted. A gateway bug or a foreign/old gateway sending
|
||||
e.g. 512 leaves the session alive but unable to write any frame (every write fails
|
||||
`MessageTooLarge`) and unable to read most inbound frames — failing confusingly mid-session rather
|
||||
than at the handshake, contradicting the class's own declared intent (:14-18, "a nonsensical
|
||||
negotiated value is rejected at the handshake"). The gateway's own validator floors its config at
|
||||
1024 (`src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs:10,230-233`).
|
||||
|
||||
**Impact.** A misconfigured or buggy peer produces a session that handshakes cleanly and then
|
||||
fails every subsequent operation with per-frame size errors — the worst diagnostic shape for an
|
||||
operator, and exactly what the top-end check was added to prevent.
|
||||
|
||||
**Design.** Mirror the gateway's validation floor: reject negotiated values below 1024 with the
|
||||
same `InvalidConfiguration` error the ceiling uses, so the failure is a handshake-time fault frame
|
||||
(`ValidateGatewayHello` → `CompleteStartupHandshakeAsync`'s `WorkerFrameProtocolException` catch →
|
||||
`TryWriteFaultAsync` + throw) instead of a mid-session mystery. Floor value: **1024**, matching
|
||||
`GatewayOptionsValidator.MinimumMaxMessageBytes` — the worker must never reject a value the
|
||||
gateway's own validator accepts as legal configuration. Rejected alternative: a 64 KiB floor
|
||||
(review's other suggestion) — safer-sounding but would turn a gateway legitimately configured
|
||||
anywhere in [1024, 65535] into a handshake failure; symmetry of the two processes' validation
|
||||
bounds is the defensible contract, and 1024 already guarantees hellos, heartbeats, acks, and
|
||||
faults fit.
|
||||
|
||||
**Implementation.**
|
||||
1. `Ipc/WorkerFrameProtocolOptions.cs`: add
|
||||
`public const int MinNegotiableFrameBytes = 1024;` (XML doc: matches the gateway's
|
||||
`MinimumMaxMessageBytes` validation floor); in `AdoptNegotiatedMaxMessageBytes`, after the
|
||||
`== 0` early return, throw `WorkerFrameProtocolException(InvalidConfiguration)` when
|
||||
`negotiatedMaxFrameBytes < MinNegotiableFrameBytes`, message symmetrical with the ceiling text.
|
||||
2. Docs same commit: `docs/WorkerFrameProtocol.md` negotiated-max paragraph (:25-30) — state the
|
||||
accepted range [1024, 256 MiB], 0 = keep default, out-of-range rejected at the handshake.
|
||||
|
||||
**Verification.** On windev:
|
||||
`dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86 --filter "FullyQualifiedName~WorkerFrameProtocolTests|FullyQualifiedName~WorkerPipeSessionTests"`.
|
||||
New tests:
|
||||
- `WorkerFrameProtocolTests.AdoptNegotiatedMaxMessageBytes_BelowFloor_ThrowsInvalidConfiguration`
|
||||
(512 → throws; 1024 → adopted; boundary pair).
|
||||
- `WorkerPipeSessionTests.Handshake_GatewayHelloWithTinyMaxFrameBytes_FaultsAtHandshake` — mirror
|
||||
of the existing >256 MiB handshake test: `GatewayHello.MaxFrameBytes = 512` → fault frame
|
||||
written, handshake throws, no message loop entered.
|
||||
|
||||
---
|
||||
|
||||
## WRK-25 — WRK-12 flush coalescing never engages on the event hot path `Low` · `P2`
|
||||
|
||||
**Finding.** The event drain loop awaits each event's `WriteAsync` — which completes only after
|
||||
that frame is written *and flushed* — before writing the next
|
||||
(`Ipc/WorkerPipeSession.cs:374-382`), so the writer's event queue holds at most one frame from the
|
||||
drain loop and every event still costs one flush. The batch flush path
|
||||
(`Ipc/WorkerFrameWriter.cs:120-124, 160-182`) only coalesces when independent producers happen to
|
||||
queue behind a blocked in-progress write. Residual of prior Done item WRK-12: the tracking-log
|
||||
claim "a burst of N events costs 1 flush not N" does not describe production behavior.
|
||||
|
||||
**Impact.** Performance only: under event bursts (the gateway's core load) each event pays a
|
||||
semaphore round-trip and a pipe flush; the shipped coalescing mechanism sits idle on the exact
|
||||
path it was built for.
|
||||
|
||||
**Design.** Give the drain loop a real batch entry point: `WriteBatchAsync` on the writer —
|
||||
enqueue every frame of the drained batch under one `_gate` acquisition (preserving intra-batch
|
||||
order), take the write lock once, drain (existing logic: control-first, one flush per batch),
|
||||
then await all completions. The existing single-drainer machinery then does exactly what WRK-12
|
||||
intended: N events, one flush. Guarantees preserved: control-before-event holds because the batch
|
||||
still lands in `_eventFrames` and any concurrently queued control frame is drained first by
|
||||
`DequeueNext`; intra-batch event order holds because the enqueue is atomic under `_gate` and the
|
||||
event queue is FIFO; the "written and flushed before completion" contract is unchanged. A
|
||||
per-frame rejection inside the batch (an oversized single event) surfaces from the awaited
|
||||
completions and terminates the session exactly as today — that policy is IPC-30's, deliberately
|
||||
not changed here. Rejected alternative (review's other option): fire N `WriteAsync` calls and
|
||||
`Task.WhenAll` — works (each call enqueues synchronously before its first await) but costs N
|
||||
semaphore acquisitions and N no-op drains for zero benefit over the explicit batch. Depends on
|
||||
WRK-22 landing first or together: both touch the enqueue/`DequeueNext` seam, and batch cancellation
|
||||
reuses WRK-22's tombstoning.
|
||||
|
||||
**Implementation.**
|
||||
1. `Ipc/WorkerFrameWriter.cs`: add
|
||||
`public async Task WriteBatchAsync(IReadOnlyList<WorkerEnvelope> envelopes, WorkerFrameWritePriority priority, CancellationToken cancellationToken = default)`
|
||||
— build a `PendingFrame[]`, enqueue all under one `lock (_gate)`, single
|
||||
`await _writeLock.WaitAsync(cancellationToken)` (on `OperationCanceledException`: tombstone all
|
||||
unclaimed frames per WRK-22, rethrow), `DrainQueuedFramesAsync()`, release, then await each
|
||||
completion in order (plain loop over the array — no LINQ needed, net48-safe either way).
|
||||
2. `Ipc/WorkerPipeSession.cs`, `RunEventDrainLoopAsync` (:374-382): replace the per-event awaited
|
||||
loop with building the envelope list for the drained batch and one
|
||||
`WriteBatchAsync(envelopes, WorkerFrameWritePriority.Event, cancellationToken)`. Keep the
|
||||
priority comment, updated.
|
||||
3. Docs same commit: `docs/WorkerFrameProtocol.md` scheduling section (WRK-26, which lands earlier
|
||||
as P1) — amend its flush-coalescing sentence to say the event drain loop submits drained
|
||||
batches through the batch entry point, so a burst of N events costs one flush.
|
||||
|
||||
**Verification.** On windev:
|
||||
`dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86 --filter "FullyQualifiedName~WorkerFrameProtocolTests|FullyQualifiedName~WorkerPipeSessionTests"`.
|
||||
New tests:
|
||||
- `WorkerFrameProtocolTests.WriteBatchAsync_FlushesOnceAndPreservesOrder` — flush-counting fake
|
||||
stream; N event envelopes → exactly one flush, wire order = batch order, sequences monotonic.
|
||||
- `WorkerFrameProtocolTests.WriteBatchAsync_ControlFrameQueuedDuringBatch_JumpsRemainingEvents` —
|
||||
control-before-event still holds mid-batch.
|
||||
- `WorkerPipeSessionTests.EventBurst_DrainLoopCoalescesFlushes` — fake runtime yields a 128-event
|
||||
batch; assert the capturing stream saw one flush for the batch (this is the assertion the WRK-12
|
||||
tracking claim needs to actually be true).
|
||||
|
||||
---
|
||||
|
||||
## WRK-26 — Write-priority and overflow doc drift from the WRK-07 change `Low` · `P1`
|
||||
|
||||
**Finding.** `docs/MxAccessWorkerInstanceDesign.md:607-613` still specifies the five-level write
|
||||
priority (faults > replies > shutdown acks > heartbeats > events) while the implementation is a
|
||||
two-class scheduler (Control FIFO, then Event — `Ipc/WorkerFrameWritePriority.cs:10-17`,
|
||||
`Ipc/WorkerFrameWriter.cs:200-216`); within Control a fault does not jump queued
|
||||
heartbeats/replies. `docs/WorkerFrameProtocol.md` describes the negotiated max (:25-30) but says
|
||||
nothing about priority classes, flush coalescing, write-time sequence stamping, or the per-frame
|
||||
vs stream-failure semantics. The adjacent overflow-policy text
|
||||
(`docs/MxAccessWorkerInstanceDesign.md:615-624`, "stop accepting new commands") is also stale
|
||||
against the implemented fail-fast (WRK-18's doc half). CLAUDE.md requires affected docs to change
|
||||
in the same commit as the source; this is the residual of Done item WRK-07. Roadmap P1 doc-drift
|
||||
batch (item 8); **discharges IPC-29**, which asks for the same `WorkerFrameProtocol.md` section.
|
||||
|
||||
**Impact.** The component design doc actively misleads: a maintainer adding a new frame kind will
|
||||
look for a five-level queue that does not exist, and the accepted FIFO-within-control decision is
|
||||
recorded nowhere, so it reads as a defect rather than a choice.
|
||||
|
||||
**Design.** Doc-only; make the docs describe the implemented scheduler and record the decision.
|
||||
No code change — the two-class scheduler is correct and accepted (the review's deep-dive found it
|
||||
deadlock-free and lost-wakeup-free; within-Control FIFO is bounded by the small control queue
|
||||
depth). Ordering note: this is P1 and will likely land before WRK-23/WRK-25 (P2/untierred), so the
|
||||
new text must describe HEAD truthfully at commit time — sequence numbers *are* consumed by rejected
|
||||
frames until WRK-23 lands, and event flushes are *not* yet coalesced on the drain path until
|
||||
WRK-25 lands; write those two facts as they are, and WRK-23/WRK-25 update the sentences in their
|
||||
own commits (same-commit docs rule applies to them too).
|
||||
|
||||
**Implementation.**
|
||||
1. `docs/MxAccessWorkerInstanceDesign.md:602-624`: rewrite the "Outbound Queues" section —
|
||||
two-class cooperative scheduler (`WorkerFrameWriter` + `WorkerFrameWritePriority`): Control
|
||||
(faults, command replies, shutdown acks, heartbeats, hello/ready) FIFO among themselves and
|
||||
always ahead of Event frames; enqueue-then-contend, lock-holder drains all queued frames;
|
||||
record the accepted decision that the original five-level order was collapsed and why (control
|
||||
queue is shallow, so intra-control FIFO delay is bounded and the machinery stays simple).
|
||||
Rewrite the overflow paragraph to the implemented fail-fast (fault written, drain loop throws,
|
||||
process exits — currently the generic exit code; WRK-18 still tracks the dedicated code).
|
||||
2. `docs/WorkerFrameProtocol.md`: add a "Write scheduling and sequencing" section — priority
|
||||
classes; `Sequence` stamped at the moment of writing under the write lock so wire order and
|
||||
stamped sequence always agree; per-frame rejections (`InvalidEnvelope`, `MessageTooLarge`,
|
||||
version/session mismatch) fail only that frame while stream failures fail the batch and all
|
||||
queued frames; flush coalescing across a drained batch and the "written and flushed" completion
|
||||
contract; the rejection/sequence-gap behavior as it stands at commit time (see ordering note);
|
||||
cancellation semantics once WRK-22 lands (or carried by WRK-22's commit if it lands later).
|
||||
3. Cross-check `gateway.md:328-330` (sequence prose) still reads true; touch only if wrong.
|
||||
|
||||
**Verification.** Doc-only — no build impact. Sanity on any host:
|
||||
`grep -n "faults" docs/MxAccessWorkerInstanceDesign.md` shows no remaining five-level list;
|
||||
`grep -n "scheduling" docs/WorkerFrameProtocol.md` finds the new section. Follow
|
||||
`docs/style-guides/StyleGuide.md` (present tense, explain why). Confirm the IPC remediation
|
||||
register marks IPC-29 as discharged by this item.
|
||||
|
||||
---
|
||||
|
||||
## WRK-27 — Alarm poll bypasses the watchdog's in-flight suppression (15 s vs 75 s) `Low` · `—`
|
||||
|
||||
**Finding.** `MxAccess/MxAccessStaSession.cs:245-253` invokes `handler.PollOnce()` via
|
||||
`staRuntime.InvokeAsync` directly, bypassing `StaCommandDispatcher`; `CaptureHeartbeat` takes
|
||||
`CurrentCommandCorrelationId` only from the dispatcher (:364-381), so during a long `PollOnce` the
|
||||
heartbeat shows no in-flight command and the watchdog fires `StaHung` as soon as staleness exceeds
|
||||
`HeartbeatGrace` (15 s default) rather than the 75 s `HeartbeatStuckCeiling` granted to dispatched
|
||||
commands (`Ipc/WorkerPipeSession.cs:843-861`).
|
||||
|
||||
**Impact.** A healthy-but-slow `GetXmlCurrentAlarms2` (large alarm set, busy provider) blocking
|
||||
the STA for >15 s faults a healthy session. Partially defensible — a blocked STA cannot deliver
|
||||
data events either — but the 15 s vs 75 s asymmetry between polls and commands is unintentional.
|
||||
|
||||
**Design.** Surface poll-in-progress to the watchdog through the heartbeat snapshot as a boolean,
|
||||
and have the suppression branch honor it alongside `CurrentCommandCorrelationId`. The poll then
|
||||
gets exactly the command treatment: suppressed up to the ceiling, faulted past it (a poll that
|
||||
blocks the STA >75 s without pumping *should* fault — that is the ceiling's contract). All types
|
||||
touched are worker-internal; the heartbeat proto is unchanged, so no contract/client work.
|
||||
Rejected alternatives: (a) routing the poll through `StaCommandDispatcher` — inflates
|
||||
`PendingCommandCount`, subjects polls to the 128-entry command backpressure and
|
||||
shutdown-rejection semantics, and perturbs dispatch ordering for real gateway commands; the poll
|
||||
is infrastructure, not a command. (b) A synthetic correlation id (e.g. `"alarm-poll"`) in the
|
||||
heartbeat — visible to the gateway, which treats correlation ids as real command identifiers
|
||||
(dashboards, per-command timeout logic); misrepresenting a nonexistent command crosses an
|
||||
observability contract for a purely local suppression need.
|
||||
|
||||
**Implementation.**
|
||||
1. `MxAccess/MxAccessStaSession.cs`: add `private volatile bool staAlarmPollInProgress;`; in the
|
||||
delegate passed to `staRuntime.InvokeAsync` in `RunAlarmPollLoopAsync` (:247-253), set it
|
||||
`true` immediately before `EnsureOnAlarmConsumerThread()`/`handler.PollOnce()` and clear it in
|
||||
a `finally` — the flag flips on the STA thread exactly around the COM call.
|
||||
2. `MxAccess/WorkerRuntimeHeartbeatSnapshot.cs`: add a `bool StaCallInProgress` get-only property
|
||||
and constructor parameter (plain ctor assignment — net48-safe; update all construction sites,
|
||||
including tests). Named generically ("an STA call outside the dispatcher is executing") so any
|
||||
future non-dispatcher STA work reuses it.
|
||||
3. `MxAccess/MxAccessStaSession.CaptureHeartbeat` (:364-381): pass `staAlarmPollInProgress`.
|
||||
4. `Ipc/WorkerPipeSession.ReportWatchdogFaultIfNeededAsync` (:850-861): change the suppression
|
||||
condition to
|
||||
`(!string.IsNullOrEmpty(snapshot.CurrentCommandCorrelationId) || snapshot.StaCallInProgress) && staleFor <= _sessionOptions.HeartbeatStuckCeiling`;
|
||||
extend the branch comment.
|
||||
5. Docs same commit: `docs/MxAccessWorkerInstanceDesign.md` watchdog section (~:656-702) — one
|
||||
paragraph: the alarm poll runs outside the dispatcher, advertises itself via the snapshot's
|
||||
STA-call-in-progress flag, and receives the same grace-to-ceiling suppression as dispatched
|
||||
commands.
|
||||
|
||||
**Verification.** On windev:
|
||||
`dotnet build src/ZB.MOM.WW.MxGateway.Worker/ZB.MOM.WW.MxGateway.Worker.csproj -p:Platform=x86`
|
||||
then
|
||||
`dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86 --filter "FullyQualifiedName~WorkerPipeSessionTests|FullyQualifiedName~MxAccessStaSessionTests"`.
|
||||
New tests:
|
||||
- `WorkerPipeSessionTests.Watchdog_StaCallInProgress_SuppressedUntilCeiling` — fake runtime
|
||||
returns a snapshot with activity stale beyond `HeartbeatGrace` but within the ceiling, empty
|
||||
correlation id, `StaCallInProgress = true`; assert no `StaHung` fault; then a snapshot stale
|
||||
beyond the ceiling; assert the fault fires (parity with the existing command-suppression tests).
|
||||
- `MxAccessStaSessionTests.CaptureHeartbeat_DuringAlarmPoll_ReportsStaCallInProgress` — block
|
||||
`PollOnce` on a gate in a stub alarm handler, capture a heartbeat mid-poll, assert the flag;
|
||||
release, assert it clears.
|
||||
|
||||
---
|
||||
|
||||
## WRK-28 — 10,000 drain cap is a duplicated magic constant with a comment-only sync contract `Low` · `—`
|
||||
|
||||
**Finding.** `Ipc/WorkerPipeSession.cs:26` (`MaxDrainEventsPerReply = 10_000`, "Kept in step with
|
||||
that public ceiling") vs `src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGrpcRequestValidator.cs:13`
|
||||
(`MaxDrainEventsPerRequest = 10_000`). Both projects already reference the Contracts assembly
|
||||
(`net10.0;net48`), which could own the constant.
|
||||
|
||||
**Impact.** Conventions/maintenance: nothing enforces the sync; the next person to tune one side
|
||||
silently desynchronizes the gateway's loud rejection threshold from the worker's clamp.
|
||||
|
||||
**Design.** Move the ceiling into the Contracts project next to the existing protocol constants in
|
||||
`GatewayContractInfo` (`src/ZB.MOM.WW.MxGateway.Contracts/GatewayContractInfo.cs`) and reference
|
||||
it from both sides. A C# `const` only — **no `.proto` change**, so no generated-code regeneration
|
||||
and no language-client impact (the cap is documented behavior, not wire schema). Land in the same
|
||||
commit cluster as WRK-21, which rewrites the exact worker lines that consume it. Rejected
|
||||
alternative: leaving two constants with a unit test asserting equality — works, but the shared
|
||||
constant removes the failure mode instead of detecting it.
|
||||
|
||||
**Implementation.**
|
||||
1. `src/ZB.MOM.WW.MxGateway.Contracts/GatewayContractInfo.cs`: add
|
||||
`public const uint MaxDrainEventsPerCommand = 10_000;` with XML doc naming both consumers (the
|
||||
gateway's request-validation ceiling and the worker's per-reply clamp) and the byte-cap caveat
|
||||
from WRK-21 (count is necessary, not sufficient).
|
||||
2. `src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGrpcRequestValidator.cs:13,81-85`: delete the
|
||||
private const; reference `GatewayContractInfo.MaxDrainEventsPerCommand` (error text unchanged).
|
||||
3. `src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs:26,545-557`: delete
|
||||
`MaxDrainEventsPerReply`; reference the shared constant; trim the "kept in step" comment to
|
||||
point at the shared home.
|
||||
4. Docs same commit: none required (no behavior change); if `docs/GatewayConfiguration.md` or the
|
||||
DrainEvents prose names the literal 10,000, point it at the shared constant's home.
|
||||
|
||||
**Verification.** Both sides build: on macOS
|
||||
`dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx` and
|
||||
`dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~MxAccessGrpcRequestValidator"`;
|
||||
on windev
|
||||
`dotnet build src/ZB.MOM.WW.MxGateway.Worker/ZB.MOM.WW.MxGateway.Worker.csproj -p:Platform=x86`
|
||||
and
|
||||
`dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86 --filter "FullyQualifiedName~WorkerPipeSessionTests"`.
|
||||
Existing drain-bound tests on both sides are the regression guard; no new tests needed beyond
|
||||
WRK-21's.
|
||||
|
||||
---
|
||||
|
||||
## Cross-references and sequencing
|
||||
|
||||
- **WRK-21 = IPC-23** (single defect, single fix — owned here; the IPC register should link to
|
||||
this entry). Its `ProcessCommandAsync` backstop narrows command replies only; the oversized
|
||||
*event-stream* frame policy remains **IPC-30** (out of scope here — a delivery-expectations
|
||||
decision, not a mechanics fix).
|
||||
- **WRK-22 = IPC-26** (single defect, single fix — owned here). WRK-25 builds on WRK-22's
|
||||
claim/tombstone machinery — land WRK-22 first or together.
|
||||
- **WRK-23** interacts with WRK-21: byte capping makes writer-level `MessageTooLarge` a rare
|
||||
backstop, and WRK-23 then makes the gap-free sequence guarantee (already asserted by an existing
|
||||
test) actually hold across that backstop.
|
||||
- **WRK-26** (P1) lands before the P2/untierred code items — its text must describe HEAD at commit
|
||||
time; WRK-23 and WRK-25 each update the affected sentences in their own commits per the
|
||||
same-commit docs rule. WRK-26 also discharges **IPC-29**.
|
||||
- Suggested commit clusters: (1) WRK-21 + WRK-28 [P0]; (2) WRK-26 [P1 doc batch, with TST-27 /
|
||||
CLI-42 per roadmap item 8]; (3) WRK-22 + WRK-25 [writer seam]; (4) WRK-23; (5) WRK-24; (6)
|
||||
WRK-27. Run the full worker suite on windev once per cluster batch, not per task
|
||||
(`dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86`).
|
||||
- Prior-cycle open items touching the same code (WRK-02/03/05/08/09/10/13/14/16/17/18/19 —
|
||||
confirmed still open by this review) remain tracked in `archreview/remediation/20-worker.md`;
|
||||
WRK-26 deliberately folds in WRK-18's stale overflow *prose* but leaves its exit-code half open.
|
||||
@@ -0,0 +1,253 @@
|
||||
# Contracts & IPC Protocol — Remediation Design & Implementation (2026-07-12 cycle)
|
||||
|
||||
Source review: [../30-contracts-ipc.md](../30-contracts-ipc.md) (main @ `4f5371f`) · Prior cycle's plan: [../../remediation/30-contracts-ipc.md](../../remediation/30-contracts-ipc.md) · Roadmap: [../00-overall.md](../00-overall.md) · Generated: 2026-07-13
|
||||
|
||||
This cycle's findings cluster in two seams left by the prior remediation: **residual byte-size failure modes one layer out from the shipped fixes** (IPC-23/IPC-30 — the DrainEvents count cap and the per-frame-rejection machinery both stop short of the invariant "no single oversized payload may kill a session silently"), and **holes in the codegen freshness net** (IPC-24/IPC-25 — the CI Java churn-revert masks exactly the files where message-level drift lands, and the Go/Python worker bindings are demonstrably stale at HEAD with no guard). The rest is documentation drift and test-coverage blind spots.
|
||||
|
||||
Two findings in this domain share a defect with the worker domain and split ownership the same way the review does: **IPC-23 ≡ WRK-21** and **IPC-26 ≡ WRK-22**. For those, this plan defines the *contract-level requirements and documentation* and defers the worker-code mechanics to the worker plan; the worker plan's fix must satisfy the requirements stated here. IPC-30 (oversized *event* frame) is designed and implemented fully in this plan because the decision is a protocol-policy call, but its code lands in the same worker file and the same P0 batch as WRK-21, sharing one windev verification run.
|
||||
|
||||
All `path:line` citations were re-verified against the working tree at `4f5371f`.
|
||||
|
||||
## Finding register
|
||||
|
||||
| ID | Sev | Tier | Eff | Dep | Status | Title |
|
||||
|----|-----|------|-----|-----|--------|-------|
|
||||
| IPC-23 | Medium | P0 | S¹ | WRK-21 | Not started | DrainEvents bound is count-based only; byte-heavy queue still builds a session-killing reply frame (contract requirements here; fix mechanics in WRK-21) |
|
||||
| IPC-24 | Medium | P0 | S | — | Not started | CI's unconditional Java churn-revert masks real generated-code drift for message-level proto changes |
|
||||
| IPC-25 | Medium | P0 | M | — | Not started | Committed Go/Python worker bindings are stale at HEAD; no guard covers them |
|
||||
| IPC-26 | Low | P2 | S¹ | WRK-22 | Not started | Cancelled write leaves a ghost frame that is still written (contract requirement here; fix mechanics in WRK-22) |
|
||||
| IPC-27 | Low | P2 | S | — | Not started | Descriptor freshness test blind to enums, enum values, services/methods, and the Galaxy contract |
|
||||
| IPC-28 | Low | — | S | — | Not started | `docs/Grpc.md` omits the `CommandTooLarge` → `ResourceExhausted` mapping |
|
||||
| IPC-29 | Low | — | S | — | Not started | Worker writer priority scheduling and write-time sequence stamping undocumented in the frame-protocol doc |
|
||||
| IPC-30 | Low | P0 | M | WRK-21 (same file/batch) | Not started | Oversized worker→gateway event frame is session-fatal — make the death deliberate, structured, and diagnosable |
|
||||
| IPC-31 | Info | — | — | — | N/A | Gateway stamps sequence at creation, worker at write — accepted divergence; sequence is documented diagnostic-only (`gateway.md:328-330`); revisit only if sequence ever becomes load-bearing |
|
||||
| IPC-32 | Info | — | S | IPC-25 | Not started | `check-codegen.ps1` check labels miscounted (folded into the IPC-25 script edit) |
|
||||
|
||||
¹ Effort for the work owned by *this* plan (proto comments + docs + acceptance criteria). The code mechanics are M and are tracked under WRK-21 / WRK-22 in the worker plan.
|
||||
|
||||
---
|
||||
|
||||
## IPC-23 — DrainEvents bound is count-based only; a byte-heavy queue still builds a session-killing reply frame `Medium` · `P0` · fix mechanics in **WRK-21**
|
||||
|
||||
**Finding.** `Worker/Ipc/WorkerPipeSession.cs:537-562` (`CreateDrainEventsReply`) clamps the drain to `MaxDrainEventsPerReply = 10_000` (`:26`) but never sizes the reply. At the default negotiated frame max (16 MiB + 64 KiB), events averaging above ~1.7 KiB overshoot `MaxMessageBytes`; the writer correctly rejects the frame per-frame (`Worker/Ipc/WorkerFrameWriter.cs:250-255`, `IsPerFrameRejection` `:192-198`) so the stream survives, but the exception propagates from `WriteControlReplyAsync` (`:481,485-496`) through `DispatchGatewayEnvelopeAsync` (`:406`) into `RunMessageLoopAsync` (`:280`) and out of `RunAsync` — the worker exits, and because `runtimeSession.DrainEvents(maxEvents)` already removed the events from the queue (`:551`), they are lost. This is the prior IPC-04 fix narrowed, not closed.
|
||||
|
||||
**Impact.** A diagnostics command (`DrainEvents`, `max_events = 0` or any large value) against an array-heavy queue — exactly the payload profile this gateway exists for — kills the session and destroys the drained events. Violates the invariant the roadmap states for this P0 item: *no diagnostics command may be session-fatal*.
|
||||
|
||||
**Design (contract-level requirements — the WRK-21 implementation must satisfy all of these).**
|
||||
|
||||
1. **R1 — Reply fits the negotiated frame.** Every worker→gateway control reply, `DrainEventsReply` included, must serialize (as a full `WorkerEnvelope`) within the negotiated `MaxMessageBytes`. Concretely: `CreateDrainEventsReply` must stop packing when the running `reply.CalculateSize()` plus the next event's serialized size plus a fixed envelope-overhead allowance would exceed the cap. The byte cap binds *in addition to* the existing `MaxDrainEventsPerReply` count cap, whichever is hit first.
|
||||
2. **R2 — No event loss on truncation.** Events that do not fit the current reply must remain in (or be returned to) the event queue in order — the drain must be incremental (peek-pack-commit or drain-one-at-a-time), not bulk-drain-then-pack. Losing undelivered events would silently break the event stream's faithfulness.
|
||||
3. **R3 — Truncation is expressible with the existing contract; no new field.** `DrainEventsReply` (`mxaccess_gateway.proto:678-680`) already permits returning fewer events than requested or available; the caller contract is *drain iteratively until an empty reply*. An additive `uint32 remaining`/`bool truncated` field was considered and **rejected for this pass**: it forces a full five-client regeneration fan-out (plus descriptor + `Generated/` + Rust vendored copies) for a diagnostics nicety the loop semantics already provide. Revisit only if operators need one-shot depth reporting — `WorkerHeartbeat.outbound_event_queue_depth` (`mxaccess_worker.proto:99`) already reports depth out-of-band.
|
||||
4. **R4 — Byte-cap semantics are documented at the contract.** Proto comments and `docs/WorkerFrameProtocol.md` must state the rule (see Implementation).
|
||||
|
||||
**Implementation.** (Ordered; steps 1–2 are this plan's deliverable and should land in the *same commit* as the WRK-21 code fix so the proto-comment regeneration wave happens exactly once.)
|
||||
|
||||
1. `src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_worker.proto`: extend the `GatewayHello.max_frame_bytes` comment (`:45-50`) with one sentence: *every worker→gateway frame — events, heartbeats, faults, and control replies including DrainEvents — must serialize within this limit; reply builders truncate to fit rather than emit an oversized frame.*
|
||||
2. `src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_gateway.proto` (`DrainEventsReply`, `:678-680`): add a comment: *the reply is bounded by both a server-side count cap and the negotiated worker-frame byte cap; a reply may therefore carry fewer events than `max_events` and fewer than are queued. Callers drain iteratively until an empty reply.*
|
||||
3. Comment-only proto edits still trigger the full regen chain (comments surface in generated C# XML docs, Go doc comments, Java javadoc, and the Rust vendored copies are byte-compared by `check-codegen.ps1` Check 3):
|
||||
- regenerate `Contracts/Generated/` (`dotnet build src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj`, after `rm src/ZB.MOM.WW.MxGateway.Contracts/Generated/*.cs`) and **commit** (net48 CS0246 rule);
|
||||
- refresh `clients/rust/protos/*.proto` (byte-copy from Contracts/Protos);
|
||||
- regenerate Go (`pwsh clients/go/generate-proto.ps1`, pinned tools per IPC-25), Java (`gradle generateProto` on the pinned toolchain), Python (`pwsh clients/python/generate-proto.ps1` — comment-only edits normally produce *no* Python delta since `_pb2` embeds a source-info-free descriptor; commit only a real delta, revert noise);
|
||||
- regenerate the descriptor set (`pwsh scripts/publish-client-proto-inputs.ps1` with pinned protoc 34.1) so grpcurl users see the new comments.
|
||||
4. `docs/WorkerFrameProtocol.md`: add the byte-cap rule to the size-limits paragraph (`:19-30`): control replies are size-aware-packed; `DrainEvents` truncates to fit; drain-until-empty caller contract. (This lands in the same section IPC-29 adds — do the two doc edits together.)
|
||||
5. `docs/Grpc.md` DrainEvents rules row (`:180` area): note the reply may be truncated by byte size as well as count; drain iteratively.
|
||||
6. Worker code fix + tests: **deferred to WRK-21** (size-aware packing loop in `CreateDrainEventsReply`, incremental drain to satisfy R2, worker test that a queue of ~10,000 large events drains over multiple replies with the session alive and no event lost).
|
||||
|
||||
**Verification.**
|
||||
- Contract side (this plan): `pwsh scripts/check-codegen.ps1` passes after the regen wave (all checks); `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx`; `dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter FullyQualifiedName~ClientProtoInputTests`.
|
||||
- Mechanics (WRK-21, on windev via the isolated `C:\build` worktree): `dotnet build src/ZB.MOM.WW.MxGateway.Worker/ZB.MOM.WW.MxGateway.Worker.csproj -p:Platform=x86` and `dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86 --filter FullyQualifiedName~WorkerPipeSession` — including the WRK-21 repro test (enqueue ~10,000 large events, `DrainEvents max_events=0`, assert session survives and total drained count equals enqueued count across iterative replies).
|
||||
|
||||
---
|
||||
|
||||
## IPC-24 — CI's unconditional churn-revert step masks real Java generated-code drift for message-level proto changes `Medium` · `P0`
|
||||
|
||||
**Finding.** `.gitea/workflows/ci.yml:114-119` runs `git checkout -- clients/java/.../MxaccessGateway.java` and `.../MxaccessWorker.java` unconditionally before the "Verify generated tree is clean" step (`:120-121`). The Java bindings are single-file aggregates (`java_multiple_files` unset), so any message/field-level proto change lands **only** in those two files — a `.proto` edit committed without regenerating the Java client passes CI because the freshly regenerated (correct) files are reverted to the stale committed ones before the diff check. Only service-level changes (which touch the separate `*Grpc.java` stubs) would still be caught. The step's own comment scopes it to "when no `.proto` changed", but nothing enforces that condition.
|
||||
|
||||
**Impact.** The `checkGeneratedClean` intent from IPC-09 is neutered for the most common contract-change class. Combined with IPC-25, message-level drift currently has **zero** automated coverage in three of five clients.
|
||||
|
||||
**Design.** The churn the revert step works around is protobuf-gencode-version drift (`validateProtobufGencodeVersion(... major/minor/patch ...)` stanzas rewritten when the regenerating toolchain differs from the one that produced the committed files — repo memory `project_java_generated_churn`, ~64k lines). But the toolchain is now fully pinned: `clients/java/build.gradle:8,11` pin `grpcVersion = 1.76.0` / `protobufVersion = 4.33.1`, the protoc artifact and grpc plugin derive from those pins (`zb-mom-ww-mxgateway-client/build.gradle:40-46`), and the committed `MxaccessWorker.java` already stamps gencode 4.33.1. Under matching pins, regeneration should be **byte-identical** — meaning the revert step is a fossil from the pre-pin era, and the correct fix is to *delete it* and let `git diff --exit-code` be the real gate.
|
||||
|
||||
Discriminator design, in preference order:
|
||||
|
||||
1. **Preferred — prove the churn is gone under the pin and delete the revert.** Empirically regenerate on the pinned toolchain and check the tree. If clean, remove the revert step outright. Simplest, no new machinery, converts the Java job into a true drift gate.
|
||||
2. **Fallback (only if churn persists) — diff modulo the known volatile lines.** Replace the unconditional checkout with a script step: after regeneration, take `git diff -U0` over the two aggregates, strip additions/deletions matching the volatile patterns (`validateProtobufGencodeVersion`, the `/* major= */ … /* patch= */` literals, generator-version header comments); empty residue → churn → revert; non-empty residue → **fail** with "Java generated bindings are stale — regenerate with the pinned toolchain and commit". Put the script in `scripts/check-java-generated.sh` so local use matches CI.
|
||||
3. **Rejected — gate the revert on the push/PR diff containing no `*.proto` change** (`git diff --name-only $BASE..HEAD -- '*.proto'`). Two defects: the base-ref plumbing differs per trigger (push `github.event.before` vs PR base vs the nightly `schedule` run, which has no meaningful base), and it discriminates on the *delta*, not the *tree* — a stale binding merged in an earlier commit passes every subsequent run forever. The tree-level checks above are strictly stronger.
|
||||
|
||||
**Implementation.**
|
||||
1. Prove/disprove churn locally (the Java client builds on the Mac now — memory `project_java_build_host`): `cd clients/java && JAVA_HOME=/opt/homebrew/opt/openjdk@17 gradle generateProto && git status --porcelain -- src/main/generated`. Expected: clean (pins match the committed gencode stamps).
|
||||
2. If clean: delete the "Revert spurious protobuf-version churn" step (`.gitea/workflows/ci.yml:114-119`), keep `Verify generated tree is clean` (`:120-121`) as-is. Also delete the churn-revert comment at `:92-95`.
|
||||
3. If churn persists: add `scripts/check-java-generated.sh` implementing design 2 and replace `ci.yml:114-119` with a step invoking it.
|
||||
4. Same-commit doc updates (docs-change-with-source rule): `docs/GatewayTesting.md:421-425` (CI section prose describing the revert), `docs/ClientProtoGeneration.md:96-100` (the "revert that one file when no `.proto` changed" caveat), and the `checkGeneratedClean` comment block in `clients/java/zb-mom-ww-mxgateway-client/build.gradle:61-70` (the "CI reverts that one file" caveat). Update the repo memory note `project_java_generated_churn` guidance if step 1 shows the churn is gone.
|
||||
5. Negative-path proof on a throwaway branch: add a scratch field to `mxaccess_worker.proto`, regenerate *only* `Contracts/Generated/` (not Java), push, and confirm the `java` CI job now **fails** at the diff step. Delete the branch.
|
||||
|
||||
**Verification.**
|
||||
- Local: step 1's `git status --porcelain` output recorded in the PR description (clean or not — that decides branch 2 vs 3).
|
||||
- CI-run evidence: a green `java` job on the fix branch (churn really is gone / discriminator correctly classifies), plus the deliberately-red run from step 5 proving message-level drift is now caught.
|
||||
- No .NET/proto surface changes → no gateway/worker builds needed beyond CI's normal jobs.
|
||||
|
||||
---
|
||||
|
||||
## IPC-25 — Committed Go and Python worker bindings are stale at HEAD, and no guard covers them `Medium` · `P0`
|
||||
|
||||
**Finding.** `clients/go/internal/generated/mxaccess_worker.pb.go` (last regenerated 2026-05-23, commit `397d3c5`) and `clients/python/src/zb_mom_ww_mxgateway/generated/mxaccess_worker_pb2.py` (2026-06-18, `e328758`) both predate the 2026-07-09 `max_frame_bytes` addition — `grep -c max_frame_bytes` returns 0 for both at HEAD, while the Java aggregate has it (×6). The freshness net covers `Contracts/Generated` (check-codegen Check 2), the descriptor (Check 1 + `ClientProtoInputTests`), Rust vendored protos (Check 3), and Java (`checkGeneratedClean`) — nothing compares the committed Go/Python bindings against the protos, and their unit tests don't exercise the worker types, so CI stays green.
|
||||
|
||||
**Impact.** Functional impact today is nil (no language client consumes `GatewayHello`), but the published packages misrepresent the wire contract, and the CLAUDE.md verification-matrix rule ("regenerate every generated client touched by the contract") was violated with zero signal — the exact silent-drift class IPC-01/IPC-09 were meant to end.
|
||||
|
||||
**Design.** Two parts: (1) regenerate and commit both binding sets now; (2) close the guard hole with a **Check 4** in `scripts/check-codegen.ps1` that regenerates Go and Python bindings via the existing per-client scripts and fails on any git diff — the same regenerate-and-diff pattern Check 2 uses for `Generated/`. The per-client scripts already carry the churn-tolerance machinery this needs: `clients/python/generate-proto.ps1:36-48` hard-asserts grpcio-tools 1.80.0 (so a regen is deterministic and a diff means real drift, not toolchain noise), and `clients/go/generate-proto.ps1:9-43` resolves pinned-on-PATH tools. The one gap: the Go script does not assert plugin versions, and plugin-version drift stamps header churn (`protoc-gen-go v1.36.11` / `protoc-gen-go-grpc v1.6.2` in the committed headers) — add the same fail-fast version assertion the Python script has, so Check 4 cannot false-fail (or worse, mask drift under churn) on an off-pin machine.
|
||||
|
||||
Rejected alternatives: a semantic grep for known-new symbols (catches this instance, not the class); folding Go/Python regen into `ClientProtoInputTests` (the test project must stay toolchain-free — that is its documented value, `docs/ClientProtoGeneration.md:76-81`); comparing committed bindings against the descriptor set semantically (a weaker parse-level check that misses generator-behavior drift and requires per-language descriptor parsing for no gain over regenerate-and-diff).
|
||||
|
||||
**Implementation.**
|
||||
1. **Pin the Go generators in the script.** `clients/go/generate-proto.ps1`: after `Resolve-Tool`, assert `& $protocGenGo --version` equals `protoc-gen-go v1.36.11` and `& $protocGenGoGrpc --version` equals `protoc-gen-go-grpc 1.6.2` (match the committed header stamps); throw with an install hint (`go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11`, `go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.6.2`) on mismatch. protoc itself: warn-if-off-pin (34.1) matching `publish-client-proto-inputs.ps1` semantics.
|
||||
2. **Regenerate Go** (macOS or Windows; PATH-first scripts work on both):
|
||||
```
|
||||
go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11
|
||||
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.6.2
|
||||
pwsh clients/go/generate-proto.ps1
|
||||
```
|
||||
Expected delta: `max_frame_bytes` additions confined to `mxaccess_worker.pb.go` (descriptor bytes + `MaxFrameBytes` accessor). If other files churn, the pins are wrong — stop and reconcile before committing. Then from `clients/go`: `gofmt -l .` (must be empty), `go build ./...`, `go test ./...`.
|
||||
3. **Regenerate Python** with the pinned toolchain (memory `project_python_client_regen_pin`):
|
||||
```
|
||||
python -m pip install 'grpcio-tools==1.80.0' # bundles protobuf 6.31.1 codegen
|
||||
pwsh clients/python/generate-proto.ps1 # Assert-GrpcioToolsVersion enforces the pin
|
||||
```
|
||||
Expected delta: the serialized-descriptor line(s) in `mxaccess_worker_pb2.py` only — keep only the real descriptor delta; revert any file the regen did not meaningfully change. Then from `clients/python`: `python -m pytest`.
|
||||
4. **Java untouched** (no `.proto` changed): if gradle ran incidentally, `git checkout -- clients/java/src/main/generated` (memory `project_java_generated_churn`; goes away if IPC-24 step 1 confirms the pin killed the churn).
|
||||
5. **Add Check 4 to `scripts/check-codegen.ps1`:** "Go and Python client bindings match a fresh regeneration" — invoke `clients/go/generate-proto.ps1` and `clients/python/generate-proto.ps1`, then fail on non-empty `git status --porcelain -- clients/go/internal/generated clients/python/src/zb_mom_ww_mxgateway/generated`. Tool-missing must **fail** the check, not skip it (a skipped guard is this finding). Update the script's header comment ("Three checks" → four, list the new check) and **relabel all check banners `1/4`…`4/4`** (absorbs IPC-32).
|
||||
6. **CI:** in `.gitea/workflows/ci.yml` `portable` job, before the `Codegen / descriptor freshness` step (`:60-62`), add the toolchain installs Check 4 needs: `go install .../protoc-gen-go@v1.36.11`, `go install .../protoc-gen-go-grpc@v1.6.2` (Go is already set up at `:28-30`), and `python -m pip install 'grpcio-tools==1.80.0'` (Python at `:32-34`; protoc 34.1 already installed at `:43-47`).
|
||||
7. **Same-commit docs:** `docs/ClientProtoGeneration.md` pinned-versions table (`:88-95`): add `protoc-gen-go v1.36.11` / `protoc-gen-go-grpc v1.6.2` rows and change the Go/Python guard column to "check-codegen Check 4"; `docs/Contracts.md` cross-link if it enumerates the checks.
|
||||
8. **Publishing note (cross-domain, CLI):** the currently *published* Go/Python packages carry the stale worker bindings. Do not republish from this fix alone — CLI-39 (version constants aligned to an already-published 0.1.2) must be resolved first; flag to the clients-domain plan.
|
||||
|
||||
**Verification.**
|
||||
- `pwsh scripts/check-codegen.ps1` locally: all four checks green post-commit; then a negative run — touch a scratch field in `mxaccess_worker.proto`, regenerate only `Contracts/Generated/`, rerun and confirm **Check 4 fails** naming both stale directories; revert.
|
||||
- `grep -c max_frame_bytes clients/go/internal/generated/mxaccess_worker.pb.go clients/python/src/zb_mom_ww_mxgateway/generated/mxaccess_worker_pb2.py` → both non-zero.
|
||||
- `cd clients/go && gofmt -l . && go build ./... && go test ./...`; `cd clients/python && python -m pytest`.
|
||||
- CI-run evidence: green `portable` job on the fix branch with the Check 4 banner visible in the log.
|
||||
- Nothing here needs windev (no worker/x86 surface).
|
||||
|
||||
---
|
||||
|
||||
## IPC-26 — Worker frame writer: cancelling while waiting for the write lock leaves a ghost frame that is still written `Low` · `P2` · fix mechanics in **WRK-22**
|
||||
|
||||
**Finding.** `Worker/Ipc/WorkerFrameWriter.cs:87-113`: the `PendingFrame` is enqueued (`:88-98`) *before* `await _writeLock.WaitAsync(cancellationToken)` (`:103`). If the caller's token fires while waiting, `WriteAsync` throws `OperationCanceledException` but the frame stays queued — the next lock-holder writes it anyway (`DequeueNext` `:200-216` has no tombstone concept), its `Completion` resolves unobserved, and during shutdown leftover cancelled event frames can drain *behind* the `WorkerShutdownAck` that is nominally the last frame. If no writer ever follows, the frame lingers unwritten with an unobserved completion.
|
||||
|
||||
**Impact.** Benign today (the gateway drops post-shutdown/unknown frames), but the "cancelled means not written" contract is silently violated, and post-ack event emission is exactly the kind of latent seam a future gateway-side strictness change would trip over.
|
||||
|
||||
**Design (contract-level requirement — the WRK-22 implementation must satisfy this).** *A `WriteAsync` that completes with `OperationCanceledException` guarantees its frame is never subsequently written to the stream.* The recommended shape (owned by WRK-22): on cancellation, tombstone the pending frame — `Completion.TrySetCanceled()` under `_gate`, and have `DequeueNext` (or the drain loop) skip frames whose completion is already terminal. Removal-by-scan is equivalent; tombstoning avoids O(n) queue rewrites. Additionally, `WorkerShutdownAck` remains the last frame written: since the ack is a control frame and the scheduler drains control-before-event, the tombstone rule alone restores this (cancelled event frames were the only leak path). No proto or gateway change; no doc change beyond the IPC-29 section noting the cancellation semantics once WRK-22 lands.
|
||||
|
||||
**Implementation.**
|
||||
1. Code + tests: **deferred to WRK-22** (`Worker/Ipc/WorkerFrameWriter.cs`; worker test: cancel a `WriteAsync` blocked on the lock, then drive another write, assert the cancelled envelope never reaches the stream and its task is `Canceled`).
|
||||
2. This plan: when writing the IPC-29 doc section, include one sentence — *a write cancelled while queued is tombstoned and never reaches the wire* — phrased to land in the same commit as the WRK-22 fix (do not document it before it is true).
|
||||
|
||||
**Verification.** WRK-22's tests on windev: `dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86 --filter FullyQualifiedName~WorkerFrameProtocolTests`. Doc cross-check against the shipped behavior.
|
||||
|
||||
---
|
||||
|
||||
## IPC-27 — Descriptor freshness test checks messages and fields only; enums, enum values, services/methods, and the Galaxy contract are blind spots `Low` · `P2`
|
||||
|
||||
**Finding.** `src/ZB.MOM.WW.MxGateway.Tests/Contracts/ClientProtoInputTests.cs:35-52` collects only `{message}` and `{message}/{field}` symbols and enumerates only `MxaccessGatewayReflection.Descriptor` and `MxaccessWorkerReflection.Descriptor`. A new enum value (e.g. a new `MxCommandKind` case — the most likely future contract change), a new RPC, or any change confined to `galaxy_repository.proto` would not redden the test. The script-side `-Check` in CI does catch all of these (full descriptor compare), so the gate as-a-whole holds — but `docs/ClientProtoGeneration.md:76-81` presents the test as the protoc-free **primary** gate, and on any runner without protoc it is the only one.
|
||||
|
||||
**Impact.** The documented primary guard is weaker than its documentation claims; a protoc-less environment (e.g. a future runner change) silently loses enum/service/Galaxy coverage.
|
||||
|
||||
**Design.** Extend the same reflection walk — all surfaces are available on `FileDescriptor` without protoc:
|
||||
- **Enums:** top-level `file.EnumTypes` and nested `message.EnumTypes`, collecting `{enumFullName}` and `{enumFullName}/{valueName}` (value *presence* suffices; a renumber would be a non-additive change caught by review and the script check — asserting numbers here would duplicate that at the cost of a noisier failure mode).
|
||||
- **Services/methods:** `file.Services`, collecting `{serviceFullName}` and `{serviceFullName}/{methodName}`.
|
||||
- **Galaxy:** add `GalaxyRepositoryReflection.Descriptor` to the enumerated files (it is compiled into Contracts — `galaxy_repository.proto` is retained there as the client-codegen source per the csproj comment).
|
||||
The published-side collection (`CollectPublishedSymbols`, `:61-79`) grows matching walks over `FileDescriptorProto.EnumType`/`Service` and `DescriptorProto.EnumType`. Keep the flat string-set comparison — it stays order-insensitive and protoc-free.
|
||||
|
||||
**Implementation.**
|
||||
1. `ClientProtoInputTests.cs`: extend `Descriptor_ContainsEveryContractMessageAndField` (rename to `Descriptor_ContainsEveryContractSymbol`) per the design; failure message keeps listing missing symbols.
|
||||
2. `docs/ClientProtoGeneration.md:76-81`: update the prose from "message or field" to the full symbol coverage — same commit.
|
||||
3. No proto, config, or CI change (the test already runs in the portable job via the gateway test step).
|
||||
|
||||
**Verification.**
|
||||
- `dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter FullyQualifiedName~ClientProtoInputTests` (macOS).
|
||||
- Red-path proof during development: extract the pre-IPC-01 stale protoset (`git show 0f88a95:clients/proto/descriptors/mxaccessgw-client-v1.protoset > <scratch>/stale.protoset`), point the test's path at it temporarily, and confirm it fails naming (at minimum) the `max_frame_bytes` field and — with the extension — any post-`0f88a95` enum values. Restore before commit.
|
||||
|
||||
---
|
||||
|
||||
## IPC-28 — `docs/Grpc.md` exception-mapping prose omits `CommandTooLarge` → `ResourceExhausted` `Low` · `—`
|
||||
|
||||
**Finding.** `Grpc/MxAccessGatewayService.cs:955` maps `WorkerClientErrorCode.CommandTooLarge` to `ResourceExhausted` (the IPC-03 per-correlation fix), but `docs/Grpc.md:249` still enumerates only `CommandTimeout`/`GatewayShutdown`/`InvalidState`/`ProtocolViolation` with "unmapped codes fall through to `Unavailable`" — a reader concludes an oversized command surfaces as `Unavailable`. (`docs/GatewayConfiguration.md:120-129` does document the headroom rule.)
|
||||
|
||||
**Impact.** Doc drift on a shipped, client-visible status mapping; violates the docs-change-with-source rule retroactively.
|
||||
|
||||
**Design.** Doc-only. Add `CommandTooLarge → ResourceExhausted` to the `WorkerClientException` mapping sentence, and one line in the Invoke handler section: an accepted gRPC payload whose worker envelope would exceed the pipe frame limit fails *that command* with `ResourceExhausted` (per-correlation, session stays alive) — cross-reference `docs/GatewayConfiguration.md`'s headroom rule.
|
||||
|
||||
**Implementation.**
|
||||
1. `docs/Grpc.md:249`: extend the mapping prose.
|
||||
2. `docs/Grpc.md` Invoke section: add the oversized-payload sentence.
|
||||
3. If IPC-23/WRK-21 lands first, fold its DrainEvents-truncation doc row (IPC-23 step 5) into the same edit pass.
|
||||
|
||||
**Verification.** Doc-only; cross-check the prose against the `switch` in `Grpc/MxAccessGatewayService.cs:950-960` after editing.
|
||||
|
||||
---
|
||||
|
||||
## IPC-29 — Worker writer priority scheduling and write-time sequence stamping undocumented in the frame-protocol doc `Low` · `—`
|
||||
|
||||
**Finding.** The worker writer is now a cooperative priority scheduler (control frames drain ahead of event frames, `Worker/Ipc/WorkerFrameWriter.cs:11-17,116-124`) that stamps `Sequence` at the moment of writing under the lock (`:238-240`) and coalesces flushes per batch (`:125-182`). These are protocol-relevant semantics: a per-frame rejection consumes a sequence number (stamped at `:240` before the size check at `:250` throws), so the wire sequence has gaps after rejections, and event frames can be arbitrarily delayed behind a control backlog. `docs/WorkerFrameProtocol.md:1-7` still describes the frame layer as only "message boundaries, size limits, protobuf parsing, and envelope validation"; `gateway.md:328-330` covers the diagnostic-sequence half but not scheduling.
|
||||
|
||||
**Impact.** Anyone implementing an alternate-language worker (contemplated in `gateway.md`) or debugging sequence gaps from a pipe capture has no doc for the shipped semantics — the exact drift class the repo's docs-change-with-source rule exists to prevent.
|
||||
|
||||
**Design.** Doc-only: add a "Write scheduling and sequencing" section to `docs/WorkerFrameProtocol.md` covering (a) the two-class priority queues and the control-before-event drain guarantee; (b) enqueue-then-contend locking, single-writer drain; (c) sequence stamped at the point of writing so wire order and sequence order always agree on the worker side; (d) sequence gaps after per-frame rejections are expected and benign (sequence is diagnostic-only — cross-ref `gateway.md:328-330`; the gateway side stamps at creation, see IPC-31, so its sequence order is *not* a wire-order guarantee); (e) flush coalescing with the written-and-flushed completion contract; (f) once WRK-22 lands, the cancellation tombstone rule (see IPC-26 — do not document before it ships).
|
||||
|
||||
**Implementation.**
|
||||
1. `docs/WorkerFrameProtocol.md`: insert the section after "Envelope Validation"; fold in IPC-23's byte-cap sentence (step 4 there) if landing together.
|
||||
2. Optionally one cross-reference line in `gateway.md:328-330` pointing to the new section. No code change.
|
||||
|
||||
**Verification.** Doc-only; cross-check each claim against `Worker/Ipc/WorkerFrameWriter.cs` and `WorkerFrameWritePriority.cs` at time of writing.
|
||||
|
||||
---
|
||||
|
||||
## IPC-30 — Oversized worker→gateway event frame is still session-fatal despite the per-frame rejection machinery `Low` · `P0` (lands with WRK-21 in the same file/batch)
|
||||
|
||||
**Finding.** `Worker/Ipc/WorkerPipeSession.cs:374-382`: the event drain loop awaits each event write; a `MessageTooLarge` per-frame rejection from the writer (an MXAccess array event serializing above the negotiated max) throws out of `RunEventDrainLoopAsync`, is rethrown by the message loop's `await eventDrainTask` (`:294`), and terminates `RunAsync` — the whole session dies for one unsendable event, with no fault frame and no record of *which* event. The writer survives the rejection (`WorkerFrameWriter.cs:141-147`); the session dies anyway one layer up, wasting the machinery.
|
||||
|
||||
**Impact.** One pathological tag (a huge array) kills the session for every item the client has, and the operator gets a generic protocol-violation exit with no way to find the offending tag.
|
||||
|
||||
**Design — position taken: session-fatal is the correct contract; the defect is that the death is unstructured.** Rationale: an event above the negotiated frame max is **undeliverable end-to-end** — the negotiated pipe max sits only `EnvelopeOverheadReserveBytes` (64 KiB, `Server/Workers/WorkerFrameProtocolOptions.cs:20`) above the public gRPC cap, so any event the pipe rejects would also be rejected by the client-facing gRPC stream. Given that, the honest options reduce to how the session dies, not whether:
|
||||
- **Silently drop and continue — rejected.** The event stream would silently stop being faithful; that breaks the delivery expectation behind every subscription, and the repo's "don't synthesize events" rule bars papering over it with a fabricated placeholder.
|
||||
- **Drop + synthesized loss-marker event — rejected.** `ReplayGap` is a gateway-lifecycle stream element (replay-ring eviction), not a worker event; reusing it would lie about replay state, and a *new* loss-marker event is exactly the synthesized event the conventions prohibit.
|
||||
- **Chunked/segmented event frames — rejected.** Contract surgery plus reassembly state on both sides for a case that indicates misconfiguration (`MaxMessageBytes` too small for the workload); MXAccess parity treats an event as atomic.
|
||||
- **Chosen: deliberate, structured session-fatal.** Catch the per-frame `MessageTooLarge` rejection at the drain-loop seam, log the event identity — family, item/tag name, worker sequence, serialized size; **never the value payload** (redaction rule) — write a `WorkerFault` so the gateway sees a structured fault instead of a raw pipe drop, then exit. This mirrors the existing fail-fast precedent (`WORKER_FAULT_CATEGORY_QUEUE_OVERFLOW` faults the session on event-queue overflow) and the drain loop's existing fault pattern (`WorkerPipeSession.cs:356-365`). Operator remediation is config (`MxGateway:Worker:MaxMessageBytes`), and now the log names the tag that needs it.
|
||||
- **Fault category: reuse `WORKER_FAULT_CATEGORY_PROTOCOL_VIOLATION` (`mxaccess_worker.proto:130`)** with a diagnostic message carrying the event identity and sizes. A dedicated additive enum value (`..._EVENT_TOO_LARGE = 12`) was considered and rejected *for this pass*: it triggers the full five-client + descriptor + `Generated/` regen fan-out for a distinction the diagnostic message already conveys; revisit if dashboards need to alert on it specifically. This keeps IPC-30 proto-neutral.
|
||||
|
||||
**Implementation.** (Same file and same P0 batch as WRK-21 — one commit or adjacent commits, one windev verification run.)
|
||||
1. `Worker/Ipc/WorkerPipeSession.cs`, `RunEventDrainLoopAsync` (`:374-382`): wrap the per-event `WriteAsync` in a `catch (WorkerFrameProtocolException ex) when (ex.ErrorCode == WorkerFrameProtocolErrorCode.MessageTooLarge)`. In the handler: log event identity + serialized size (structured log, no values); set `_state = WorkerState.Faulted`; `await TryWriteFaultAsync(new WorkerFault { Category = ProtocolViolation, CommandMethod = "EventDrain", DiagnosticMessage = "<family> event for '<item>' (worker sequence N) serialized to X bytes, exceeding the negotiated frame max Y; raise MxGateway:Worker:MaxMessageBytes for this workload" }, ct)`; then rethrow/throw `InvalidOperationException` so `RunAsync` exits exactly as today — the fault frame is small and control-priority, so it precedes the exit. Other per-frame rejection codes (`InvalidEnvelope` etc.) keep today's behavior — they indicate worker bugs, not workload size.
|
||||
2. Gateway side: no change — `WorkerClient`'s existing `WorkerFault` handling already surfaces structured faults to the session and dashboard.
|
||||
3. Worker test (`Worker.Tests`, x86): fake runtime session emitting one event whose envelope exceeds a small test `MaxMessageBytes`; assert (a) a `WorkerFault` frame with `ProtocolViolation` and the item name in `DiagnosticMessage` is written before the pipe session ends, (b) the writer stream is not corrupted (a subsequent well-formed frame parse on the captured stream), (c) the session task completes faulted.
|
||||
4. Same-commit docs: `docs/WorkerFrameProtocol.md` (in/next to the IPC-29 section): oversized-event policy — undeliverable end-to-end, session faults with a structured `WorkerFault` naming the event, remediation is `MaxMessageBytes` config. `docs/MxAccessWorkerInstanceDesign.md` event-drain description gets the same one-paragraph policy.
|
||||
|
||||
**Verification.**
|
||||
- windev (isolated `C:\build` worktree, memory `project_windev_worker_verify`): `dotnet build src/ZB.MOM.WW.MxGateway.Worker/ZB.MOM.WW.MxGateway.Worker.csproj -p:Platform=x86`; `dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86 --filter FullyQualifiedName~WorkerPipeSession` (covers this and WRK-21 in one run).
|
||||
- Gateway fake-worker regression: `dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter FullyQualifiedName~WorkerClient` (fault propagation unchanged).
|
||||
- Until TST-25's automated Windows tier exists, record the windev run (pass counts, date) in the tracking log — CI cannot evidence this one.
|
||||
|
||||
---
|
||||
|
||||
## IPC-31 — Gateway envelope sequence stamped at creation, not at write `Info` · N/A
|
||||
|
||||
**Finding.** `Server/Workers/WorkerClient.cs:1039` stamps `Sequence` via `Interlocked.Increment` inside `CreateEnvelope`; concurrent `InvokeAsync` callers can enqueue out of stamp order, so gateway→worker wire order can disagree with sequence order — the opposite semantics from the worker's write-time stamping (`Worker/Ipc/WorkerFrameWriter.cs:238-240`).
|
||||
|
||||
**Disposition: N/A — no action.** `gateway.md:328-330` already declares sequence a per-sender diagnostic aid, never validated; both validators ignore it; the divergence is harmless and now explicitly documented by the IPC-29 doc section (which states the two sides' stamping points). If sequence ever becomes load-bearing, stamp in the gateway write loop (`WorkerClient.cs:390-397`) — recorded here so the next editor finds the decision.
|
||||
|
||||
---
|
||||
|
||||
## IPC-32 — `check-codegen.ps1` check labels are miscounted `Info` · `—` (rides IPC-25)
|
||||
|
||||
**Finding.** `scripts/check-codegen.ps1:30,42,68` print "Check 1/2", "Check 2/2", then "Check 3/3" — the middle labels predate the third check. Cosmetic; CI log readers see a miscounted progression.
|
||||
|
||||
**Impact.** Log-readability only.
|
||||
|
||||
**Design/Implementation.** Folded into the IPC-25 script edit: when Check 4 is added, relabel every banner `1/4`…`4/4` and update the header comment's check list in the same change (IPC-25 implementation step 5). If IPC-25 were descoped (it must not be — it is P0), the standalone fix is relabeling `1/3`, `2/3`, `3/3`.
|
||||
|
||||
**Verification.** Banner text in the `pwsh scripts/check-codegen.ps1` output during the IPC-25 verification run; visible in the CI `portable` job log.
|
||||
|
||||
---
|
||||
|
||||
## Cross-domain dependencies and sequencing
|
||||
|
||||
- **WRK-21 / IPC-23 / IPC-30 cluster (P0):** one worker-side change set in `Worker/Ipc/WorkerPipeSession.cs` (+ the proto-comment/doc wave from IPC-23). Land as one batch so the x86 build, `WorkerPipeSession`-filtered test run on windev, and the proto regen fan-out each happen once. WRK-21 owns the size-aware DrainEvents packing (must satisfy IPC-23's R1–R3); IPC-30's structured-fault seam is in the same drain loop.
|
||||
- **WRK-22 / IPC-26 (P2):** worker plan owns the tombstone fix; this plan's IPC-29 doc section documents the cancellation semantics only once it ships.
|
||||
- **IPC-24 + IPC-25 vs TST-25 (CI):** both edit `.gitea/workflows/ci.yml` (java job step removal; portable job toolchain installs). TST-25's Windows-tier work will extend the same file — coordinate branches to avoid workflow-merge conflicts, and note that everything windev-verified in this plan (IPC-23/30, WRK-21/22) has no CI evidence until TST-25 lands; record windev runs in the tracking log meanwhile.
|
||||
- **IPC-25 vs CLI-39 (publishing):** regenerated Go/Python bindings must not be republished to Gitea until the clients-domain version-collision finding (CLI-39) is resolved.
|
||||
@@ -0,0 +1,214 @@
|
||||
# Security & Dashboard — Remediation Design & Implementation (2026-07-12 cycle)
|
||||
|
||||
Source review: [40-security-dashboard.md](../40-security-dashboard.md) · Baseline: `main` @ `4f5371f` · Generated: 2026-07-13
|
||||
|
||||
This document turns the six NEW findings of the 2026-07-12 security/dashboard re-review (SEC-31…SEC-36) into buildable remediation entries, in the same per-finding format as the prior cycle's [archreview/remediation/40-security-dashboard.md](../../remediation/40-security-dashboard.md). Tiers follow the [00-overall.md](../00-overall.md) roadmap: SEC-31/32 are the P0 failure-limiter re-partition (roadmap item 3), SEC-33/36 ride the P1 residual sweep (item 8), SEC-34 is a P2 Low (item 9), SEC-35 is doc-only. The still-open prior backlog is referenced where a fix should co-locate: **SEC-23** (validator coverage of dashboard/cookie flags) shares the validator pass with SEC-33, **SEC-24** (effective-config view omissions) should absorb the new `MxGateway:Security` keys introduced here, and **SEC-14** (unauthenticated `/metrics`) constrains what the new limiter telemetry may expose.
|
||||
|
||||
Repo rules that bind every entry: docs change in the same commit as the source (`CLAUDE.md`), never log or commit secrets (this document references the committed LDAP credential **by location only**, never by value), `TreatWarningsAsErrors=true` (new public members need XML docs), and targeted `dotnet test --filter` runs per task — not the full suite.
|
||||
|
||||
## Register
|
||||
|
||||
| ID | Sev | Tier | Eff | Dep | Status | Title |
|
||||
|----|-----|------|-----|-----|--------|-------|
|
||||
| SEC-31 | Medium | P0 | M | — | Not started | Failure limiter partitions on attacker-controlled key id and blocks before verification (lockout DoS) |
|
||||
| SEC-32 | Low | P0 | S | SEC-31 | Not started | Failure-limiter LRU is flushable by junk-token spray; token prefix never validated |
|
||||
| SEC-33 | Low | P1 | M | — (co-locate SEC-23) | Not started | Any-platform path-rooting acceptance re-opens SEC-01 on Unix; Galaxy `SnapshotCachePath` unvalidated |
|
||||
| SEC-34 | Low | P2 | S | — | Not started | Verification cache: expiry outlives TTL; `Invalidate` races in-flight repopulation |
|
||||
| SEC-35 | Info | — | S | — | N/A (doc-only note) | Production hard-stops key on the exact `Production` environment name |
|
||||
| SEC-36 | Low | P1 | M | cross-repo (`scadaproj/infra/glauth`) | Not started | Committed dev LDAP service-account password: remove from repo and rotate |
|
||||
|
||||
---
|
||||
|
||||
## SEC-31 — Failure limiter partitions on attacker-controlled key id and blocks before verification `Medium` · `P0`
|
||||
|
||||
**Finding.** `ResolvePeerKey` derives the limiter partition from the **unauthenticated** presented token — `parts[1]` of `mxgw_<keyId>_<secret>` (`src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcAuthorizationInterceptor.cs:115-136`) — and `IsBlocked` short-circuits with `ResourceExhausted` *before* `VerifyAsync` runs (`:72-78`). The success-path `Reset` (`:97`) is only reachable after a verification, which a blocked partition never gets. Defaults: 10 failures per 60 s window (`src/ZB.MOM.WW.MxGateway.Server/Configuration/SecurityOptions.cs:51,57`), limiter internals at `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs:65-114`.
|
||||
|
||||
**Impact.** Key ids are not secrets: they are embedded in every token, shown to every dashboard Viewer, and pushed to anonymous-loopback hub clients in the snapshot's API-key inventory. Any network peer that knows (or enumerates) a key id can send 10 garbage-secret requests per minute and deny that key **indefinitely** — the legitimate client presenting the *correct* secret is rejected before its credential is ever checked, and because rejection precedes verification, the reset that would clear the block is unreachable. The NAT-caveat comment (`GatewayGrpcAuthorizationInterceptor.cs:113-114`) optimized for not locking out co-located clients and instead handed remote attackers a per-key kill switch. Ten packets per minute is all it costs to keep it held down forever.
|
||||
|
||||
**Design.** This is a re-partition plus an admission-valve redesign. The limiter exists for two guarantees that any fix must preserve: (G1) online secret-guessing against a key is bounded (~`ApiKeyFailureLimit` per window), and (G2) the failure path does not spend a SQLite read per attempt (failures are never cached by `CachingApiKeyVerifier`, so without the pre-verify short-circuit every guess hits the store). The new guarantee is (G3): no unauthenticated input may indefinitely deny a key to a holder of the correct secret.
|
||||
|
||||
*Options considered:*
|
||||
|
||||
1. **Pure transport-peer partition** (`context.Peer`, mirroring the dashboard login limiter). Satisfies G3 — an attacker only blocks their own address. Rejected as the sole partition for two reasons: **NAT sharing** — all clients behind one NAT share one budget, so a single fat-fingering (or malicious) neighbor locks the address for every key (the exact scenario the current keying was built to avoid, per `glauth.md`'s NAT caveat); and **IPv6 rotation** — an attacker holding a /64 mints fresh source addresses at will, so the per-key guessing bound (G1) collapses to `limit × addresses`, i.e. effectively unbounded.
|
||||
|
||||
2. **Post-verification-only failure counting** (verify first, never block pre-verify). Satisfies G3 by construction — the correct secret always reaches the constant-time compare. Rejected as the sole mechanism because it abandons G2 entirely: every failed guess costs the full library verify (SQLite read + peppered hash + compare), which is precisely the resource the limiter shields (the interceptor comment at `:67-71` states this as the design intent).
|
||||
|
||||
3. **Composite `(transport peer, key id)` partition** for the pre-verify check. An attacker's spam only trips (their address × that key); the legitimate holder on any other address is untouched, and co-located NAT clients using *different* keys are untouched. Rejected as the sole mechanism because IPv6 rotation again weakens G1: `limit` guesses per window per minted address.
|
||||
|
||||
4. **Chosen: composite partition + per-key-id aggregate, with probe admission instead of absolute blocking.** Two layers over one shared window mechanism:
|
||||
- **Layer 1 — composite `(peer, keyId)` partition**, checked pre-verify as today. Bounds per-address guessing at `ApiKeyFailureLimit`/window and preserves G2 for the common single-source case.
|
||||
- **Layer 2 — per-key-id aggregate**: failures for a (validly-shaped, see SEC-32) key id are also counted across *all* peers. When the aggregate exceeds `ApiKeyFailureAggregateLimit` (new, default 30/window), the key id enters **probe mode**.
|
||||
- **Probe admission**: an over-limit state (either layer) is not an absolute wall — at most one request per `ApiKeyFailureProbeIntervalSeconds` (new, default 5) is admitted through to the real verifier; all others still short-circuit with `ResourceExhausted`. A successful verification resets both the composite partition and the aggregate (making the success-reset path *reachable while throttled*, which is the structural fix for the lockout inversion).
|
||||
|
||||
*Resulting bounds:* a single attacking peer gets `limit`/window + 1 probe per interval (≈ 10 + 12 per minute at defaults) — the same order as the current intent. A distributed/rotating attacker gets at most `ApiKeyFailureAggregateLimit` guesses in the first window before the aggregate trips, then ~1 probe per interval **globally for that key** — which is *stronger* than option 3 and within the same order as today's per-key bound (G1 holds). A legitimate holder presenting the correct secret authenticates immediately from any un-tripped address, and under active spray recovers within a small number of probe intervals (it competes for probe slots, but a win resets all state; the attacker must then re-trip the aggregate from scratch). Residual accepted: a high-rate distributed sprayer can *delay* (not deny) the legitimate holder — but that now requires sustained, metrically-visible attack traffic instead of 10 packets/min, and G3's "indefinitely" is eliminated.
|
||||
|
||||
*Also rejected:* trusting the key id after a cheap local check (any format check is attacker-satisfiable — the design must assume key ids are public); allowlisting known key ids by querying the store pre-verify (reintroduces the per-attempt store read G2 forbids).
|
||||
|
||||
*Telemetry note:* add throttle counters tagged only by stage (`peer`/`aggregate`) — never by key id or peer address (cardinality per the SEC-20 precedent, and `/metrics` is still unauthenticated per open **SEC-14**, so counters must carry no key material). When prior-cycle **SEC-24** (effective-config view) is picked up, the new `MxGateway:Security` keys added here belong in that projection.
|
||||
|
||||
**Implementation.**
|
||||
|
||||
1. `src/ZB.MOM.WW.MxGateway.Server/Configuration/SecurityOptions.cs`: add `int ApiKeyFailureAggregateLimit { get; init; } = 30;` (0 disables the aggregate layer) and `int ApiKeyFailureProbeIntervalSeconds { get; init; } = 5;` (0 disables probe admission → absolute block, documented as not recommended). Full XML docs (warnings-as-errors).
|
||||
2. `src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs` `ValidateSecurity`: `AddIfNegative` for both new keys, with messages naming the config paths (`MxGateway:Security:ApiKeyFailureAggregateLimit`, `...:ApiKeyFailureProbeIntervalSeconds`).
|
||||
3. `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs`: rework the public surface from `IsBlocked/RecordFailure/Reset(string peer)` to a partition-pair API, e.g. `ThrottleDecision Check(ApiKeyThrottlePartition partition)` / `RecordFailure(partition)` / `Reset(partition)` where `ApiKeyThrottlePartition` carries `TransportPeer` (required) and `KeyId` (optional, only when the token is validly shaped per SEC-32). Internal state: `_partitions` keyed `"{peer}|{keyId}"` (or `"{peer}"` fallback) and `_aggregates` keyed by key id, both reusing the existing sliding-window `PeerState` plus a `NextProbeAtTicks` field; probe slots are granted under the per-state lock. Keep the `TimeProvider` seam and the internal test constructor. Rewrite the class remarks — the current NAT rationale (`:13-26`) describes the defective keying.
|
||||
4. `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcAuthorizationInterceptor.cs`: replace `ResolvePeerKey` with a `ResolveThrottlePartition(authorizationHeader, context)` that returns the pair (with the SEC-32 shape check); thread the probe-admission decision through `AuthenticateAndAuthorizeAsync` (a probe-admitted request proceeds to `VerifyAsync`; a throttled one throws `ResourceExhausted` with the existing opaque message). `RecordFailure`/`Reset` take the partition pair. Update the comment block at `:67-78,112-114`.
|
||||
5. Optional but recommended: a `mxgateway.auth.throttled` counter in `Metrics/GatewayMetrics.cs` tagged `stage=peer|aggregate` only.
|
||||
6. Docs, same commit: rewrite `docs/GatewayConfiguration.md:282-284` (the three existing limiter rows describe key-id keying and "resets on success" semantics that change here) and add rows for the two new keys; update the failure-limiter paragraph in `docs/Authentication.md` (hot-path section, ~`:85-115`) to describe the two-layer partition and probe admission.
|
||||
7. Tests: new `src/ZB.MOM.WW.MxGateway.Tests/Security/Authorization/ApiKeyFailureLimiterTests.cs` (window pruning, composite-vs-aggregate trip points, probe cadence, reset clears both layers); update the two existing limiter-related interceptor tests (`GatewayGrpcAuthorizationInterceptorTests.cs:363-460`) for the new API.
|
||||
|
||||
New tests (names are the contract):
|
||||
- `AttackerSpamOnVictimKeyId_FromDifferentPeer_DoesNotBlockLegitimateHolderPresentingCorrectSecret` — flood failures for key id K from peer A until blocked; a call from peer B with the correct secret for K verifies successfully on the first attempt (assert the verifier **was** called and the RPC succeeded).
|
||||
- `BruteForceBound_StillEnforcedPerAttackingPeer` — same peer, same key id: attempt `limit`+1 wrong secrets inside the window; assert the final attempt maps to `ResourceExhausted` **before** the verifier is called (reuse `CountingFailureVerifier`).
|
||||
- `AggregateSpray_AcrossManyPeers_TripsPerKeyProbeMode` — failures for one key id from > `AggregateLimit` distinct peers; assert subsequent attempts from a fresh peer are probe-limited (at most one verifier call per probe interval, others `ResourceExhausted`).
|
||||
- `CorrectSecret_DuringProbeMode_AuthenticatesViaProbeSlotAndResets` — while throttled, advance the fake `TimeProvider` past the probe interval; the correct secret is admitted, succeeds, and fully resets the state (next wrong attempt is `Unauthenticated`, not `ResourceExhausted`).
|
||||
|
||||
**Verification.** `dotnet build src/ZB.MOM.WW.MxGateway.Server` then:
|
||||
```
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~ApiKeyFailureLimiter"
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~GatewayGrpcAuthorizationInterceptor"
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~GatewayOptionsValidator"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## SEC-32 — Failure-limiter LRU is flushable by junk-token spray; token prefix never validated `Low` · `P0` · depends on SEC-31
|
||||
|
||||
**Finding.** The tracked-peer map evicts the least-recently-active entry once it exceeds `ApiKeyFailureTrackedPeers` (default 4096) — `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs:124-148` — and the partition key is minted from attacker-chosen token text with **no prefix check**: any `a_b_c`-shaped garbage creates a fresh `key:b` partition (`GatewayGrpcAuthorizationInterceptor.cs:127-132`; the `mxgw` literal is never compared).
|
||||
|
||||
**Impact.** A guesser interleaves real guesses with ~4096 unique throwaway tokens to evict their own `PeerState` (or a SEC-31 victim's blocked state) and reset the window, resuming past the 10-per-minute bound at a cost of ~4096 requests per flush cycle. The documented guarantee ("a spray of unique peer keys cannot grow memory without limit", `SecurityOptions.cs:59-64`) is true but silently trades away the *blocking* guarantee it exists to serve.
|
||||
|
||||
**Design.** Three reinforcing changes, landed inside the SEC-31 rework (same types, same tests, one commit train):
|
||||
|
||||
1. **Validate token shape before minting a key-id partition.** Only a token whose first segment is the literal `mxgw`, with ≥ 3 non-empty `_`-segments and a key id within a sane length cap (≤ 64 chars — generated key ids are far shorter), gets a `KeyId` on its throttle partition. Everything else falls to the transport-peer fallback partition — so a junk spray from one address accumulates in **one** partition (which then throttles that address wholesale) instead of minting thousands.
|
||||
2. **Cap key-id partitions per transport peer.** SEC-31's composite keying already binds every partition to the sender's address, but a single address can still mint one partition per unique `mxgw`-shaped key id it invents. Cap distinct key-id partitions per transport peer at a small constant (32; not config — there is no legitimate reason for one address to fail against dozens of distinct keys inside one window); overflow collapses into that address's fallback partition. This bounds the total partitions one attacker can mint to 33 per source address.
|
||||
3. **Eviction never removes load-bearing state.** `EvictIfOverCapacity` preference order: (a) entries whose windows are empty (fully expired) first; (b) never evict an entry that is currently over-limit / in probe mode — such entries decay naturally within `ApiKeyFailureWindowSeconds`, so the exemption cannot pin memory indefinitely; (c) otherwise least-recently-active. Allow a documented transient overshoot of `ApiKeyFailureTrackedPeers` for non-evictable over-limit entries with a hard ceiling of 2× the cap (beyond which oldest-blocked is evicted anyway — availability of the bound beats perfection of the block under a distributed attack that large).
|
||||
|
||||
*Rejected alternatives:* validating the key id against the store pre-verify (reintroduces the per-attempt SQLite read); a cryptographic/entropy check on the key-id segment (attacker-satisfiable, adds nothing over the length/prefix check); simply raising `ApiKeyFailureTrackedPeers` (changes the flush price, not the defect).
|
||||
|
||||
**Implementation.**
|
||||
|
||||
1. `GatewayGrpcAuthorizationInterceptor.ResolveThrottlePartition` (from SEC-31 step 4): add the `mxgw` prefix + segment + key-id-length checks; on failure return a partition with `KeyId = null`.
|
||||
2. `ApiKeyFailureLimiter`: per-transport-peer key-id partition counter (cap 32, collapse to fallback); eviction policy per design point 3 (`EvictIfOverCapacity` rewrite; over-limit detection reuses the same window state the `Check` path computes).
|
||||
3. Update the limiter remarks and `SecurityOptions.ApiKeyFailureTrackedPeers` XML doc (the "cannot grow memory without limit" sentence gains the "and cannot be flushed" half plus the 2× transient note).
|
||||
4. Docs, same commit: `docs/GatewayConfiguration.md` `ApiKeyFailureTrackedPeers` row — describe the eviction preference and the flush resistance.
|
||||
5. Tests (in `ApiKeyFailureLimiterTests` / `GatewayGrpcAuthorizationInterceptorTests`):
|
||||
- `NonMxgwToken_FallsBackToTransportPeerPartition` — junk tokens of varied shapes all land on the sender's fallback partition (assert one tracked entry).
|
||||
- `JunkTokenSpray_DoesNotEvictBlockedEntry` — trip a block for (peer A, key K); spray 2× `TrackedPeers` unique tokens from other peers; assert (A, K) is still blocked.
|
||||
- `UniqueMxgwKeyIdSpray_FromOnePeer_CollapsesAtPerPeerCap` — 1000 unique `mxgw`-shaped key ids from one address mint ≤ 32 + 1 partitions and end up throttling the address's fallback partition.
|
||||
- `Eviction_PrefersExpiredWindows` — with the map at capacity, a new failure evicts an entry whose window has fully expired, not an active one.
|
||||
|
||||
**Verification.** Same filters as SEC-31 (`~ApiKeyFailureLimiter`, `~GatewayGrpcAuthorizationInterceptor`) after `dotnet build src/ZB.MOM.WW.MxGateway.Server`.
|
||||
|
||||
---
|
||||
|
||||
## SEC-33 — Any-platform path-rooting acceptance re-opens SEC-01 on Unix; Galaxy `SnapshotCachePath` unvalidated `Low` · `P1` · co-locate with open SEC-23
|
||||
|
||||
**Finding.** `IsRootedForAnyPlatform` (`src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs:541-560`) deliberately passes Windows drive-qualified/UNC paths on Unix so the shipped `appsettings.json` validates cross-platform. But the validator runs in the same process that then *uses* the path: on Unix, `C:\ProgramData\MxGateway\gateway-auth.db` (`appsettings.json:17`) contains no Unix separator, SQLite treats it as one relative filename, and the credential store lands in the CWD — the original SEC-01 mechanism, now behind a validator message promising it "never lands in the launch working directory" (`:112`). Evidence at HEAD: a stray auth DB dated 2026-07-09 (post-SEC-01-fix) under `src/ZB.MOM.WW.MxGateway.Tests/bin/Debug/net10.0/` with the literal Windows path as its filename, invisible to the hygiene test's bin/obj filter (`src/ZB.MOM.WW.MxGateway.Tests/ProjectStructure/GatewayTreeHygieneTests.cs:30-35`). Additionally Galaxy `SnapshotCachePath` (`appsettings.json:80`) gets **no** rooting validation: the shared `ZB.MOM.WW.GalaxyRepository` package binds it (`GatewayApplication.cs:103`) and deliberately ships no validator — per `A2-galaxyrepository-adoption-handoff.md:157-160` the gateway was supposed to own that validation, and doesn't.
|
||||
|
||||
**Impact.** Any non-Windows run (dev box, tests, future container) that binds the shipped config silently writes the API-key credential store — and the Galaxy snapshot — as junk-named files relative to whatever the CWD happens to be. The validator's guarantee is false on the platform where it matters; the hygiene test cannot see the failure because the artifacts land in gitignored build output.
|
||||
|
||||
**Design.** Make rooting **host-meaningful** and stop shipping foreign-platform literals, instead of teaching the validator to bless paths the process cannot use:
|
||||
|
||||
1. **Validator checks the running OS.** `AddIfNotRooted` uses plain `Path.IsPathRooted` (current platform); delete `IsRootedForAnyPlatform`. A Windows drive path on Unix becomes a fail-fast startup error naming the offending key — exactly how every other misconfiguration surfaces. The "validate prod config offline on macOS" rationale in the doc-comment dissolves with change 2, and no offline-validation tooling exists that would need the any-platform mode.
|
||||
2. **Remove the Windows literals from `appsettings.json`.** Drop `Authentication:SqlitePath` (`:17`) and `Galaxy:SnapshotCachePath` (`:80`); the `SpecialFolder.CommonApplicationData`-derived code defaults (SEC-01's own mechanism, `Configuration/AuthenticationOptions.cs:16-19`) resolve to the **identical** `C:\ProgramData\MxGateway\...` on Windows and a platform-appropriate path elsewhere. Deployed hosts are unaffected: production config rides NSSM environment variables and the Windows code default is byte-identical to the removed literal. For Galaxy, add a gateway-side `PostConfigure` default (`Path.Combine(CommonApplicationData, "MxGateway", "galaxy-snapshot.json")`) applied when the bound value is blank.
|
||||
3. **Gateway-side Galaxy options validation.** New `Configuration/GalaxyRepositoryOptionsValidator.cs` registered as `IValidateOptions<GalaxyRepositoryOptions>` (the package type — the lib binds only, by design): when `PersistSnapshot` is true, `SnapshotCachePath` must be non-blank, a valid path, and rooted on the current OS (reuse the `AddIfInvalidPath`/`AddIfNotRooted` helpers — promote them to a shared internal helper if the new validator can't reach them).
|
||||
4. **Root-cause the stray file.** Identify the test/tool run that bound the shipped `appsettings.json` and materialized the DB under the test bin dir; pin it to a temp path (the `TempDatabaseDirectory` helper exists in `Tests/Security/Authentication/`). After changes 1–2 the failure mode is loud (validation error) rather than silent, and on macOS the code default may point at an unwritable `/usr/share/...` — any test exercising the real startup path must override `SqlitePath`. Delete the stray gitignored file.
|
||||
|
||||
*Rejected alternatives:* auto-rooting/rewriting foreign-platform paths to a host default (silent relocation of a credential store — worse than a boot error, per the SEC-01 design note at `GatewayOptionsValidator.cs:515-521`); keeping `IsRootedForAnyPlatform` in the validator and adding a second use-site runtime check (two checks to keep aligned, and the validator's promise stays false); extending the hygiene test to scan bin/obj (treats the symptom; build output is transient).
|
||||
|
||||
**Implementation.**
|
||||
|
||||
1. `Configuration/GatewayOptionsValidator.cs`: `AddIfNotRooted` → `Path.IsPathRooted`; delete `IsRootedForAnyPlatform` (`:533-560`) and its doc comment.
|
||||
2. `src/ZB.MOM.WW.MxGateway.Server/appsettings.json`: remove the `SqlitePath` and `SnapshotCachePath` lines (code defaults take over).
|
||||
3. New `Configuration/GalaxyRepositoryOptionsValidator.cs` (+ registration next to `AddZbGalaxyRepository` in `GatewayApplication.cs:99-103`) and a `PostConfigure<GalaxyRepositoryOptions>` default for `SnapshotCachePath`.
|
||||
4. Find and fix the offending test per design point 4; `rm` the stray `.../bin/Debug/net10.0/C:\ProgramData\MxGateway\gateway-auth.db`.
|
||||
5. Docs, same commit: `docs/GatewayConfiguration.md` — `SqlitePath` / `SnapshotCachePath` rows now document the derived per-OS default and the rooted-on-this-host rule; `docs/Authentication.md` default-path sentence if it names the Windows literal.
|
||||
6. Tests: `Tests/Configuration/GatewayOptionsValidatorTests.cs` — on the running platform, a foreign-platform-rooted path fails (`C:\x\y.db` on Unix; assert via the injected message), a host-rooted path passes, a bare filename fails; new `GalaxyRepositoryOptionsValidatorTests` — `PersistSnapshot=true` + blank/relative path → invalid, disabled persistence skips the check, blank path gets the PostConfigure default.
|
||||
|
||||
Note for the executor: this pass touches the same `ValidateDashboard`-adjacent validator file as still-open **SEC-23** (cookie/`__Host-`/`AutoLoginUser` coverage) — if SEC-23 is picked up this cycle, land them as one validator commit.
|
||||
|
||||
**Verification.** `dotnet build src/ZB.MOM.WW.MxGateway.Server` (on macOS this also proves the shipped config still boots validation-clean after the literal removal) then:
|
||||
```
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~GatewayOptionsValidator"
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~GalaxyRepositoryOptionsValidator"
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~GatewayTreeHygiene"
|
||||
```
|
||||
Post-run, verify no new `C:\*` file exists under any `bin/` (manual `find src -name 'C:*'`).
|
||||
|
||||
---
|
||||
|
||||
## SEC-34 — Verification cache: expiry outlives TTL; `Invalidate` races in-flight repopulation `Low` · `P2`
|
||||
|
||||
**Finding.** Three bounded staleness windows in `src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/CachingApiKeyVerifier.cs`: (1) a CLI-issued revoke (separate process) keeps authenticating ≤ TTL — documented (`:41-47`); (2) a key whose `ExpiresUtc` passes while cached keeps working ≤ 15 s past expiry (`SecurityOptions.cs:21`) — expiry is enforced by the inner library verifier, which a cache hit never reaches, and this case is **not** in the documented backstop rationale; (3) race: a `VerifyAsync` that passed the inner verifier just before a dashboard revoke can `_cache.Set` (`:110-117`) *after* `Invalidate(keyId)` ran (`:123-137` — no generation check), re-caching the revoked identity for a full TTL despite the "gateway-initiated mutations take effect immediately" contract (`:11-13`).
|
||||
|
||||
**Impact.** All three are bounded (≤ 15 s; ~30 s worst case for the race), so exposure is small — but (2) silently contradicts the SEC-10 expiry feature's semantics, and (3) contradicts the class's own stated contract, which is the kind of gap the next review flags as "documented behavior is false".
|
||||
|
||||
**Design.**
|
||||
|
||||
- **Expiry (window 2): eliminate, don't document.** The shared `ApiKeyIdentity` (0.1.4) carries `ExpiresUtc`; when caching a success, cap the entry lifetime at the key's expiry: `AbsoluteExpiration = min(now + ttl, ExpiresUtc)` (skip caching entirely if already ≤ now). A cached hit can then never outlive the key. Confirm during implementation that the library verifier populates `ExpiresUtc` on the returned identity; if it does not, fall back to documenting the ≤ TTL window in the remarks and `docs/Authentication.md` and file a donor-library ask.
|
||||
- **Invalidate race (window 3): per-key generation check.** `ConcurrentDictionary<string, long> _generations`; `Invalidate(keyId)` increments the generation **before** evicting cache keys. `VerifyAsync` parses the key id from the token up front (same split the interceptor does — cheap, no store access), snapshots `g0` before calling the inner verifier, and after a success only `Set`s when the generation still equals `g0` — then re-reads the generation after the `Set` and self-evicts if it moved (bump-before-evict + set-then-recheck closes the remaining interleaving). Unparseable tokens skip caching already (`TryComputeCacheKey`).
|
||||
- **CLI window (1): accept and keep documented** — cross-process invalidation is out of scope by design; the TTL is the backstop and the remarks already say so.
|
||||
|
||||
*Rejected alternatives:* a lock around verify+set (serializes the auth hot path the cache exists to speed up); switching to a distributed/shared cache (massive machinery for a 15 s window); "accept and document" for the race (viable and cheap, but the generation counter is ~20 lines and makes the stated contract true — prefer honest code over honest caveats).
|
||||
|
||||
**Implementation.**
|
||||
|
||||
1. `Security/Authentication/CachingApiKeyVerifier.cs`: expiry-capped `MemoryCacheEntryOptions` in `VerifyAsync` (`:110-117`); `_generations` + snapshot/recheck logic; `Invalidate` bumps generation first (`:123-137`). Update the remarks (`:29-47`): add the expiry cap and the generation mechanism; keep the CLI-window caveat.
|
||||
2. Docs, same commit: `docs/Authentication.md` hot-path caching section (~`:92-105`) — one sentence each for the expiry cap and the invalidate-vs-in-flight guarantee.
|
||||
3. Tests, `src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs`:
|
||||
- `CacheEntry_DoesNotOutliveKeyExpiry` — identity expiring 5 s from now with a 15 s TTL: hit at +4 s served from cache, attempt at +6 s reaches the inner verifier.
|
||||
- `AlreadyExpiredIdentity_IsNotCached` — inner returns success with `ExpiresUtc` ≤ now (defensive); nothing is cached.
|
||||
- `Invalidate_DuringInFlightVerification_DiscardsStaleRepopulation` — inner verifier gated on a `TaskCompletionSource`; call `Invalidate(keyId)` while `VerifyAsync` is awaiting inner, then release; assert the follow-up `VerifyAsync` calls the inner verifier again (no stale cache hit).
|
||||
|
||||
**Verification.** `dotnet build src/ZB.MOM.WW.MxGateway.Server` then:
|
||||
```
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~CachingApiKeyVerifier"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## SEC-35 — Production hard-stops key on the exact `Production` environment name `Info` · `—` (N/A: doc-only)
|
||||
|
||||
**Finding.** The `DisableLogin` and plaintext-LDAP guards fire only when `IHostEnvironment.IsProduction()` (`src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs:23-27,161-165,314-318`) — i.e. `ASPNETCORE_ENVIRONMENT` unset (defaults to Production; the NSSM-deployed hosts are covered) or exactly `Production`. A host launched as `Staging`, `Prod`, or any custom name silently keeps the permissive dev posture. The fallback-environment wiring (`Configuration/GatewayConfigurationServiceCollectionExtensions.cs:24,38-51`) is correct.
|
||||
|
||||
**Impact.** None at current deployments; a latent surprise for a future operator who invents an environment name and assumes the guards travel with it.
|
||||
|
||||
**Design.** N/A for code, by deliberate acceptance: the review itself rates this acceptable-as-designed, ASP.NET Core's environment-name semantics are the platform convention, and inverting to "not Development ⇒ production-like" would hard-stop legitimate permissive staging rigs (the shared dev GLAuth is plaintext-only, so a `Staging` host pointed at it would refuse to boot) — a behavior change with real downside for an Info-severity conventions note. The remediation is the documentation contract the review asked for.
|
||||
|
||||
**Implementation.** `docs/GatewayConfiguration.md` (environment/hard-stop area, near `:244-254`): one short paragraph stating that the two production hard-stops key on `IHostEnvironment.IsProduction()` — the exact `Production` name or an unset `ASPNETCORE_ENVIRONMENT` — and that any other environment name retains the permissive dev posture, so production-like deployments must use the literal `Production`. Same-commit rider on whichever SEC-33/SEC-36 change touches that doc first.
|
||||
|
||||
**Verification.** Doc-only; no build. Cross-read against `GatewayOptionsValidator.cs:23-27`.
|
||||
|
||||
---
|
||||
|
||||
## SEC-36 — Committed dev LDAP service-account password: remove from repo and rotate `Low` · `P1` · cross-repo dependency
|
||||
|
||||
**Finding.** The SEC-06 remediation shipped the production transport hard-stop and the env-var override documentation, but the dev LDAP service-account credential itself is still committed at HEAD: `src/ZB.MOM.WW.MxGateway.Server/appsettings.json:29` (`Ldap:ServiceAccountPassword`), with the same value repeated in `glauth.md`'s samples (`:33,65,102-103,135-136,245`) and acknowledged as "should be rotated" by `docs/GatewayConfiguration.md:248`. (Value intentionally not reproduced here.)
|
||||
|
||||
**Impact.** As long as the shared GLAuth instance (`10.100.0.35:3893`) honors this credential, the repo — and its full git history — discloses a live directory service account with LDAP search capability over `dc=zb,dc=local`. Removal alone is insufficient: the value is permanently recoverable from history, so **rotation is the load-bearing half** of the fix.
|
||||
|
||||
**Design.** Rotate first, then remove; give dev a supported secret channel so the removal doesn't break the local login flow.
|
||||
|
||||
- **Rotation (cross-repo).** The GLAuth source of truth is `scadaproj/infra/glauth/` (per `glauth.md`; note `scadaproj` is a shared monorepo — stage explicit paths only). Generate a new service-account secret there, redeploy GLAuth on `10.100.0.35`, and record the rotation (date, not value) in `glauth.md`. The new value must never be committed to *this* repo in any file — `glauth.md`'s samples switch to a `<service-account-password>` placeholder with a pointer at the source of truth.
|
||||
- **Removal.** Delete the `ServiceAccountPassword` line from `appsettings.json`. The validator already fails a blank password when `Ldap.Enabled=true` (`GatewayOptionsValidator.cs:134-137`), so a host without the secret fails fast at startup instead of failing binds at runtime — extend that validation message to name the two supported channels.
|
||||
- **Where the dev secret lives instead.** Two supported channels, both flowing through the existing `MxGateway:Ldap:ServiceAccountPassword` binding:
|
||||
1. **Deployed hosts (unchanged):** the env-var override `MxGateway__Ldap__ServiceAccountPassword` set in the NSSM service environment — already the documented production mechanism (`docs/GatewayConfiguration.md:248`) and already how deployed config works.
|
||||
2. **Dev boxes (new):** .NET user-secrets — add `<UserSecretsId>` to `ZB.MOM.WW.MxGateway.Server.csproj`; `WebApplicationBuilder` loads user-secrets automatically in Development, so `dotnet user-secrets set "MxGateway:Ldap:ServiceAccountPassword" <value>` (value obtained from the GLAuth source of truth) is a one-time per-machine step. Secrets live under the user profile, outside the tree.
|
||||
- **Cutover order** (avoids a broken window): set the env var on the deployed hosts with the *new* value → rotate GLAuth → verify dashboard login on the deployed hosts → land the repo change (removal + docs) → devs set user-secrets on next pull (startup error message tells them exactly what to do). Check the second deployment (`wonder-app-vd03`) for an LDAP binding before assuming it needs the var (its dashboard is disabled; if `Ldap.Enabled=false` there, nothing to do).
|
||||
|
||||
*Rejected alternatives:* committing a placeholder string instead of deleting the key (passes blank-validation with a wrong value and fails later at bind time — a silent misconfiguration where fail-fast is available); `appsettings.Development.json` (still a committed file — same defect relocated); rotating prod-only and keeping the dev credential committed (the finding *is* the committed live dev credential); encrypting the value in config (key-management theater for a dev secret with a working out-of-band channel).
|
||||
|
||||
**Implementation.**
|
||||
|
||||
1. *(cross-repo, first)* Rotate in `scadaproj/infra/glauth/` and redeploy GLAuth; pre-stage `MxGateway__Ldap__ServiceAccountPassword` in the NSSM environment on `10.100.0.48` (and `wonder-app-vd03` only if LDAP is enabled there); verify dashboard login post-rotation.
|
||||
2. `src/ZB.MOM.WW.MxGateway.Server/appsettings.json:29`: delete the `ServiceAccountPassword` entry.
|
||||
3. `src/ZB.MOM.WW.MxGateway.Server/ZB.MOM.WW.MxGateway.Server.csproj`: add `<UserSecretsId>mxaccessgw-server</UserSecretsId>`.
|
||||
4. `Configuration/GatewayOptionsValidator.cs:134-137`: extend the blank-password message to point at user-secrets (dev) and the env-var override (deployed).
|
||||
5. Docs, same commit as 2–4: `docs/GatewayConfiguration.md:248` — replace the "dev-only committed value / should be rotated" wording with the two channels and the rotation note; `glauth.md` — placeholder in all samples (`:33,65,102-103,135-136,245`), rotation recorded, pointer to `scadaproj/infra/glauth/`; `CLAUDE.md`'s `glauth.md` summary line if it references the credential.
|
||||
6. Repo scrub check: `git grep` for the old value across tracked files must return nothing (run manually — do not encode the secret into a test).
|
||||
|
||||
**Verification.** `dotnet build src/ZB.MOM.WW.MxGateway.Server`; then
|
||||
```
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~GatewayOptionsValidator"
|
||||
```
|
||||
(asserts the blank-password validation still fires with the updated message). Manual: with user-secrets set on the dev box, `dotnet run --project src/ZB.MOM.WW.MxGateway.Server/...` and a dashboard `/login` as `multi-role` succeeds against the rotated GLAuth; the deployed-host login re-check from step 1 counts as the production verification. Live-LDAP integration tests (`MXGATEWAY_RUN_LIVE_LDAP_TESTS=1`) only where the GLAuth instance is reachable; otherwise document skipped per the testing matrix.
|
||||
@@ -0,0 +1,251 @@
|
||||
# Language Clients — Remediation Design & Implementation (2026-07-12 cycle)
|
||||
|
||||
Source review: [50-clients.md](../50-clients.md) · Generated: 2026-07-13 · Baseline: `main @ 4f5371f`
|
||||
|
||||
This document turns the 2026-07-12 Language Clients re-review's **new** findings (CLI-35..CLI-45) into buildable remediation entries. Tier assignments come from the [overall roadmap](../00-overall.md). Prior-cycle findings CLI-01..CLI-34 remain tracked in `archreview/remediation/50-clients.md`; the one prior finding this cycle re-activates is **CLI-08**, which is closed by CLI-38 below.
|
||||
|
||||
Operating constraints carried from prior work:
|
||||
|
||||
- **Java builds locally now**: `JAVA_HOME=/opt/homebrew/opt/openjdk@17 gradle test` from `clients/java`. The Gradle build regenerates the tracked `clients/java/**/src/main/generated/**/MxaccessGateway.java` with spurious protobuf-version churn (~64k lines) — **revert it (`git checkout -- clients/java/**/generated`) after every Java build in this plan, because no `.proto` changes here**.
|
||||
- **No `.proto` changes anywhere in this plan** — no contracts regen, no Python `grpcio-tools` regen, no vendored-Rust-proto refresh.
|
||||
- Per-language verification follows CLAUDE.md's Source Update Workflow table: `gofmt` + `go build ./...` + `go test ./...`; `cargo fmt` + `cargo check --workspace` + `cargo test --workspace` + `cargo clippy --all-targets -- -D warnings`; `python -m pytest`; `gradle test`; `dotnet build clients/dotnet/ZB.MOM.WW.MxGateway.Client.slnx` + tests.
|
||||
- **Cross-language behavior findings (CLI-37/38/41/45) are conformance exercises, not five independent fixes**: each Design section below defines one canonical behavior first (citing the wire contract), then per-language implementation. Where the behavior is specified in `docs/CrossLanguageSmokeMatrix.md` or `docs/ClientLibrariesDesign.md`, those docs change in the same commit (CLAUDE.md same-commit rule).
|
||||
- Shared behavior fixtures live under `clients/proto/fixtures/behavior/` (manifest: `clients/proto/fixtures/behavior/manifest.json`, described by `docs/ClientBehaviorFixtures.md`) — new conformance cases land there so all five suites lock in the same bytes.
|
||||
|
||||
## Finding register
|
||||
|
||||
| ID | Sev | Tier | Eff | Dep | Status | Title |
|
||||
|----|-----|------|-----|-----|--------|-------|
|
||||
| CLI-35 | Medium | P0 | S | — | Not started | Python CLI `stream-events` crashes on a ReplayGap |
|
||||
| CLI-36 | Medium | P0 | S | — | Not started | Go CLI `stream-events` silently destroys the ReplayGap signal |
|
||||
| CLI-37 | Medium | P1 | M | CLI-38 | Not started | Status-array validation must branch on `category` per the proto contract (4-vs-1 divergence) |
|
||||
| CLI-38 | Medium | P1 | S | — | Not started | Align .NET/Go/Java on `hresult < 0` — lands prior CLI-08 and cures the design-doc drift |
|
||||
| CLI-39 | Medium | P1 | S | CLI-35..38, CLI-45 | Not started | Bump client versions off the already-published 0.1.2 before the next publish; add registry-collision guard |
|
||||
| CLI-40 | Low | — | M | — | Not started | Port the exact-secret credential scrub to Rust/Java/.NET |
|
||||
| CLI-41 | Low | — | M | — | Not started | Uniform malformed-reply contract for AuthenticateUser/ArchestrAUserToId/AddBufferedItem |
|
||||
| CLI-42 | Low | P1 | S | — | Not started | Document the vendored Rust proto layout (CLI-02's missing doc half) |
|
||||
| CLI-43 | Low | — | S | — | Not started | Java style guide still prescribes "Java 21 preferred" |
|
||||
| CLI-44 | Low | — | S | — | Not started | Go event goroutine can mislabel a genuine terminal error as `ErrSlowConsumer` |
|
||||
| CLI-45 | Low | P1 | M | — | Not started | Standardize CLI credential env-var name and fail fast on missing/empty passwords |
|
||||
|
||||
Cross-domain dependencies: **CLI-35/CLI-36 pair with GWC-25** (gateway emits `oldest_available_sequence = 0` on an empty replay ring — the server-side half of the same reconnect story; the CLI fixes here are independently landable but the end-to-end resume walk in the smoke matrix needs both). **CLI-39 pairs with the publishing process** (`scripts/pack-clients.ps1`, `scripts/tag-go-module.ps1`, Gitea package registry).
|
||||
|
||||
---
|
||||
|
||||
## CLI-35 — Python CLI `stream-events` crashes on a ReplayGap `Medium` · `P0`
|
||||
|
||||
**Finding.** `Session.stream_events()` yields `pb.MxEvent | ReplayGap` since the CLI-15 library work (`clients/python/src/zb_mom_ww_mxgateway/session.py:816-870`; `ReplayGap` dataclass at `clients/python/src/zb_mom_ww_mxgateway/events.py:11`), but the CLI's `_stream_events` (`clients/python/src/zb_mom_ww_mxgateway_cli/commands.py:1098-1106`) feeds every collected item into `_message_dict` (`commands.py:1627-1632`), which calls `google.protobuf.json_format.MessageToDict`. `ReplayGap` is a plain dataclass, not a proto message, so `MessageToDict` raises and the command exits with an error — after having already consumed the stream. No test covers it (`grep -i replay clients/python/tests/test_cli.py` → nothing).
|
||||
|
||||
**Impact.** The operator tool crashes on exactly the resume-after-outage path where it is most needed. Detach-grace and replay retention are on by default, so any `mxgw-py stream-events --after-worker-sequence` resume that outlived the replay ring hits this. P0 per the roadmap (item 1, alongside GWC-25).
|
||||
|
||||
**Design.** Branch on `isinstance(item, ReplayGap)` in the CLI's row formatter and emit the same distinct JSON row the Rust CLI already prints (`clients/rust/crates/mxgw-cli/src/main.rs:1019-1035`): `{"replayGap": {"requestedAfterSequence": <n>, "oldestAvailableSequence": <n>}}` — camelCase, matching `_message_dict`'s `preserving_proto_field_name=False` output for normal events, so the JSON stream stays uniform and the cross-language e2e matrix can compare rows across CLIs. The gap is rendered, never dropped and never re-synthesized into an event (no-synthesized-events invariant). Rejected alternatives: (a) yielding the raw sentinel `MxEvent` from the CLI path — would require bypassing the typed library surface the CLI already uses and reintroduces the "gap looks like a data change" hazard the library deliberately closed; (b) filtering gaps out — silent loss, precisely CLI-36's defect.
|
||||
|
||||
**Implementation.**
|
||||
1. `clients/python/src/zb_mom_ww_mxgateway_cli/commands.py`: import `ReplayGap` from `zb_mom_ww_mxgateway.events`. Add a small row helper, e.g. `_event_row(item)` → `_message_dict(item)` for proto messages, and for `ReplayGap` return `{"replayGap": {"requestedAfterSequence": item.requested_after_sequence, "oldestAvailableSequence": item.oldest_available_sequence}}`.
|
||||
2. `_stream_events` (`commands.py:1106`): change `{"events": [_message_dict(event) for event in events]}` to use `_event_row`.
|
||||
3. New test in `clients/python/tests/test_cli.py`: `test_stream_events_renders_replay_gap` — monkeypatch `GatewayClient.connect` with the existing fake-connect pattern (see `test_cli.py:21-54`); the fake session's `stream_events` yields a `ReplayGap(requested_after_sequence=7, oldest_available_sequence=42)` followed by one normal `pb.MxEvent`; invoke `stream-events` through `CliRunner` and assert exit code 0, the output JSON contains one `replayGap` row with both sequences, and the normal event row follows it. This is the regression the review found missing.
|
||||
4. Docs: extend the ReplayGap section of `docs/CrossLanguageSmokeMatrix.md:33-39` with a per-CLI rendering row (Python: `replayGap` JSON row — same shape as Rust) in the same commit.
|
||||
|
||||
**Verification.** From `clients/python`: `python -m pytest` (targeted first: `python -m pytest tests/test_cli.py -k replay_gap`). No proto change → no regen, no pinned `grpcio-tools` concern.
|
||||
|
||||
---
|
||||
|
||||
## CLI-36 — Go CLI `stream-events` silently destroys the ReplayGap signal `Medium` · `P0`
|
||||
|
||||
**Finding.** The Go library deliberately clears `EventResult.Event` on a gap and populates `EventResult.ReplayGap` (`clients/go/mxgateway/session.go:1036-1042`), but the CLI loop (`clients/go/cmd/mxgw-go/main.go:969-983`) only checks `result.Err` and then formats `result.Event`: JSON mode marshals a nil `*MxEvent` (empty object), text mode prints `0 MX_EVENT_FAMILY_UNSPECIFIED`. The gap's cursors — `requested_after_sequence` and `oldest_available_sequence`, the exact data an operator needs to resume — are discarded.
|
||||
|
||||
**Impact.** The typed signal the library was specifically built to preserve is destroyed one layer up; an operator resuming after an outage sees a meaningless zero row instead of the resume cursor. P0 per the roadmap (item 1, alongside GWC-25 and CLI-35).
|
||||
|
||||
**Design.** Add an `if result.IsReplayGap()` branch to the CLI loop that renders a dedicated gap row, conforming to the Rust CLI's canonical rendering (`mxgw-cli/src/main.rs:1019-1035`): text mode prints `REPLAY_GAP requested_after=<n> oldest_available=<n>`; JSON mode prints `{"replayGap": {"requestedAfterSequence": <n>, "oldestAvailableSequence": <n>}}` (marshal `result.ReplayGap` with `protojson` under a `replayGap` key, which yields exactly those camelCase fields). The gap row counts toward `-limit` like any other emitted row, mirroring Rust's `events` array accounting. Rejected alternative: printing the raw sentinel event as .NET/Java CLIs do — the Go library intentionally clears `Event`, so there is no sentinel left to print; reconstructing one would synthesize an event.
|
||||
|
||||
**Implementation.**
|
||||
1. `clients/go/cmd/mxgw-go/main.go` `runStreamEvents` (loop at `:969-983`): after the `result.Err` check, insert:
|
||||
- `if result.IsReplayGap()` → JSON: wrap `protojson.Marshal(result.ReplayGap)` as `{"replayGap":<bytes>}` (or a tiny struct with a `json.RawMessage`); text: `fmt.Fprintf(stdout, "REPLAY_GAP requested_after=%d oldest_available=%d\n", result.ReplayGap.GetRequestedAfterSequence(), result.ReplayGap.GetOldestAvailableSequence())`; then the shared `count++` / limit check, `continue`.
|
||||
2. New test in `clients/go/cmd/mxgw-go/main_test.go`: `TestRunStreamEventsPrintsReplayGap` — reuse the fake-gateway pattern (`TestRunPingPlainText`, `main_test.go:195`): a fake `StreamEvents` server implementation sends one sentinel `MxEvent` with `replay_gap{requested_after_sequence: 7, oldest_available_sequence: 42}` set, then one normal data event, then returns. Run `stream-events` in text mode and assert stdout contains the typed line `REPLAY_GAP requested_after=7 oldest_available=42` and does **not** contain `0 MX_EVENT_FAMILY_UNSPECIFIED`; run again with `-json` and assert a `"replayGap"` object with both sequences.
|
||||
3. Docs: the same `docs/CrossLanguageSmokeMatrix.md:33-39` per-CLI rendering row as CLI-35 (Go: typed `REPLAY_GAP` line / `replayGap` JSON row) — land the doc row once, covering both findings.
|
||||
|
||||
**Verification.** From `clients/go`: `gofmt -l .` (clean), `go build ./...`, `go test ./...` (targeted first: `go test ./cmd/mxgw-go -run TestRunStreamEventsPrintsReplayGap`).
|
||||
|
||||
---
|
||||
|
||||
## CLI-37 — Status-array validation must branch on `category`, per the proto's own rule `Medium` · `P1`
|
||||
|
||||
**Finding.** The wire contract (`src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_gateway.proto:982-992`) states that `MxStatusProxy.success` "is NOT a boolean … carried verbatim for diagnostics; the authoritative success/failure of the operation is `category` (MX_STATUS_CATEGORY_OK marks success) … Clients should branch on `category`, not on a specific `success` value." Four clients branch only on `success`: Rust `clients/rust/src/error.rs:326` (`status.success == 0` — written fresh for CLI-03), Go `clients/go/mxgateway/status.go:4-6` (`GetSuccess() != 0`), Java `clients/java/zb-mom-ww-mxgateway-client/src/main/java/com/zb/mom/ww/mxgateway/client/MxStatuses.java:26`, Python `clients/python/src/zb_mom_ww_mxgateway/errors.py:141`. .NET requires **both** (`clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxStatusProxyExtensions.cs:11-17`: `Success != 0 && Category is MxStatusCategory.Ok`). A reply with `success = 1, category = COMMUNICATION_ERROR` passes four clients and throws in .NET; `success = 0, category = OK` does the reverse.
|
||||
|
||||
**Impact.** The wire contract and four of five implementations disagree; identical gateway replies produce opposite outcomes per language. This is the same validation function CLI-38 touches — the two must land together as one conformance pass.
|
||||
|
||||
**Design — canonical behavior (this is a conformance exercise, not five fixes).** Per the proto comment — which pre-dates every implementation and records the worker's mapping intent (`success` is the raw 16-bit COM value carried verbatim; `category` is the worker's normalized verdict) — the canonical per-entry rule is:
|
||||
|
||||
> **An `MxStatusProxy` entry represents failure iff `category != MX_STATUS_CATEGORY_OK`. The `success` field is diagnostics only and never participates in the verdict.**
|
||||
|
||||
Edge decisions, uniform across all five: an absent entry (Go `nil`, Java `null`) remains success (nothing to report — preserves existing nil-tolerance); a **present** entry with `MX_STATUS_CATEGORY_UNSPECIFIED` (0) is a **failure** (an unmapped category is not proven OK, and the worker always maps one — an unspecified category on the wire is itself anomalous). Rejected alternatives: (a) declare the proto comment wrong and standardize on `success` — rejected: the comment is the contract, `success` is explicitly documented as a raw non-boolean diagnostic, and rewriting the contract to match four accidental implementations inverts the authority relationship; (b) require both, as .NET does today — rejected: makes `success = 0, category = OK` fail, directly contradicting "the authoritative … is `category`".
|
||||
|
||||
**Implementation** (one cross-client conformance commit, co-landed with CLI-38 since the same `EnsureMxAccessSuccess`-family functions change):
|
||||
1. **Shared fixtures first** — add to `clients/proto/fixtures/behavior/command-replies/`: `write.status-category-error-success-set.reply.json` (protocol OK, one status with `success: 1, category: MX_STATUS_CATEGORY_COMMUNICATION_ERROR` → expected **failure**) and `write.status-category-ok-success-zero.reply.json` (`success: 0, category: MX_STATUS_CATEGORY_OK` → expected **success**); register both in `clients/proto/fixtures/behavior/manifest.json` and describe them in `docs/ClientBehaviorFixtures.md`.
|
||||
2. **.NET** `clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxStatusProxyExtensions.cs:11-17`: `IsSuccess` → `return status.Category is MxStatusCategory.Ok;` (drop the `Success != 0` conjunct); update the XML doc (line 8) accordingly.
|
||||
3. **Go** `clients/go/mxgateway/status.go:4-6`: `StatusSucceeded` → `return status == nil || status.GetCategory() == pb.MxStatusCategory_MX_STATUS_CATEGORY_OK` (import the generated enum); rewrite the function comment.
|
||||
4. **Java** `MxStatuses.java:25-27`: `succeeded` → `return status == null || status.getCategory() == MxStatusCategory.MX_STATUS_CATEGORY_OK;`; fix the class-level Javadoc (`:10-12`) and the `success()` accessor Javadoc (`:46-47`), which currently document the "non-zero success" convention.
|
||||
5. **Python** `clients/python/src/zb_mom_ww_mxgateway/errors.py:140-141`: `if mx_status.success == 0:` → `if mx_status.category != pb.MX_STATUS_CATEGORY_OK:`.
|
||||
6. **Rust** `clients/rust/src/error.rs:326`: `status.success == 0` → `status.category != MxStatusCategory::Ok as i32` (prost stores enums as `i32`); update the doc comment at `:313-314` ("A MXSTATUS_PROXY entry is treated as a failure when its `success` member is `0`") to the category rule; adjust the existing tests at `error.rs:410-452` that construct failing statuses via `success`.
|
||||
7. Per-client tests: wire the two new fixtures into each fixture-driven suite (Go fixture event/reply tests, `clients/python/tests/`, `MxGatewayClientSessionTests.cs`, `MxGatewayClientSessionTests.java`, `clients/rust/tests/client_behavior.rs`), asserting failure/success exactly per the canonical rule.
|
||||
8. Docs same-commit: `docs/ClientLibrariesDesign.md` Typed Command Parity section — extend the reply-validation sentence to state the per-item rule ("per-item `MxStatusProxy` failure iff `category != OK`; `success` is diagnostic only"); `docs/ClientBehaviorFixtures.md` (step 1).
|
||||
|
||||
**Verification.** All five, per CLAUDE.md: `dotnet build clients/dotnet/ZB.MOM.WW.MxGateway.Client.slnx` + `dotnet test` (client tests); `gofmt`/`go build ./...`/`go test ./...` from `clients/go`; `JAVA_HOME=/opt/homebrew/opt/openjdk@17 gradle test` from `clients/java` (**revert the regenerated `MxaccessGateway.java` churn afterward — no `.proto` changed**); `python -m pytest` from `clients/python`; `cargo fmt` + `cargo check --workspace` + `cargo test --workspace` + `cargo clippy --all-targets -- -D warnings` from `clients/rust`.
|
||||
|
||||
---
|
||||
|
||||
## CLI-38 — Align .NET/Go/Java on `hresult < 0` (lands prior CLI-08; cures the doc drift) `Medium` · `P1`
|
||||
|
||||
**Finding.** `docs/ClientLibrariesDesign.md:118-119` (the Typed Command Parity section added with CLI-04) claims every client "runs the same MXAccess-level reply validation (HRESULT `< 0` + per-item `MxStatusProxy`)". Actual: .NET `clients/dotnet/ZB.MOM.WW.MxGateway.Client/MxCommandReplyExtensions.cs:34` (`reply.HasHresult && reply.Hresult != 0`), Go `clients/go/mxgateway/errors.go:187` (`reply.Hresult != nil && reply.GetHresult() != 0`), Java `clients/java/zb-mom-ww-mxgateway-client/.../MxGatewayErrors.java:50` (`reply.hasHresult() && reply.getHresult() != 0`) all use `!= 0`. Only Python (`errors.py:133`) and Rust (`error.rs:325`) implement the documented `< 0`.
|
||||
|
||||
**Impact.** A parity-preserving reply carrying a positive COM success code (`S_FALSE = 1`) errors in three languages and succeeds in two — and every CLI-04 helper (WriteSecured, AuthenticateUser, …) inherits the divergence. The load-bearing design doc describes behavior three clients don't have.
|
||||
|
||||
**Design.** **This finding is the vehicle that finally closes prior-cycle CLI-08** — the fix was fully designed in `archreview/remediation/50-clients.md` (CLI-08 section) two cycles ago and never landed; landing it now is strictly better than the alternative (correcting the doc to describe the divergence), because it makes the already-written doc true, closes CLI-08 and CLI-38 simultaneously, and completes the COM-correct semantics Rust/Python already have. Canonical rule, per COM semantics and the existing doc: **HRESULT failure iff `hresult` is present and `< 0`** (negative = failure; positive success codes like `S_FALSE` pass). Rejected alternative: doc-only correction — leaves a real cross-client behavioral divergence in the shipped typed surface and keeps CLI-08 open for a third cycle. Co-land with CLI-37: same functions, one conformance commit.
|
||||
|
||||
**Implementation.**
|
||||
1. **.NET** `MxCommandReplyExtensions.cs:34`: `bool hResultFailure = reply.HasHresult && reply.Hresult < 0;`
|
||||
2. **Go** `errors.go:187`: `if reply.Hresult != nil && reply.GetHresult() < 0 {`
|
||||
3. **Java** `MxGatewayErrors.java:50`: `if (reply.hasHresult() && reply.getHresult() < 0) {`
|
||||
4. **Shared fixtures**: add `clients/proto/fixtures/behavior/command-replies/write.hresult-s-false.reply.json` (protocol OK, `hresult: 1` → expected **success**) and `write.hresult-e-fail.reply.json` (`hresult: -2147467259` /0x80004005/ → expected **failure**) to the behavior manifest; wire into all five suites (Python/Rust already assert `< 0` in unit tests — the fixture locks it cross-client).
|
||||
5. Docs/tracking same-commit: no change needed to `docs/ClientLibrariesDesign.md:118-119` — the change makes it true. Update the remediation tracking (`archreview/remediation/00-tracking.md` CLI-08 row → Done, noting it landed via CLI-38) and note the corrected semantics in the .NET/Go/Java README error sections (per the prior CLI-08 design).
|
||||
|
||||
**Verification.** Per language: `dotnet build clients/dotnet/ZB.MOM.WW.MxGateway.Client.slnx` + client tests (new case: `hresult = 1` passes, negative fails); `go test ./...` from `clients/go`; `JAVA_HOME=/opt/homebrew/opt/openjdk@17 gradle test` from `clients/java` (revert generated churn — no `.proto` changed). Python/Rust suites run unchanged as regression (`python -m pytest`; `cargo test --workspace`).
|
||||
|
||||
---
|
||||
|
||||
## CLI-39 — Bump versions off the already-published 0.1.2; guard the publish pipeline `Medium` · `P1`
|
||||
|
||||
**Finding.** All four non-Java clients pin the *already-published* 0.1.2: `clients/rust/Cargo.toml:3` (`[package]`) and `:28` (`[workspace.package]`, inherited by `crates/mxgw-cli` via `version.workspace = true`), `clients/python/pyproject.toml:9` + `clients/python/src/zb_mom_ww_mxgateway/version.py:3`, `clients/go/mxgateway/version.go:7`, `clients/dotnet/ZB.MOM.WW.MxGateway.Client/ZB.MOM.WW.MxGateway.Client.csproj:22`. Java is at 0.2.0 (`clients/java/build.gradle:16`). Since the 0.1.2 publish, the public API changed incompatibly: Rust's `EventStream` item type became `Result<EventItem, Error>` and `Error` gained a variant; Python's `stream_events` yields `MxEvent | ReplayGap`; Go's `EventResult` gained `ReplayGap` and the `Events()` overflow contract changed to a terminal `ErrSlowConsumer`. The next `pack-clients.ps1` run either collides with the existing registry artifact or ships a different API under an identical version string.
|
||||
|
||||
**Impact.** Registry rejection at best; at worst an identical version labeling two different APIs for anyone who cached 0.1.2. Release hygiene for the whole client family.
|
||||
|
||||
**Design.** Bump **all five clients to 0.2.0** before the next publish. Rationale: the stream-surface changes are breaking, so under 0.x semver conventions (cargo treats 0.x minor as breaking) the right bump is minor, not patch — and 0.2.0 aligns every client on one number (Java is already there), simplifying the cross-language matrix and the e2e runner. Landing order: this is the **last** entry in the P0/P1 train — bump after CLI-35/36/37/38/45 so the published 0.2.0 carries the conformant behavior, not a third behavior set. Additionally, close the two publish-pipeline gaps this cycle re-confirmed: the tag-time Go version guard designed for CLI-21 but never implemented, and a registry-collision check in `scripts/pack-clients.ps1` (coordinate with the existing publish process: Gitea registry, credentials in `~/.zshenv`, Go module tags via `scripts/tag-go-module.ps1` with `clients/go/vX.Y.Z` prefixed tags). Rejected alternatives: (a) 0.1.3 patch bump — mislabels breaking changes and cargo consumers with `^0.1` would auto-upgrade into them; (b) per-language different bumps (0.2.0 Rust/Python, 0.1.3 Go/.NET) — legal but forfeits the alignment that makes the matrix and publish script auditable.
|
||||
|
||||
**Implementation.**
|
||||
1. `clients/rust/Cargo.toml:3` and `:28`: `version = "0.2.0"` (both `[package]` and `[workspace.package]`; `crates/mxgw-cli` inherits; `CLIENT_VERSION` derives via `env!("CARGO_PKG_VERSION")` — no other Rust edit).
|
||||
2. `clients/python/pyproject.toml:9` and `clients/python/src/zb_mom_ww_mxgateway/version.py:3`: `0.2.0` in both. Add/extend a test asserting `version.__version__` equals the version parsed from `pyproject.toml` (closes the CLI-26 residual drift mode the review noted — `test_cli.py:213` currently only asserts self-consistency).
|
||||
3. `clients/go/mxgateway/version.go:7`: `ClientVersion = "0.2.0"`. The module tag `clients/go/v0.2.0` is created at publish time via `scripts/tag-go-module.ps1`.
|
||||
4. `clients/dotnet/ZB.MOM.WW.MxGateway.Client/ZB.MOM.WW.MxGateway.Client.csproj:22`: `<Version>0.2.0</Version>`.
|
||||
5. `clients/java/build.gradle:16`: stays `0.2.0` **only after verifying** (Gitea packages API) that Java 0.2.0 was not already published with the pre-remediation API; if it was, bump Java to 0.2.1 and record the exception in the packaging doc.
|
||||
6. `scripts/tag-go-module.ps1`: implement the CLI-21 guard that was designed but skipped — after semver validation, read `clients/go/mxgateway/version.go` and fail the tag when `ClientVersion` ≠ the tag version (strip the `v`).
|
||||
7. `scripts/pack-clients.ps1`: before each per-language publish step, query the Gitea package API for the target name+version and abort with a clear error when the artifact already exists (never force-overwrite). Reuse the verify-API calls the publish workflow already uses post-push.
|
||||
8. Docs same-commit: grep `0\.1\.2` across `docs/ClientPackaging.md` and the five client READMEs; update any stale version references; note the version-bump-before-publish rule in `docs/ClientPackaging.md`'s versioning section.
|
||||
|
||||
**Verification.** Per-language builds prove the constants compile: `cargo check --workspace` + the existing `version.rs` test; `python -m pytest` (new pyproject-vs-`__version__` test); `go build ./...` + `go test ./...`; `dotnet build clients/dotnet/ZB.MOM.WW.MxGateway.Client.slnx`; `JAVA_HOME=/opt/homebrew/opt/openjdk@17 gradle test` only if `build.gradle` changes (revert generated churn). Script checks: `pwsh -NoProfile -Command` dry-run of `tag-go-module.ps1` with a mismatched version (must fail) and a matched one (must pass); `pack-clients.ps1` guard exercised against the existing 0.1.2 artifacts (must refuse) — actual publish happens only when the release train is cut.
|
||||
|
||||
---
|
||||
|
||||
## CLI-40 — Port the exact-secret credential scrub to Rust/Java/.NET `Low` · `—`
|
||||
|
||||
**Finding.** The credential-redaction seam is uniform in name only. Go scrubs the *exact secret values* from any surfaced error (`clients/go/mxgateway/errors.go:16-66` `secretRedactingError`/`redactSecrets`, applied at `session.go:745`/`:836` for WriteSecured payloads and the AuthenticateUser password); Python does the same (`session.py:573-592` `_invoke_redacted`). Rust scrubs only whitespace-delimited tokens shaped like API keys (`clients/rust/src/error.rs:363-375` `redact_credentials`: `mxgw_` prefix or `bearer`); Java likewise (`clients/java/zb-mom-ww-mxgateway-client/.../MxGatewaySecrets.java:44-57`); .NET has **no library-level scrub** (exception text embeds `status.DiagnosticText` verbatim via `MxStatusProxyExtensions.ToDiagnosticSummary`; only the CLI redacts, `MxGatewayCliSecretRedactor.cs`). Yet `docs/ClientLibrariesDesign.md:123-127` and the Rust helper docs (`session.rs:876-883`) claim credentials can never reach exception text. The per-client tests pass because the fakes never echo the credential.
|
||||
|
||||
**Impact.** If the gateway or MXAccess echoes a credential back in `diagnostic_text` (which the client cannot prevent), Rust/Java/.NET surface it verbatim in exception messages — contradicting the documented guarantee. Low severity (requires a server-side echo), but the doc claims more than three clients deliver.
|
||||
|
||||
**Design — canonical behavior.** Every credential-bearing helper (`AuthenticateUser` password, `WriteSecured`/`WriteSecured2` string payloads) must scrub the **exact secret values** it was called with from any error text it surfaces, replacing each occurrence with `<redacted>` (the marker all five already use). This is defense-in-depth on top of the by-construction guarantee (exceptions carry reply-derived text, never the request). Go and Python define the reference semantics: wrap/post-process the error at the credential-bearing call site; typed-error matching (`errors.Is`/`except`/`catch`) must keep working through the redaction. Rejected alternative: downgrade the doc claim to "requests are never embedded in errors" — weaker guarantee than two clients already deliver and than the docs have promised since CLI-04; porting the seam is bounded work.
|
||||
|
||||
**Implementation.**
|
||||
1. **Rust**: extend `MxAccessError` (and the error path used by credential helpers) to carry `secrets: Vec<String>`; in its `Display` (which already funnels through `redact_credentials`, `error.rs:363`), first replace each exact secret occurrence with `<redacted>`, then apply the existing pattern scrub. Credential helpers in `clients/rust/src/session.rs` (`authenticate_user`, `write_secured`, `write_secured2`) attach the secrets when mapping errors. Keep the helper-doc claim at `session.rs:876-883` — it becomes true.
|
||||
2. **Java**: add `MxGatewaySecrets.redactExact(String message, String... secrets)` (exact-substring replacement, null/blank-tolerant) alongside the pattern scrub; route the credential commands in `MxGatewaySession.java` (`authenticateUser`, `writeSecured`, `writeSecured2`) through a private `invokeCommandRedacted(command, String... secrets)` that catches `MxGatewayException`, rebuilds the message via `redactExact`, and rethrows the **same exception type** (mirror Python's `_invoke_redacted`).
|
||||
3. **.NET**: introduce a library-level equivalent in `MxGatewaySession.cs`: the credential helpers (`AuthenticateUserAsync`, `WriteSecuredAsync`, `WriteSecured2Async`) wrap their invoke+validate in a catch that rethrows the same exception type with each exact secret replaced by `<redacted>` in `Message` (reuse the `<redacted>` marker; factor the replacement into a small internal `MxGatewaySecretRedaction` helper so the CLI's `MxGatewayCliSecretRedactor` can share it).
|
||||
4. **Shared fixture**: add `clients/proto/fixtures/behavior/command-replies/authenticate-user.echoed-credential.reply.json` — an MXAccess-failure reply whose `diagnostic_text` embeds the fixture credential string; every client's suite asserts the surfaced error message does **not** contain the credential and does contain `<redacted>`. Register in the manifest + `docs/ClientBehaviorFixtures.md`.
|
||||
5. Docs: none beyond the fixture doc — the design-doc claim becomes accurate.
|
||||
|
||||
**Verification.** `cargo fmt`/`check`/`test`/`clippy -D warnings` from `clients/rust`; `JAVA_HOME=/opt/homebrew/opt/openjdk@17 gradle test` from `clients/java` (revert generated churn); `dotnet build clients/dotnet/ZB.MOM.WW.MxGateway.Client.slnx` + client tests; Go/Python suites re-run the new shared fixture as regression.
|
||||
|
||||
---
|
||||
|
||||
## CLI-41 — Uniform malformed-reply contract for the id/handle-returning helpers `Low` · `—`
|
||||
|
||||
**Finding.** A gateway reply that omits the typed payload yields five different behaviors. `AuthenticateUser`: Go falls back to `GetReturnValue().GetInt32Value()` → silent `0` when both are absent (`clients/go/mxgateway/session.go:807-816`); Python reads `reply.authenticate_user.user_id` → proto3 default `0`, no fallback, no error (`clients/python/src/zb_mom_ww_mxgateway/session.py:713`); Java falls back like Go → `0` (`MxGatewaySession.java:868-880`); .NET `reply.AuthenticateUser?.UserId ?? reply.ReturnValue.Int32Value` → `NullReferenceException` when both are absent (`clients/dotnet/.../MxGatewaySession.cs:1289`); Rust returns a typed `Error::MalformedReply` (`clients/rust/src/session.rs:1073-1089`) — but Rust's own `add_buffered_item_handle` (`:1057-1071`) *does* fall back to `return_value`, so Rust is inconsistent internally.
|
||||
|
||||
**Impact.** Silent `0` from `AuthenticateUser` is the worst mode — callers feed it into `WriteSecured` as a real user id. The NRE is a crash on a malformed-but-received reply.
|
||||
|
||||
**Design — canonical behavior.** For every helper that extracts a scalar from a reply (`AuthenticateUser`, `ArchestrAUserToId`, `AddBufferedItem`, and the existing `AddItem2`-style handle extractors):
|
||||
|
||||
> **Prefer the typed payload; when it is absent, fall back to `return_value` only if `return_value` is actually present with the expected variant; when neither is present, raise/return a typed malformed-reply error. Never surface a proto3 default `0`, never NRE.**
|
||||
|
||||
This is Rust's `add_buffered_item_handle` pattern (payload → present `return_value` → `MalformedReply`), generalized. The `return_value` fallback is kept because it is the documented compatibility path for replies from workers that populate only the legacy field; the typed error replaces only the "both absent" case. Rejected alternatives: (a) Rust's stricter payload-or-error (no fallback, current `authenticate_user_id`) — breaks the legacy-`return_value` compatibility the other four clients and Rust's own handle extractors honor; (b) documenting the divergence — a silent-`0` user id is a correctness hazard, not a doc gap.
|
||||
|
||||
**Implementation.**
|
||||
1. **Go** `clients/go/mxgateway/session.go`: add a `MalformedReplyError{Op string, Detail string}` type to `errors.go` (Error/Unwrap in the house style). In `AuthenticateUser` (`:807`), `ArchestrAUserToId`, and `AddBufferedItem`: check the typed payload; else `if reply.ReturnValue != nil` use it; else return the typed error.
|
||||
2. **Python** `clients/python/src/zb_mom_ww_mxgateway/session.py` + `errors.py`: add `MalformedReplyError(MxGatewayError)`. In `authenticate_user` (`:713`), `archestra_user_to_id`, `add_buffered_item`: `if reply.HasField("authenticate_user")` → typed payload; `elif reply.HasField("return_value")` and the value oneof is `int32_value` → fallback; else raise.
|
||||
3. **Java** `MxGatewaySession.java:868-880` (+ `archestrAUserToId`, `addBufferedItem`): `if (reply.hasAuthenticateUser())` → payload; `else if (reply.hasReturnValue())` → fallback; else throw a new `MxGatewayMalformedReplyException extends MxGatewayException`.
|
||||
4. **.NET** `MxGatewaySession.cs:1289` (+ the archestra/buffered equivalents at `:1332`/`:935`): `reply.AuthenticateUser?.UserId ?? reply.ReturnValue?.Int32Value ?? throw new MxGatewayMalformedReplyException(...)` (new exception type deriving from `MxGatewayException`).
|
||||
5. **Rust** `clients/rust/src/session.rs:1073-1089`: add the `return_value` fallback to `authenticate_user_id` and `archestra_user_id`, matching `add_buffered_item_handle` — payload → `return_value` via `int32_reply_value` → `Error::MalformedReply`.
|
||||
6. **Shared fixtures**: `clients/proto/fixtures/behavior/command-replies/authenticate-user.missing-payload.reply.json` (protocol OK, no payload, no `return_value` → expected typed error) and `authenticate-user.return-value-only.reply.json` (`return_value.int32_value = 7`, no typed payload → expected `7`); manifest + `docs/ClientBehaviorFixtures.md` + all five suites.
|
||||
7. Docs: one sentence in `docs/ClientLibrariesDesign.md`'s Typed Command Parity section stating the payload → return_value → typed-error contract.
|
||||
|
||||
**Verification.** All five per CLAUDE.md: dotnet slnx build + tests; `go test ./...`; `JAVA_HOME=/opt/homebrew/opt/openjdk@17 gradle test` (revert generated churn); `python -m pytest`; `cargo fmt`/`check`/`test`/`clippy -D warnings`.
|
||||
|
||||
---
|
||||
|
||||
## CLI-42 — Document the vendored Rust proto layout (CLI-02's missing doc half) `Low` · `P1 (doc batch)`
|
||||
|
||||
**Finding.** CLI-02's code fix landed (repo-path-first/vendored-fallback `build.rs`, `clients/rust/protos/*.proto` vendored and `include`d in `Cargo.toml:20`, `--no-verify` removed from packaging) but the documentation half was skipped: `clients/rust/README.md:21-25` still says `build.rs` reads only `../../src/ZB.MOM.WW.MxGateway.Contracts/Protos`, and `docs/ClientPackaging.md`'s Rust section (`:114-127`) never mentions `clients/rust/protos/` (repo-wide grep for "vendored" in `docs/*.md` → zero hits). The vendored copies are load-bearing published build inputs with a refresh obligation on every `.proto` change, currently enforced only by `scripts/check-codegen.ps1` Check 3 and a `build.rs` comment. CLAUDE.md's docs-in-same-commit rule was not met.
|
||||
|
||||
**Impact.** Anyone editing the contracts protos has no prose telling them the Rust crate ships its own copies that must be refreshed; the packaging doc describes a crate layout that no longer matches what `cargo package` ships.
|
||||
|
||||
**Design.** Pure documentation: describe the resolution order and the refresh rule in both places the review names. No code. Batched with the other P1 doc work.
|
||||
|
||||
**Implementation.**
|
||||
1. `clients/rust/README.md:21-25`: rewrite the generation paragraph — `build.rs` resolves protos repo-path-first (`../../src/ZB.MOM.WW.MxGateway.Contracts/Protos`, keeping in-repo `.proto` edits live) and falls back to the vendored copies in `clients/rust/protos/` (shipped in the published crate via `Cargo.toml`'s `include`, making the tarball self-sufficient — this is what `cargo package` verification exercises). State the refresh rule: any change to the Contracts protos must copy the changed files into `clients/rust/protos/` in the same commit; `scripts/check-codegen.ps1` (Check 3) fails on byte drift.
|
||||
2. `docs/ClientPackaging.md` Rust section (after the `build.rs` sentence around `:116-118`): add the same vendored-layout paragraph plus a note that `cargo package`/`cargo publish` run **with** verification (no `--no-verify`) precisely because the vendored protos make the standalone build possible.
|
||||
|
||||
**Verification.** `grep -rn "protos/" clients/rust/README.md docs/ClientPackaging.md` shows the vendored dir documented in both; `grep -i vendored docs/ClientPackaging.md clients/rust/README.md` non-empty. No build required (docs only).
|
||||
|
||||
---
|
||||
|
||||
## CLI-43 — Java style guide still prescribes "Java 21 preferred" `Low` · `—`
|
||||
|
||||
**Finding.** `docs/style-guides/JavaStyleGuide.md:8` — "Target the Java version defined by the client build, with Java 21 preferred." CLAUDE.md declares `docs/style-guides/` authoritative, and this line contradicts the shipped `toolchain 17` / `options.release = 17` build (`clients/java/build.gradle`) and the CLI-12-corrected docs. (Historical plan docs also say 21 but are archival — leave them.)
|
||||
|
||||
**Impact.** An authoritative guide instructing contributors toward the wrong language level for the Ignition 8.3 target.
|
||||
|
||||
**Design.** One-line doc fix, mirroring CLI-12's wording. No code.
|
||||
|
||||
**Implementation.**
|
||||
1. `docs/style-guides/JavaStyleGuide.md:8`: replace with "Target Java 17 (the Ignition 8.3 baseline; the client build enforces `options.release = 17` with a Gradle toolchain 17). Code must compile and run on 17; newer JDKs may host the build."
|
||||
|
||||
**Verification.** `grep -rn "Java 21" docs/style-guides/` → empty. No build (docs only).
|
||||
|
||||
---
|
||||
|
||||
## CLI-44 — Go event goroutine can mislabel a genuine terminal error as `ErrSlowConsumer` `Low` · `—`
|
||||
|
||||
**Finding.** When `stream.Recv()` returns a real terminal error while the 16 data slots are full, the Recv-error path (`clients/go/mxgateway/session.go:1051-1057`) reuses `sendEventResult`, whose overflow branch (`:1082-1101`) fires first: it substitutes `ErrSlowConsumer` for the caller's `GatewayError{Err: err}`. The consumer gets a loud terminal error (good) with the wrong root cause — the real gRPC status is lost.
|
||||
|
||||
**Impact.** Misleading diagnostics on a narrow race (full buffer + genuine stream failure), not silent loss — hence Low.
|
||||
|
||||
**Design.** Terminal sends must bypass the overflow substitution and use the reserved slot directly with the caller's error. The reserved slot (`eventBufferReservedSlots`, `session.go:47-56`) exists exactly to guarantee one terminal result is always deliverable; the sole-producer invariant (CLI-01's design) means at most one terminal send ever races for it. Rejected alternative: enlarging the reserve to two slots so both errors can be delivered — the consumer only ever acts on the first terminal error; delivering two contradicting terminal results complicates the contract for no diagnostic gain.
|
||||
|
||||
**Implementation.**
|
||||
1. `clients/go/mxgateway/session.go`: add a small `sendTerminalEventResult(results chan<- EventResult, result EventResult)` doing a non-blocking `select { case results <- result: default: }` (comment: the reserved slot guarantees the send unless a terminal result was already enqueued). Use it in the Recv-error path (`:1051-1057`) instead of `sendEventResult` (the stream has already ended — no `cancel()` needed beyond the deferred cleanup; keep the existing `cancel` call if one is required for stream teardown symmetry). The overflow branch in `sendEventResult` is unchanged.
|
||||
2. New test in `clients/go/mxgateway/client_session_test.go`: `TestEventsFullBufferTerminalErrorKeepsRootCause` — fill all 16 data slots without draining (reuse the CLI-01 slow-consumer test scaffolding at `:150-185`), then make the fake stream's `Recv` return a non-EOF gRPC error; drain and assert the final `EventResult.Err` unwraps to the gRPC status error and `errors.Is(err, ErrSlowConsumer)` is **false**.
|
||||
3. Docs: adjust the `Events`/`EventsAfter` doc comments if they state that a full buffer always yields `ErrSlowConsumer` (the contract becomes: overflow yields `ErrSlowConsumer`; a genuine stream error is reported as itself even under overflow).
|
||||
|
||||
**Verification.** From `clients/go`: `gofmt -l .`, `go build ./...`, `go test ./...` (targeted: `go test ./mxgateway -run TestEventsFullBufferTerminalErrorKeepsRootCause`).
|
||||
|
||||
---
|
||||
|
||||
## CLI-45 — Standardize CLI credential env-var name; fail fast on missing/empty passwords `Low` · `P1`
|
||||
|
||||
**Finding.** The new `authenticate-user` CLI credential flags diverge per language. Go (`clients/go/cmd/mxgw-go/main.go:441-457`): `-password`/`-password-env`, default env `MXGATEWAY_VERIFY_PASSWORD`, but an unresolved (empty) password is **silently sent to the wire**. Java (`clients/java/zb-mom-ww-mxgateway-cli/.../MxGatewayCli.java:1136-1160`): same flag/env names, and an unset env falls back to `resolvedPassword = ""` — also sent as an empty credential. Rust (`clients/rust/crates/mxgw-cli/src/main.rs:136-155`): same names, missing → `Error::InvalidArgument` (conformant). Python (`commands.py:832-847` `_resolve_password`): no default env name — `--password-env` must be passed explicitly; missing → `click.UsageError`. .NET (`MxGatewayClientCli.cs:349-388`): different flag names (`--verify-user-password`/`--verify-user-password-env`) and a different default env (`MXGATEWAY_VERIFY_USER_PASSWORD`); missing → `ArgumentException`. Subcommand coverage also diverges (.NET exposes all nine new commands; Rust adds unregister + the two credential commands; Go/Python/Java add only `write-secured` + `authenticate-user`).
|
||||
|
||||
**Impact.** The same operator workflow (`export MXGATEWAY_VERIFY_PASSWORD=… && <cli> authenticate-user …`) works in three CLIs, needs an explicit flag in Python, and needs a different variable in .NET — and Go/Java silently authenticate with an empty password, turning a misconfigured env into a real (failing or, worse, succeeding) MXAccess authentication attempt.
|
||||
|
||||
**Design — canonical behavior.** One CLI credential contract for all five:
|
||||
|
||||
> Flags `--password` (explicit value; discouraged — stays out of shell history via the env path) and `--password-env` (name of the environment variable; **default `MXGATEWAY_VERIFY_PASSWORD`**). Resolution order: flag, then env. When the resolved credential is **missing or empty**, the CLI fails fast with a usage error naming the flag and the env var — it never sends an empty password to the wire and never echoes the value.
|
||||
|
||||
`MXGATEWAY_VERIFY_PASSWORD` wins as the canonical name because three of five CLIs and their docs already use it — changing .NET (one CLI, with aliases) is the smallest blast radius; rejected alternative: standardizing on .NET's `MXGATEWAY_VERIFY_USER_PASSWORD` would churn three CLIs plus the smoke matrix for no benefit. The fail-fast rule is CLI argument validation, not a parity violation: the *library* still transmits whatever credential it is given (MXAccess parity preserved); only the operator tool refuses to fabricate an empty one — matching what Rust/Python/.NET already do. Subcommand-coverage leveling is out of scope here (larger feature work); the deltas get documented instead.
|
||||
|
||||
**Implementation.**
|
||||
1. **Go** `clients/go/cmd/mxgw-go/main.go` (`runAuthenticateUser`, after the resolution block at `:454-457`): `if resolvedPassword == "" { return fmt.Errorf("a password is required via -password or the %s environment variable", *passwordEnv) }` — before dialing.
|
||||
2. **Java** `MxGatewayCli.java` `AuthenticateUserCommand.call()` (`:1151-1159`): replace the `resolvedPassword = ""` fallback with a picocli `ParameterException` (via `common.spec.commandLine()`) when the resolved value is null or blank, message naming `--password` and the `--password-env` variable.
|
||||
3. **Python** `clients/python/src/zb_mom_ww_mxgateway_cli/commands.py`: give the `--password-env` click option the default `"MXGATEWAY_VERIFY_PASSWORD"` (both `authenticate-user` and any secured-write command that shares the option); `_resolve_password` keeps its `click.UsageError` (already conformant once the default exists).
|
||||
4. **.NET** `MxGatewayClientCli.cs` (`TryResolveVerifyUserPassword`/`ResolveVerifyUserPassword`, `:349-388`): accept `--password` and `--password-env` as the primary names with default env `MXGATEWAY_VERIFY_PASSWORD`; keep `--verify-user-password`, `--verify-user-password-env`, and the `MXGATEWAY_VERIFY_USER_PASSWORD` env as deprecated aliases/fallbacks for one release (resolution order: `--password`, `--verify-user-password`, env named by `--password-env` (default `MXGATEWAY_VERIFY_PASSWORD`), legacy `MXGATEWAY_VERIFY_USER_PASSWORD`). The existing `ArgumentException` fail-fast stays; ensure the empty-string case (flag or env set but empty) also fails.
|
||||
5. **Rust** `mxgw-cli/src/main.rs:143-152`: verify the empty-string env case — if `std::env::var` returns an empty string it must be treated as missing (add the check if absent). Otherwise already conformant.
|
||||
6. Tests (new, per CLI): Go `TestRunAuthenticateUserRejectsEmptyPassword` in `main_test.go` (env unset → error mentioning `MXGATEWAY_VERIFY_PASSWORD`, no RPC attempted — no fake server needed); Java a picocli-level test invoking `authenticate-user` without credential and asserting the usage error; Python a `test_cli.py` case asserting the default env name is honored (monkeypatch `os.environ`) and that missing → `UsageError`; .NET CLI tests asserting the new names, the alias fallback, and the empty-value failure; Rust a case for empty-string env → `InvalidArgument`.
|
||||
7. Docs same-commit: `docs/CrossLanguageSmokeMatrix.md` — document the canonical flag/env contract in the smoke-command section and add a per-CLI subcommand-coverage table noting the current deltas (.NET: all nine; Rust: unregister + credential pair; Go/Python/Java: `write-secured` + `authenticate-user`) so the divergence is at least specified until a leveling pass; client READMEs' `authenticate-user` sections updated where they name the env var (.NET README gains the rename + deprecation note).
|
||||
|
||||
**Verification.** Go: `gofmt -l .`, `go build ./...`, `go test ./...`. Java: `JAVA_HOME=/opt/homebrew/opt/openjdk@17 gradle test` from `clients/java` (**revert the `MxaccessGateway.java` regen churn — no `.proto` changed**). Python: `python -m pytest` from `clients/python`. .NET: `dotnet build clients/dotnet/ZB.MOM.WW.MxGateway.Client.slnx` + client/CLI tests. Rust: `cargo fmt`, `cargo check --workspace`, `cargo test --workspace`, `cargo clippy --all-targets -- -D warnings` from `clients/rust`.
|
||||
@@ -0,0 +1,178 @@
|
||||
# Testing, Documentation & Underdeveloped Areas — Remediation Design & Implementation (2026-07-12 cycle)
|
||||
|
||||
Source review: [60-testing-docs-gaps.md](../60-testing-docs-gaps.md) · Baseline: `main` @ `4f5371f` · Generated: 2026-07-13
|
||||
|
||||
This cycle's new findings (TST-25..29) are all fallout or residue of the prior remediation: the Windows CI tier was removed after act_runner proved broken (TST-25), the docs describing that tier were not updated in the same commit (TST-26), a flag SEC-25 made live is still documented as dead (TST-27), a handshake field is asserted only on the side CI cannot run (TST-28), and the root-artifact triage from TST-14 needs its retirement decision refreshed (TST-29). Every citation below was re-verified against the working tree on 2026-07-13. **TST-30** was added later on 2026-07-13, surfaced while running TST-25's acceptance checks — the CI infrastructure itself (a single shared runner) is a throughput/availability bottleneck.
|
||||
|
||||
Prior-cycle open findings (TST-05..24 where still open) are tracked in the prior design doc, `archreview/remediation/60-testing-docs-gaps.md`; this doc covers only the new findings. Note TST-25 is the unlock for two of those: a working Windows tier is where the scheduled live-MXAccess smoke (old TST-05) and CI-run client wire tests (old TST-24) land.
|
||||
|
||||
## Finding register
|
||||
|
||||
| ID | Sev | Tier | Eff | Dep | Status | Title |
|
||||
|----|-----|------|-----|-----|--------|-------|
|
||||
| TST-25 | High | P1 | M | — (unlocks TST-05, TST-24) | Done | Windows/x86 test tier has zero automation — restore via SSH-driven windev CI job |
|
||||
| TST-26 | Medium | P1 (folded into TST-25) | S | TST-25 | Done | docs/GatewayTesting.md, check-codegen.ps1, and ci.yml comments describe removed CI jobs |
|
||||
| TST-27 | Medium | P1 (doc batch) | S | — | Not started | `ShowTagValues` config row still says "Reserved" after SEC-25 made the flag live |
|
||||
| TST-28 | Low | P2 | S | relates IPC-02 | Not started | Gateway-side `max_frame_bytes` handshake field untested in the CI-run suite |
|
||||
| TST-29 | Low | P2 | S | — | Not started | Retire `oldtasks.md` after folding the Phase-5 governance record into DesignDecisions.md; delete root docs-review artifacts |
|
||||
| TST-30 | Low | P2 | M | — | Not started | Single shared Gitea runner is a CI throughput/availability bottleneck (cross-repo contention, no run cancel/delete) |
|
||||
|
||||
---
|
||||
|
||||
## TST-25 — Windows/x86 test tier has zero automation `High` · `P1` · Effort M
|
||||
|
||||
**Finding.** The `windows` and `live-mxaccess` CI jobs were removed in `abb0930` because Gitea act_runner v0.6.1 host-mode is broken on Windows and a `runs-on`-gated job with no matching runner wedges every run in "queued" (removal note: `.gitea/workflows/ci.yml:123-131`). Consequence: the x86/net48 Worker build, all of `src/ZB.MOM.WW.MxGateway.Worker.Tests` — including the tests for remediation-added machinery that exists nowhere else (`WorkerFrameWritePriority` ordering at `Worker.Tests/Ipc/WorkerFrameProtocolTests.cs:353-395`, worker-side `max_frame_bytes` adoption, WnWrap/subtag alarm consumers, `StaRuntime` changes) — and the 8-fact live suite (`src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs`) run only when someone remembers the manual windev worktree process. The portable job's `check-codegen.ps1` Generated-diff check substitutes for the net48 CS0246 guard; nothing substitutes for actually compiling and running the worker tests.
|
||||
|
||||
**Impact.** A whole tier of *unit* tests — not just live smoke — is unguarded. A worker-side regression (frame priority inversion, frame-max adoption break, alarm-consumer conversion bug, STA loop change) ships green through CI and is caught, if at all, by hand on windev or in production. This regressed the prior cycle's TST-03/TST-05 posture and blocks old TST-24.
|
||||
|
||||
**Design.** Three candidates, weighed against the operational facts (Gitea Actions runs on a co-located `gitea-runner` container on docker host `10.100.0.35`, job containers on network `traefik` resolving `gitea:3000`; windev is Windows 10 at `10.100.0.48`, ssh alias `windev`, passwordless ssh from the Mac, MXAccess installed, and a proven isolated-worktree flow already exists in `C:\build` driven by remote PowerShell via base64 `-EncodedCommand`):
|
||||
|
||||
- **(a) Retry a Windows act_runner in host mode with the right labels — rejected.** The failure is not a label problem: act_runner v0.6.1's hostexecutor writes each step's script to a path it then cannot resolve (independent of shell) and cannot stage JS actions, so no label configuration fixes it; Windows containers are impractical for a net48/x86/MXAccess build. And the structural hazard remains: any `runs-on` gate with no live runner sticks queued forever, so a flaky/offline Windows runner would block *every* run. Revisit only if a fixed act_runner release ships — it would then replace the SSH job with a native one.
|
||||
- **(b) SSH-driven job — recommended.** A Linux CI job (same `ubuntu-latest` runner class as `portable`, so it always schedules) ssh's to windev, fast-forwards an isolated CI clone under `C:\build` to the exact SHA under test, runs the x86 build + `Worker.Tests` there, and propagates the remote exit code through ssh → step → job. This reuses the already-proven windev worktree + `-EncodedCommand` recipe, requires no new runner software on Windows, and cannot wedge the queue (the Linux runner always exists; an unreachable windev fails the job red — loud, not stuck).
|
||||
- **(c) Trigger split — adopted as part of (b).** Worker tests have nontrivial wall time on windev, and the live suite needs provider state and must never gate a push. Split: **per-push/PR** job runs `dotnet build Worker.csproj -p:Platform=x86` + `dotnet test Worker.Tests -p:Platform=x86` (unit suite — expected minutes; if measured wall time exceeds ~10 min, demote the test step to nightly and keep the per-push x86 build, which alone catches the net48/CS0246/x86 compile class); the **nightly** scheduled job (the existing `cron: '0 6 * * *'`, currently pointless) re-runs the x86 build + full Worker.Tests on `main`, then runs the live-MXAccess smoke with `MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`, and optionally a full `src/ZB.MOM.WW.MxGateway.slnx` build (catches Windows-only analyzer diagnostics like xUnit1030 that never surface on macOS/Linux).
|
||||
|
||||
Design details for (b):
|
||||
|
||||
- **Isolation.** A dedicated CI clone `C:\build\mxaccessgw-ci` — never the Desktop checkout (dirty feature branch, per the established windev rule). The job's inline bootstrap (base64 `-EncodedCommand`) does `git fetch origin && git checkout --force <sha>` in that clone, then invokes the *checked-out* `scripts/ci/windev-worker-ci.ps1 -Sha <sha> -Mode <build|test|live>` so the CI logic itself is versioned with the code under test; only the tiny bootstrap lives in ci.yml.
|
||||
- **Serialization.** One clone, potentially overlapping runs: `windev-worker-ci.ps1` acquires a lock (`New-Item C:\build\mxaccessgw-ci.lock -ItemType Directory`, retry loop with ~20 min timeout, always released in `finally`) so concurrent pushes queue instead of stomping the worktree. Add a workflow-level `concurrency` group too if the deployed Gitea version honors it — the lock is the guarantee, `concurrency` the nicety.
|
||||
- **Secrets.** Generate a dedicated ed25519 keypair for CI (do **not** reuse the operator's personal windev key). Private key → Gitea Actions repo secret `WINDEV_SSH_KEY`; pinned host key → secret `WINDEV_SSH_KNOWN_HOSTS` (or a committed `scripts/ci/windev.known_hosts` — host keys are public data; committing is acceptable and greppable). The job writes the key to `~/.ssh/id_ed25519` mode 0600 and connects with `-o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=...`. On windev, append the pub key to `authorized_keys` for the CI account; optionally prefix with a forced `command=` restricting it to the bootstrap script. Per CLAUDE.md, the key must never be echoed into logs.
|
||||
- **Network precheck.** The job container is on the `traefik` docker network; confirm it has egress to `10.100.0.48:22` (a one-off `nc -z -w5 10.100.0.48 22` step during bring-up; docker bridge networks normally route to the LAN, but the network was created for gitea resolution, not egress, so verify rather than assume).
|
||||
- **Failure visibility.** Per-push runs are first-class CI jobs — a red windev run is a red commit/PR check exactly like `portable`/`java`, and an unreachable windev **fails** (never skips) so a down Windows tier is loud. Nightly runs surface on the repo's Actions page against `main`; because nobody watches that page, add a final `if: failure()` step to the nightly job that opens (or comments on) a Gitea issue via the API using the built-in actions token, so a red nightly produces a notification artifact. Document in `docs/GatewayTesting.md` that a red `windows-x86` job means the Windows tier — not the change — may be down, and how to check (ssh windev, read `C:\build\mxaccessgw-ci\ci.log`).
|
||||
|
||||
**Implementation.**
|
||||
|
||||
1. **Windev prep (one-time, manual):** create `C:\build\mxaccessgw-ci` as a fresh clone of the Gitea origin (read-only deploy token or the existing credential store); verify `dotnet build src/ZB.MOM.WW.MxGateway.Worker/ZB.MOM.WW.MxGateway.Worker.csproj -p:Platform=x86` and `dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86` succeed there by hand; record wall time (drives the per-push vs nightly split for the test step). Generate the CI keypair, install the pub key on windev, store `WINDEV_SSH_KEY` (+ known-hosts) in Gitea repo secrets.
|
||||
2. **New `scripts/ci/windev-worker-ci.ps1`** (runs *on windev*, invoked at the tested SHA): parameters `-Sha`, `-Mode build|test|live`; acquires/releases the lock dir; `build` = x86 Worker build; `test` = build + `dotnet test Worker.Tests -p:Platform=x86`; `live` = `test` + `MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1 dotnet test src/ZB.MOM.WW.MxGateway.IntegrationTests/... --filter FullyQualifiedName~WorkerLiveMxAccessSmokeTests` (+ optional full-slnx build); exits nonzero on any failure.
|
||||
3. **New `scripts/ci/run-windev-ci.sh`** (runs *in the Linux job*): writes the key/known-hosts from env, builds the base64 `-EncodedCommand` bootstrap (fetch + checkout `$CI_SHA` in `C:\build\mxaccessgw-ci`, then `powershell -File scripts\ci\windev-worker-ci.ps1 -Sha $CI_SHA -Mode $1`), runs `ssh -o BatchMode=yes ci@10.100.0.48 ...`, exits with the ssh exit code. Do not wrap the remote git calls in `$ErrorActionPreference='Stop'` (git writes progress to stderr — known windev gotcha).
|
||||
4. **ci.yml — add the per-push job** (same commit as steps 5–6):
|
||||
|
||||
```yaml
|
||||
windows-x86:
|
||||
# x86/net48 Worker build + Worker.Tests, executed on windev (10.100.0.48) over SSH.
|
||||
# Runs on a Linux runner so it always schedules (act_runner host-mode on Windows is
|
||||
# broken; a runs-on gate with no runner wedges the queue — see docs/GatewayTesting.md).
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Worker x86 build + tests on windev
|
||||
env:
|
||||
WINDEV_SSH_KEY: ${{ secrets.WINDEV_SSH_KEY }}
|
||||
WINDEV_SSH_KNOWN_HOSTS: ${{ secrets.WINDEV_SSH_KNOWN_HOSTS }}
|
||||
CI_SHA: ${{ github.sha }}
|
||||
run: ./scripts/ci/run-windev-ci.sh test # demote to `build` if measured wall time > ~10 min
|
||||
```
|
||||
|
||||
5. **ci.yml — add the nightly job** gated `if: github.event_name == 'schedule'` (safe: the gate is an `if`, not an unsatisfiable `runs-on`, so push runs mark it skipped, not queued), `runs-on: ubuntu-latest`, calling `run-windev-ci.sh live`; final `if: failure()` step posts a Gitea issue/comment via the API token. Fix the cron comment at `ci.yml:12-14` to describe what the schedule now actually runs.
|
||||
6. **ci.yml — replace the removal note** at `:123-131` with a short pointer: host-mode act_runner remains broken; the Windows tier runs via the SSH-driven `windows-x86`/nightly jobs; restore native jobs only with a proven runner.
|
||||
7. **TST-26 doc edits in the same commit** — see TST-26 below.
|
||||
8. After the first green nightly, mark old **TST-05** addressed (live smoke is scheduled again) and unblock old **TST-24** (client wire tests can join CI).
|
||||
|
||||
**Verification.**
|
||||
- Push a branch: `portable`, `java`, and `windows-x86` all green; confirm on windev that `C:\build\mxaccessgw-ci` sits at the pushed SHA.
|
||||
- **Deliberate red:** on a scratch branch, add a failing `[Fact]` to `Worker.Tests` (or an intentional x86-only compile error), push, confirm `windows-x86` turns red and the run fails; revert. This proves exit-code propagation end-to-end, the whole point of the SSH design.
|
||||
- **Unreachable-host red:** temporarily point the script at a bogus port (or stop sshd), confirm the job fails fast with a clear message rather than hanging or passing.
|
||||
- **Concurrency:** push two branches back-to-back; confirm the second run's remote step waits on the lock and both complete.
|
||||
- **Nightly:** temporarily set the cron a few minutes ahead (or trigger the schedule path manually), confirm the live job runs with `MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1` and, with a forced failure, that the on-failure issue step fires.
|
||||
- Confirm no secret material appears in the job logs.
|
||||
|
||||
---
|
||||
|
||||
## TST-26 — Docs describe CI jobs that no longer exist `Medium` · `P1 (folded into TST-25)` · Effort S
|
||||
|
||||
**Finding.** `docs/GatewayTesting.md:426-431` still documents the **`windows`** job ("the only host that builds the x86/net48 Worker… the definitive guard for the 'regenerate and commit `Generated/`' rule") and the **`live-mxaccess`** scheduled job — both deleted from `.gitea/workflows/ci.yml` in `abb0930`, which touched only the workflow file (a violation of the repo's docs-change-in-same-commit rule). Same stale claim at `scripts/check-codegen.ps1:17` ("guarded by the Windows CI job, not here") and in the `ci.yml:12-14` cron comment ("Nightly live-MXAccess smoke (windev)") though the schedule currently re-runs only `portable`+`java`.
|
||||
|
||||
**Impact.** Actively misleading in the worst direction: a reader concludes the worker build is CI-guarded when it is not, and the "definitive guard" claim is doubly wrong — the surviving guard for the Generated-commit rule is the portable job's `check-codegen.ps1` diff check, not a Windows compile.
|
||||
|
||||
**Design.** No standalone commit: these edits land **in the same commit as the TST-25 ci.yml change** (that is the same-commit rule the removal violated). Content rules: describe the jobs that exist (`portable`, `java`, `windows-x86`, nightly), name `check-codegen.ps1`'s Generated-diff check as the primary guard for the regenerate-and-commit rule with the windows-x86 net48 compile as the second line, and document the manual windev fallback (isolated `C:\build` worktree) as the degraded-mode procedure with its trigger condition (SSH job red for infra reasons). If TST-25 is somehow delayed, do **not** hold these edits hostage — an interim commit correcting the docs to the two-job reality plus a mandatory manual cadence (per-merge for worker-touching changes, weekly otherwise) is strictly better than the current false claim; rewrite again when the SSH job lands.
|
||||
|
||||
**Implementation.**
|
||||
1. `docs/GatewayTesting.md:426-431` — replace the `windows` and `live-mxaccess` bullets with: `windows-x86` (SSH-driven, per-push, x86 build + Worker.Tests on windev; failure semantics — red job = tier or change broken, how to distinguish) and the nightly scheduled job (full Worker.Tests + live-MXAccess smoke, `MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`, never gates a push, on-failure issue notification). Reword the paragraph at `:433-437` so the Generated/ guard attribution is: primary = `check-codegen.ps1` regeneration diff (portable job), secondary = net48 compile (windows-x86).
|
||||
2. `scripts/check-codegen.ps1:17` — change "are guarded by the Windows CI job, not here" to "are guarded by the SSH-driven `windows-x86` CI job (see docs/GatewayTesting.md § Continuous Integration), not here".
|
||||
3. `.gitea/workflows/ci.yml:12-14` — cron comment now describes the nightly windev run (covered by TST-25 step 5); `:123-131` removal note replaced (TST-25 step 6).
|
||||
|
||||
**Verification.** Greps prove the drift is gone: `grep -n '"windows"' docs/GatewayTesting.md` and `grep -n 'live-mxaccess' docs/GatewayTesting.md` return only descriptions of jobs that exist in `ci.yml`; `grep -n 'Windows CI job' scripts/check-codegen.ps1` returns the corrected text; `grep -rn 'definitive guard' docs/` attributes the Generated/ rule to `check-codegen.ps1`. Read the CI section back against the final `ci.yml` job list — every named job must exist, and every job in `ci.yml` must be named.
|
||||
|
||||
---
|
||||
|
||||
## TST-27 — `ShowTagValues` config row still says "Reserved" `Medium` · `P1 (doc batch)` · Effort S
|
||||
|
||||
**Finding.** `docs/GatewayConfiguration.md:185` reads "Reserved display control for tag values. The dashboard does not show full tag values by default." — but SEC-25 wired the flag live: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardEventBroadcaster.cs:16-29` blanks tag values from the deep-cloned event before mirroring to SignalR when `Dashboard:ShowTagValues` is `false` (the default). `docs/GatewayDashboardDesign.md:170` was updated in the SEC-25 change; the authoritative config reference was not — remediation-created drift, same-commit rule violated.
|
||||
|
||||
**Impact.** An operator consulting the options table concludes toggling the flag does nothing, when it actually controls whether tag values reach **every** dashboard hub subscriber — security-relevant precisely because the per-session hub ACL (TST-15) does not exist yet: the redaction is currently the only thing between a low-trust Viewer and other sessions' tag values.
|
||||
|
||||
**Design.** Documentation-only correction of the one stale row; no code change. State three things in the row: (1) what `false` (default) does — tag values are blanked from the `MxEvent` copy mirrored to the SignalR events hub; metadata (tag reference, quality, status, timestamps) still renders; (2) the security relevance — with no per-session hub ACL yet (TST-15), `true` exposes all sessions' tag values to every authenticated dashboard viewer; (3) the honest scope limit — the flag gates only the hub mirror, **not** the `/browse` live-value display (the TST-16 residual, tracked separately; do not paper over it by implying full coverage). Ship in the cycle's P1 doc-drift batch (with WRK-26, CLI-42, SEC residuals per the 00-overall roadmap) — one commit for the batch is fine since each edit is independently verifiable.
|
||||
|
||||
**Implementation.**
|
||||
1. `docs/GatewayConfiguration.md:185` — replace the row description with the three-part text above, cross-referencing `docs/GatewayDashboardDesign.md` for the mechanism.
|
||||
2. While in the file, confirm the shape block's `"ShowTagValues": false` at `:60` needs no change (it doesn't — value and default are correct; only the prose row lies).
|
||||
3. No edit to `DashboardEventBroadcaster.cs` or `GatewayDashboardDesign.md` (both already correct).
|
||||
|
||||
**Verification.** `grep -n 'Reserved' docs/GatewayConfiguration.md` no longer matches the `ShowTagValues` row; read the new row back against `DashboardEventBroadcaster.cs:16-29` (behavior), `Configuration/DashboardOptions.cs` (default `false`), and `GatewayDashboardDesign.md:170` (consistency). Confirm the row does **not** claim `/browse` coverage.
|
||||
|
||||
---
|
||||
|
||||
## TST-28 — Gateway-side `max_frame_bytes` untested in the CI-run suite `Low` · `P2` · Effort S · relates IPC-02
|
||||
|
||||
**Finding.** `src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs:988` populates `MaxFrameBytes = (uint)_connection.FrameOptions.MaxMessageBytes` in the `GatewayHello` (wired from `Worker.MaxMessageBytes` via `Sessions/SessionWorkerClientFactory.cs:73-76`), but no test under `src/ZB.MOM.WW.MxGateway.Tests` references `MaxFrameBytes` at all. The adoption side is asserted only in `Worker.Tests` — Windows-only, i.e. inside the TST-25 gap.
|
||||
|
||||
**Impact.** A regression to sending `0` (which the worker deliberately treats as "older gateway, use default" per the IPC-02 negotiation contract) would pass every automated test while silently reverting the negotiated frame limit — an invisible downgrade of shipped IPC-02 behavior.
|
||||
|
||||
**Design.** One cheap fake-worker assertion; no new infrastructure needed. `FakeWorkerHarness.CompleteStartupAsync` already reads and **returns** the `GatewayHello` envelope (`Tests/Gateway/Workers/Fakes/FakeWorkerHarness.cs:149-165`), and `CreateConnectedPairAsync` already accepts a `maxMessageBytes` override that flows into the `WorkerFrameProtocolOptions` shared with the `WorkerClient` the harness creates — so the test is: start the client, complete the handshake, assert on the returned envelope. Cover both the default (`WorkerFrameProtocolOptions.DefaultMaxMessageBytes`) and an overridden value (e.g. `2 * 1024 * 1024`) as a `[Theory]`, which also pins that the field tracks configuration rather than a constant. Rejected alternative: asserting via the worker-side adoption tests only — that is exactly the Windows-only blind spot this finding names.
|
||||
|
||||
**Implementation.**
|
||||
1. Add a `[Theory]` to `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs` (natural home — its `CompleteHandshakeAsync` helper at `:26` already drives this path), e.g. `StartAsync_SendsGatewayHelloWithConfiguredMaxFrameBytes`: create the harness with `CreateConnectedPairAsync(maxMessageBytes: value)` for `value` ∈ {default, 2 MiB}, create/start the `WorkerClient`, capture the envelope from `CompleteStartupAsync`, assert `envelope.GatewayHello.MaxFrameBytes == (uint)value` (and `!= 0` for readability of intent).
|
||||
2. No production code change; no doc change (`docs/WorkerFrameProtocol.md:25-30` already documents the negotiation).
|
||||
|
||||
**Verification.** `dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter FullyQualifiedName~WorkerClientTests` green on macOS/Linux (i.e. it runs in the `portable` CI job — the point of the finding). Mutation check while developing: temporarily hard-code `MaxFrameBytes = 0` in `WorkerClient.cs:988` and confirm the new test fails, then revert.
|
||||
|
||||
---
|
||||
|
||||
## TST-29 — Retire `oldtasks.md`; delete root docs-review artifacts `Low` · `P2` · Effort S
|
||||
|
||||
**Finding.** TST-14's plan was "keep `oldtasks.md` only until the epic resumes, then delete", but TST-04 made it the durable governance record: `oldtasks.md:27-35,75-88` is now the only tracked statement that Phase 5 (orphan-worker reattach) is **deferred, not planned** and that `EnableOrphanReattach` (Task 26) "does not exist and must not be referenced". CLAUDE.md:79 links to it for exactly that statement. Meanwhile the mechanical half of TST-14 — deleting the five untracked, gitignored root working files (`MxAccessGateway-docs-{issues,fixed,final}.md`, `MxGatewayClient-docs-{issues,fixed}.md`, ~15k lines) — remains undone and keeps tripping greps.
|
||||
|
||||
**Impact.** Low, but real: deleting `oldtasks.md` naively would orphan a load-bearing governance record; keeping it forever contradicts its own "just a human-readable mirror" charter (`oldtasks.md:12-14`) and leaves two sources that can drift from `docs/plans/2026-06-15-session-resilience.md.tasks.json`. The untracked artifacts are pure grep noise.
|
||||
|
||||
**Design.** Move the durable decisions to where invariant-adjacent decisions live, then delete the mirror. `docs/DesignDecisions.md` gains a "Session-resilience epic scope" entry recording: Phase 5 deferred-not-planned with the rationale (reverses the "gateway restart does not reattach orphan workers" invariant; adoption-manifest store + phone-home protocol; deferred unless a concrete requirement appears); the explicit rule that `EnableOrphanReattach` does not exist and must not be referenced until its task lands; and the settled Phase-4 Viewer-default decision (admin-sees-all, Viewer strictly scoped to owned/granted sessions — `oldtasks.md:71-72`) so the TST-15 implementation doesn't have to rediscover it. The `tasks.json` remains the sole resume state for the still-pending Phase 4 tasks — one authority, no mirror. Do this now rather than waiting for the epic to close: the mirror's residual value (Task 14 Java remainder, Phase 4 checklist) is fully carried by `tasks.json` + the archreview register. Alternative rejected: keep `oldtasks.md` as the governance record — a file named "oldtasks" that must never be deleted is a trap for future cleanup passes.
|
||||
|
||||
**Implementation.**
|
||||
1. `docs/DesignDecisions.md` — add the "Session-resilience epic scope" decision entry (content above, dated, citing the epic plan and TST-04).
|
||||
2. Update the two tracked references before deletion: CLAUDE.md:79 ("see `oldtasks.md` (session-resilience epic Phase 5)" → point at `docs/DesignDecisions.md`); `stillpending.md:7,165` ("remaining epic tasks tracked in `oldtasks.md`" → point at `docs/plans/2026-06-15-session-resilience.md.tasks.json`). Leave `docs/plans/2026-06-16-stillpending-section8-design.md` (historical plan; a stale pointer in a dated plan is acceptable — optionally annotate).
|
||||
3. `git rm oldtasks.md` — same commit as steps 1–2 (the record must never be absent from `main`).
|
||||
4. `rm` the five untracked root files: `MxAccessGateway-docs-{issues,fixed,final}.md`, `MxGatewayClient-docs-{issues,fixed}.md` (gitignored at `.gitignore:152-154` — deletion is local-only, zero repo impact; no commit involved).
|
||||
5. Do **not** touch `stillpending.md`'s own promote-or-migrate question, `REVIEW-PROCESS.md`, or `A2-galaxyrepository-adoption-handoff.md` — those remain TST-14's open triage in the prior-cycle doc.
|
||||
|
||||
**Verification.** `grep -rn 'oldtasks' --include='*.md' .` (excluding `archreview/`, which is a historical record) returns no tracked references; `grep -n 'EnableOrphanReattach\|deferred, not planned' docs/DesignDecisions.md` finds the migrated statements; `ls *-docs-*.md` at the repo root returns nothing; `git status` clean. Read CLAUDE.md:79 back to confirm the invariant sentence still holds and its pointer resolves.
|
||||
|
||||
---
|
||||
|
||||
## TST-30 — Single shared Gitea runner is a CI throughput/availability bottleneck `Low` · `P2` · Effort M
|
||||
|
||||
**Finding.** All CI for this repo runs on **one** co-located `gitea-runner` container on docker host `10.100.0.35` with `maxParallel=1` (observed in the runner logs during TST-25 acceptance: `Running job with maxParallel=1`). That runner is **shared across repos** — it interleaves `dohertj2/mxaccessgw` and `dohertj2/lmxopcua` jobs on the single slot — and Gitea's per-repo runner list is empty (`GET /repos/dohertj2/mxaccessgw/actions/runners` → `total_count: 0`), i.e. it is an instance/shared runner, not repo-scoped. Every job in every run executes serially: a single `mxaccessgw` push fans out to `portable` (~5 min), `java`, `windows-x86`, so the wall time for the queue behind it is additive, and an active `lmxopcua` run blocks `mxaccessgw` entirely. Compounding it, this Gitea version (1.26.4) exposes **no run cancel or delete via the API** (`POST .../actions/runs/{id}/cancel` → 404; `DELETE .../actions/runs/{id}` → 400), so superseded or stale runs cannot be cleared and consume the slot in full before newer runs start.
|
||||
|
||||
**Impact.** Low severity, real drag. CI feedback latency is unbounded under cross-repo contention (observed ~20–30 min queue depth during TST-25 verification, forcing direct local `run-windev-ci.sh` verification to bypass the queue). Because the runner is the *only* one, it is also a single point of failure — if it wedges or its host is down, **all** CI (both repos) stops, with no failover; and there is no way to abort a run that is hung or was obsoleted by a newer push, so a stuck job can hold the slot until its internal timeout. This does not threaten correctness (jobs still run and report accurately) but it undercuts the fast-feedback purpose of the TST-25 tier and makes the CI system fragile in exactly the way act_runner history (TST-25) warns about.
|
||||
|
||||
**Design.** Add a **second runner** so the two repos (and the fan-out jobs) stop contending on one slot, and prefer repo/label scoping so `mxaccessgw`'s Windows-adjacent jobs aren't starved by an unrelated repo. Options, cheapest first:
|
||||
|
||||
- **(a) Raise `maxParallel` / register a second runner instance on `10.100.0.35` — recommended.** The host already runs `gitea-runner`; add a second `act_runner` instance (or bump the existing runner's capacity where the docker-in-docker/resource budget allows) so at least two jobs run concurrently. Cheapest change, keeps the co-located-with-`gitea:3000` network property that TST-03 depended on (`container.network: traefik`).
|
||||
- **(b) Dedicate a runner to `mxaccessgw`** with a distinct label and gate the CI jobs on it, so `lmxopcua` load never blocks this repo. Cleaner isolation than (a) but needs label wiring in `ci.yml` (`runs-on: [ubuntu-latest, mxgw]` or similar) and a labelled runner registration.
|
||||
- **(c) Put the runner on windev / a second host** — rejected as the primary fix: windev is the Windows build target, not a CI host, and co-locating the Linux runner there loses the `gitea:3000` resolution TST-03 relies on. Only consider if `10.100.0.35` runs out of capacity.
|
||||
|
||||
Independent of the runner count, document the **no-cancel** reality (Gitea 1.26 API) and the escape hatch: a specific SHA can be verified out-of-band via `scripts/ci/run-windev-ci.sh` (Linux) or the manual windev worktree flow, bypassing the queue — this is already the degraded-mode path in `scripts/ci/README.md`; TST-30 generalizes it from "windev tier down" to "runner contended".
|
||||
|
||||
**Implementation.**
|
||||
1. Register a second `act_runner` on `10.100.0.35` (option a) with the same `container.network: traefik` config (repo memory `project_gitea_ci`), or a labelled dedicated runner (option b) and add the label to `ci.yml`'s `runs-on`. Confirm both runners appear and pick up jobs concurrently (two pushes should no longer serialize end-to-end).
|
||||
2. `docs/GatewayTesting.md` (Continuous Integration section) — note that the runner is shared/finite, that queue latency under cross-repo contention is expected, that this Gitea version has no run cancel/delete, and that `scripts/ci/run-windev-ci.sh <sha>` / the manual worktree flow verify a commit without waiting for the queue.
|
||||
3. Optional: add a workflow-level `concurrency` group to `ci.yml` so a newer push to the same branch supersedes an in-flight run *if* the deployed Gitea honors it (belt-and-suspenders against the missing cancel API; verify it actually cancels before relying on it).
|
||||
|
||||
**Verification.** Push two branches back-to-back and confirm their runs execute concurrently (not serially) once a second runner exists; `GET /repos/dohertj2/mxaccessgw/actions/runners` (or the instance runner list) shows ≥2 runners online; `docs/GatewayTesting.md` describes the shared-runner/no-cancel reality and the bypass. Re-run the TST-25 acceptance push and confirm queue depth is materially lower under a concurrent `lmxopcua` run.
|
||||
|
||||
---
|
||||
|
||||
## Cross-domain dependencies
|
||||
|
||||
- **TST-25 → old TST-05 / old TST-24:** the SSH-driven nightly is where the scheduled live-MXAccess smoke lands (closes old TST-05), and a working CI Windows tier unblocks wiring client wire-behavior tests into CI (old TST-24).
|
||||
- **TST-25 ↔ IPC-24/IPC-25:** the nightly windev job is also the natural home for any Windows-side codegen verification the contracts/IPC remediation adds; coordinate job naming so both plans extend the same `windows-x86`/nightly jobs rather than adding parallel ones.
|
||||
- **TST-26 ⊂ TST-25:** same commit, by rule.
|
||||
- **TST-27:** ships in the cycle's P1 doc-drift batch alongside WRK-26 and CLI-42 (roadmap item 8); its `/browse` residual stays with TST-16 (prior cycle).
|
||||
- **TST-28 ← IPC-02:** the test pins the gateway half of the IPC-02 negotiation; the worker half's tests become CI-run once TST-25 lands.
|
||||
@@ -215,7 +215,7 @@ Full design + implementation for each row lives in the linked domain doc under i
|
||||
|---|---|:-:|:-:|---|---|---|
|
||||
| TST-01 | High | P2 | L | TST-04 | Done | Reconnect/replay has no e2e test and no client consumer |
|
||||
| TST-02 | High | P0 | M | TST-04 | Done | Reconnect owner re-validation not implemented |
|
||||
| TST-03 | High | P1 | M | — | In review | No CI exists |
|
||||
| TST-03 | High | P1 | M | — | Done | No CI exists |
|
||||
| TST-04 | High | P2 | L | — | Done | Session-resilience epic 16/28 tasks unfinished |
|
||||
| TST-05 | Medium | P1 | S | TST-03 | Not started | Real-worker control/COM paths verified opt-in only |
|
||||
| TST-06 | Medium | — | M | — | Not started | Dashboard live-data path untested |
|
||||
@@ -253,6 +253,7 @@ Findings the review flagged as one coordinated design pass — sequence them tog
|
||||
|
||||
| Date | Change |
|
||||
|---|---|
|
||||
| 2026-07-10 | **TST-15 design fleshed out** (still `Not started` — design only, not implementation): `docs/plans/2026-07-10-dashboard-session-acl-tst15.md`. Resolves the crux the deferral left open — the dashboard is LDAP-identity (Admin/Viewer) while sessions are API-key-owned (`OwnerKeyId`), two disjoint identity domains — via a **session tag** sourced from the owning API key (rides in the existing `ApiKeyConstraints` JSON blob, no SQLite migration). Admin-sees-all; Viewer may `SubscribeSession` iff `session.Tags ∩ viewer.GrantedTags ≠ ∅` (new `Dashboard:GroupToTag` map → hub-token tag claims); untagged sessions Admin-only by default (`Dashboard:UntaggedSessionVisibility`). Includes the enforcement path (`HubTokenPayload.Tags` + `IDashboardSessionAcl` gate at `SubscribeSession`), task breakdown (epic Tasks 16–19), test plan incl. live-LDAP, and rejected alternatives (client-supplied tag; group→key-id map). Tracker + `60-testing-docs-gaps.md` TST-15 section point at the doc. **TST-03 investigated:** the CI never ran because the repo had **zero registered Gitea Actions runners** (Actions is enabled; runs are created on push/PR/nightly but fail instantly with nothing to execute them). A Mac runner proved the pipeline executes but cannot clone — this Gitea hands runners the internal `http://gitea:3000` URL, reachable only by a runner co-located on the gitea Docker network. Fix = run a co-located runner on the Gitea host (recipe prepared, `scratchpad/gitea-runner/setup-gitea-host-runner.sh`); pending host access. TST-03 stays `In review`. |
|
||||
| 2026-07-09 | **P2 Epic wrap — user decision: DEFER TST-15 + TST-24, close the epic.** Epic bucket result: 5 of 7 findings `Done` (CLI-15, CLI-04, CLI-30, TST-01, TST-04); **TST-15** and **TST-24** intentionally deferred to a follow-up (kept `Not started`, not `Won't fix` — they are gated, not rejected). **TST-15** (dashboard EventsHub per-session ACL) is epic Phase 4 — a real feature needing a new session-"tag" mechanism + dashboard group→tag config, not a mechanical fix; the `EventsHub` `TODO(per-session-acl)` stays, and the already-shipped **SEC-25** mitigation (tag *values* redacted from the dashboard mirror by default) means no sensitive payload leaks through the hub today regardless of the missing ACL — so deferring carries no value-leak risk. **TST-24** (per-client wire tests) depends on **TST-03** (CI), which is `In review` (YAML authored, never run on a Gitea runner) — no point wiring client tests into a pipeline that isn't live yet. Net P2: 35/38 `Done`; remaining = TST-15 (deferred feature), TST-24 (deferred, CI-gated), TST-14 (user deletes their own untracked gitignored `*-docs-*.md` files). |
|
||||
| 2026-07-09 | P2 Epic — **Java client completes CLI-15 + CLI-04 locally** (commit `1cc0fa4`); **CLI-15, CLI-04, CLI-30, TST-01 all → `Done` (5/5 clients + server e2e)**. Java CLI-15: `MxEventStreamItem` record + `MxEventStream.nextItem()` (`isReplayGap()`/`replayGap()`/`event()`); existing `Iterator<MxEvent>` path unchanged, sentinel never swallowed. Java CLI-04: Phase 1 `adviseSupervisory`/`writeSecured`/`writeSecured2`/`authenticateUser`/`archestrAUserToId` + Phase 2 `addBufferedItem`/`setBufferedUpdateInterval`/`suspend`/`activate` (unregister already present) on `MxGatewaySession`, each through `invokeCommand` → `ensureProtocolSuccess`+`ensureMxAccessSuccess`; credentials scrubbed via `MxGatewaySecrets.redactCredentials` (tests assert absent from message/toString/CLI). `gradle test` 106/0 (58 client + 48 cli), no generated churn. Built locally with `JAVA_HOME=/opt/homebrew/opt/openjdk@17` — Java toolchain now works on the Mac (see prior note). Shared docs `ClientLibrariesDesign.md` + CLAUDE.md updated to "all five clients". **TST-01 → Done** (server e2e `fed0685` + all 5 client `ReplayGap` consumers). This closes session-resilience epic Phase 3 fully. |
|
||||
| 2026-07-09 | P2 Epic Wave E2 — typed-command parity (commit `bde042b`). CLI-04 → `In progress` (4/5 clients; Java pending in the SAME session — see next note), CLI-30 → `Done`. Every parity-critical single-item command now has a typed session helper across .NET/Go/Rust/Python: Phase 1 `AdviseSupervisory`/`WriteSecured`/`WriteSecured2`/`AuthenticateUser`/`ArchestrAUserToId`, Phase 2 `AddBufferedItem`/`SetBufferedUpdateInterval`/`Suspend`/`Activate`, plus `Unregister` (CLI-30: Rust + .NET added; Go/Java/Python already had it — CLI-30 fully Done). Each wraps existing raw-command machinery (no new wire surface) and runs the client's MXAccess-level reply validation (`hresult < 0` + `MxStatusProxy`). MXAccess parity preserved (WriteSecured-before-authenticate surfaces the native failure). **Credentials route through each client's secret-redaction seam** (never in logs/exceptions/ToString/Debug/Display; each suite asserts a distinctive credential is absent from surfaced errors); new CLI subcommands take credentials via flag/env, never echoed. Verified per toolchain: .NET build clean 102 passed; Go gofmt/vet/build/test clean; Rust fmt/check/test/clippy clean; Python 145 passed. Shared doc: `ClientLibrariesDesign.md` "Typed Command Parity" section. |
|
||||
@@ -278,3 +279,4 @@ Findings the review flagged as one coordinated design pass — sequence them tog
|
||||
| 2026-07-09 | P1 Wave 2a (security). SEC-01/04/06 → `Done` (config path-rooting + production validator guards; Server build clean, validator+hygiene tests 53/53). SEC-10 → `Done`: the shared `ZB.MOM.WW.Auth.ApiKeys` gained optional `ExpiresUtc` (expired keys rejected, auth DB auto-migrates to schema v3) via a concurrent HistorianGateway-remediation session's "G-2"; this repo consumes it by bumping the four `Auth.*` refs 0.1.2→0.1.4 (commit 197731a). Remaining SEC-10 polish (`apikey create --expires` + dashboard staleness badge) tracked as a small follow-up. |
|
||||
| 2026-07-09 | P1 Wave 1 (CI + codegen freshness + Rust buildability) via parallel agents. IPC-01/09/19/20, CLI-02 → `Done`; TST-03 → `In review` (CI pipeline authored + YAML/layout-validated, but not yet executed on a Gitea runner — proven on first push). Verified on macOS: NonWindows build clean, `ClientProtoInputTests` 5/5, `publish-client-proto-inputs.ps1 -Check` exit 0, `cargo package` (no `--no-verify`) compiles standalone. Added a vendored-Rust-proto drift guard (Check 3) to `check-codegen.ps1` closing the CLI-02 static-copy risk. IPC-09 script guards not executed end-to-end (need pinned python/JRE toolchains); logic is PATH-resolution + version assertion. |
|
||||
| 2026-07-09 | WRK-01 → `Done`. Verified on Windows host (windev) via an isolated `origin/main` worktree: worker builds x86 clean, `StaRuntimeTests`+`WorkerPipeSessionTests` 33/33 pass. Fixed an `xUnit1030` build error (the new worker test used `.ConfigureAwait(false)` in `[Fact]` bodies) that the macOS tree could not surface. Also ran GWC-01's Windows-only `WorkerClientTests` on windev: 18/18 pass (incl. `ReadEventsAsync_SecondEnumerator_Throws`). All 8 P0 findings now `Done`. Not yet committed. |
|
||||
| 2026-07-10 | **TST-03 → `Done`: CI is live and green** on branch `fix/ci-selfhosted-tooling`. The pipeline was authored (P1) but had never executed. Root cause it never ran: the co-located `gitea-runner` on the docker host (`10.100.0.35`) spawned job containers on an isolated network (`container.network: ""`) that could not resolve Gitea's internal clone URL `http://gitea:3000`; one-line host fix `container.network: "traefik"` + `docker restart gitea-runner`. The self-hosted `catthehacker` act image also lacks tooling GitHub-hosted runners preinstall — ci.yml now installs pwsh (dotnet global tool, for `check-codegen.ps1`) and Gradle 9.5.1 directly (act can't resolve the `gradle/actions` monorepo action; no gradle wrapper in repo). Driving to green surfaced and fixed **five real latent defects** (TST-03 doing its job): `check-codegen.ps1` `.Trim()`-on-`$null` on a clean tree; **stale vendored rust proto** (`clients/rust/protos/mxaccess_worker.proto` missing canonical `max_frame_bytes`); `OrphanWorkerTerminatorTests` hard-coded `C:\` path failing Linux `Path.GetFullPath` (0 kills); **stale java generated** `MxaccessWorker.java` (missing `max_frame_bytes`, regenerated); a sync python test building a `grpc.aio.Channel` with no current event loop on py3.12 (autouse conftest fixture). Result: `portable` **success** (NonWindows build + codegen freshness + 808/808 gateway tests + .NET/Go/Rust/Python clients) and `java` **success**. `windows` + `live-mxaccess` jobs remain `queued` pending a self-hosted **windev** runner (`10.100.0.48`) with those labels — separate follow-up, does not gate portable/java. |
|
||||
|
||||
@@ -327,7 +327,7 @@ If TST-02's interim mitigation (flip retention off) is chosen instead of impleme
|
||||
|
||||
**Impact.** Acceptable for a single-tenant dashboard; wrong the moment `GroupToRole` admits low-trust viewers. It is the dashboard-side twin of the gRPC owner-revalidation gap (TST-02).
|
||||
|
||||
**Design.** Covered by epic Phase 4 (Tasks 16–19, TST-04). Introduce a role/scope that scopes a hub connection to permitted sessions, plus a hub-token session tag (Task 18). The open design decision (`oldtasks.md:52`) — Viewer default of admin-sees-all vs strict per-session — must be settled first. Recommendation: admin-sees-all, Viewer strictly scoped to sessions they own/are granted, matching TST-02's gRPC owner binding for consistency. Until Phase 4 lands, keep the TODO (it correctly documents the accepted single-tenant assumption); do not silently remove it.
|
||||
**Design.** Fully fleshed out in `docs/plans/2026-07-10-dashboard-session-acl-tst15.md` (epic Phase 4, Tasks 16–19, TST-04). In brief: the dashboard authenticates LDAP users (Admin/Viewer) while sessions are API-key-owned (`OwnerKeyId`) — two disjoint identity domains — so the ACL needs a bridge: a **session tag** sourced from the owning API key (riding in the existing `ApiKeyConstraints` JSON blob, no SQLite migration). Admin sees all; a Viewer may `SubscribeSession` iff `session.Tags ∩ viewer.GrantedTags ≠ ∅` (granted via a new `Dashboard:GroupToTag` map, carried into the hub token as tag claims); untagged sessions are Admin-only by default. The Viewer-default decision (admin-sees-all vs strict) is settled there. Until Phase 4 lands, keep the TODO (it correctly documents the accepted single-tenant assumption); do not silently remove it.
|
||||
|
||||
**Implementation.** `Dashboard/Hubs/EventsHub.cs` (ACL check on group join), hub-token minting to carry the session tag, `Configuration/DashboardOptions.cs` for any group-to-tag config (Task 17). Tests: `...Tests/Gateway/Dashboard/` hub ACL cases incl. live-LDAP users (Task 19). Docs: `docs/Sessions.md`/`gateway.md` dashboard section document the ACL model; CLAUDE.md dashboard-auth paragraph.
|
||||
|
||||
|
||||
@@ -44,6 +44,7 @@ internal sealed class CliArguments
|
||||
|
||||
/// <summary>Returns whether the named flag was present in the arguments.</summary>
|
||||
/// <param name="name">The flag name (without '--' prefix).</param>
|
||||
/// <returns><c>true</c> if the flag was present; otherwise, <c>false</c>.</returns>
|
||||
public bool HasFlag(string name)
|
||||
{
|
||||
return _flags.Contains(name);
|
||||
@@ -51,6 +52,7 @@ internal sealed class CliArguments
|
||||
|
||||
/// <summary>Returns the value for a named argument, or <c>null</c> if absent.</summary>
|
||||
/// <param name="name">The argument name (without '--' prefix).</param>
|
||||
/// <returns>The argument value, or <c>null</c> if the argument is absent.</returns>
|
||||
public string? GetOptional(string name)
|
||||
{
|
||||
return _values.TryGetValue(name, out string? value)
|
||||
@@ -60,6 +62,7 @@ internal sealed class CliArguments
|
||||
|
||||
/// <summary>Returns the value for a required named argument, or throws if absent.</summary>
|
||||
/// <param name="name">The argument name (without '--' prefix).</param>
|
||||
/// <returns>The argument value.</returns>
|
||||
public string GetRequired(string name)
|
||||
{
|
||||
string? value = GetOptional(name);
|
||||
@@ -74,6 +77,7 @@ internal sealed class CliArguments
|
||||
/// <summary>Parses and returns an int32 argument, or the default value if absent.</summary>
|
||||
/// <param name="name">The argument name (without '--' prefix).</param>
|
||||
/// <param name="defaultValue">The default value if the argument is absent; if <c>null</c>, the argument is required.</param>
|
||||
/// <returns>The parsed int32 value.</returns>
|
||||
public int GetInt32(string name, int? defaultValue = null)
|
||||
{
|
||||
string? value = GetOptional(name);
|
||||
@@ -93,6 +97,7 @@ internal sealed class CliArguments
|
||||
/// <summary>Parses and returns a uint32 argument, or the default value if absent.</summary>
|
||||
/// <param name="name">The argument name (without '--' prefix).</param>
|
||||
/// <param name="defaultValue">The default value if the argument is absent.</param>
|
||||
/// <returns>The parsed uint32 value.</returns>
|
||||
public uint GetUInt32(string name, uint defaultValue)
|
||||
{
|
||||
string? value = GetOptional(name);
|
||||
@@ -104,6 +109,7 @@ internal sealed class CliArguments
|
||||
/// <summary>Parses and returns a uint64 argument, or the default value if absent.</summary>
|
||||
/// <param name="name">The argument name (without '--' prefix).</param>
|
||||
/// <param name="defaultValue">The default value if the argument is absent.</param>
|
||||
/// <returns>The parsed uint64 value.</returns>
|
||||
public ulong GetUInt64(string name, ulong defaultValue)
|
||||
{
|
||||
string? value = GetOptional(name);
|
||||
@@ -115,6 +121,7 @@ internal sealed class CliArguments
|
||||
/// <summary>Parses and returns a TimeSpan argument, or the default value if absent. Supports "ms", "s", and standard TimeSpan format.</summary>
|
||||
/// <param name="name">The argument name (without '--' prefix).</param>
|
||||
/// <param name="defaultValue">The default value if the argument is absent.</param>
|
||||
/// <returns>The parsed duration.</returns>
|
||||
public TimeSpan GetDuration(string name, TimeSpan defaultValue)
|
||||
{
|
||||
string? value = GetOptional(name);
|
||||
|
||||
@@ -108,7 +108,8 @@ internal sealed class MxGatewayCliClientAdapter : IMxGatewayCliClient
|
||||
return _galaxyClient.Value.BrowseChildrenRawAsync(request, cancellationToken);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>Disposes the adapted gateway client and, if created, the lazily-initialized Galaxy Repository client.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_galaxyClient.IsValueCreated)
|
||||
|
||||
@@ -10,6 +10,7 @@ internal static class MxGatewayCliSecretRedactor
|
||||
/// </summary>
|
||||
/// <param name="value">The message text to redact.</param>
|
||||
/// <param name="secrets">The secret values to remove (API key, verify-user password, secured payloads).</param>
|
||||
/// <returns>The value with any occurrences of the supplied secrets replaced by a redacted placeholder.</returns>
|
||||
public static string Redact(string value, params string?[] secrets)
|
||||
{
|
||||
if (string.IsNullOrEmpty(value) || secrets is null)
|
||||
|
||||
@@ -22,6 +22,7 @@ public static class MxGatewayClientCli
|
||||
/// <param name="args">Command-line arguments (command name followed by options).</param>
|
||||
/// <param name="standardOutput">TextWriter for command output.</param>
|
||||
/// <param name="standardError">TextWriter for error messages.</param>
|
||||
/// <returns>The process exit code: 0 on success, 1 on error.</returns>
|
||||
public static int Run(
|
||||
string[] args,
|
||||
TextWriter standardOutput,
|
||||
@@ -38,6 +39,7 @@ public static class MxGatewayClientCli
|
||||
/// <param name="standardError">TextWriter for error messages.</param>
|
||||
/// <param name="clientFactory">Optional factory to create the gateway client; defaults to MxGatewayClient.Create.</param>
|
||||
/// <param name="standardInput">Optional TextReader for batch-mode stdin; defaults to <see cref="Console.In"/>.</param>
|
||||
/// <returns>A task producing the process exit code: 0 on success, 1 on error.</returns>
|
||||
public static Task<int> RunAsync(
|
||||
string[] args,
|
||||
TextWriter standardOutput,
|
||||
@@ -173,10 +175,10 @@ public static class MxGatewayClientCli
|
||||
}
|
||||
catch (Exception exception) when (exception is not OperationCanceledException)
|
||||
{
|
||||
// Client.Dotnet-028: redact the *effective* key — from --api-key or the
|
||||
// --api-key-env environment variable — so an env-var-sourced key echoed
|
||||
// in a transport error never reaches stderr unredacted. CLI-04: also
|
||||
// redact MXAccess credentials (AuthenticateUser password, WriteSecured
|
||||
// Redact the *effective* key — from --api-key or the --api-key-env
|
||||
// environment variable — so an env-var-sourced key echoed in a
|
||||
// transport error never reaches stderr unredacted; likewise the
|
||||
// MXAccess credentials (AuthenticateUser password, WriteSecured
|
||||
// payloads) that could otherwise be echoed back in a surfaced error.
|
||||
string? apiKey = TryResolveApiKey(arguments);
|
||||
string message = MxGatewayCliSecretRedactor.Redact(
|
||||
@@ -327,7 +329,7 @@ public static class MxGatewayClientCli
|
||||
/// <c>MXGATEWAY_API_KEY</c>), returning <see langword="null"/> when neither
|
||||
/// is set. Unlike <see cref="ResolveApiKey"/> this never throws, so the
|
||||
/// error-redaction catch block can strip the env-var-sourced key from
|
||||
/// output (Client.Dotnet-028) without re-raising on the absent-key path.
|
||||
/// output without re-raising on the absent-key path.
|
||||
/// </summary>
|
||||
private static string? TryResolveApiKey(CliArguments arguments)
|
||||
{
|
||||
@@ -1019,7 +1021,6 @@ public static class MxGatewayClientCli
|
||||
/// (e.g. <c>-1</c>, an easy copy-paste mistake for "unbounded") is
|
||||
/// rejected loudly rather than silently wrapped to <c>~49.7 days</c>,
|
||||
/// which would park one worker thread per pending tag for hours.
|
||||
/// Resolves Client.Dotnet-021.
|
||||
/// </summary>
|
||||
private static uint ParseTimeoutMs(CliArguments arguments, int defaultValue)
|
||||
{
|
||||
@@ -1041,7 +1042,6 @@ public static class MxGatewayClientCli
|
||||
/// its absence must not silently fall through to
|
||||
/// <c>ReturnValue.Int32Value</c> (which would be <c>0</c> for an empty
|
||||
/// reply, driving the rest of the bench against an invalid handle).
|
||||
/// Resolves Client.Dotnet-019.
|
||||
/// </summary>
|
||||
private static int RequireRegisterServerHandle(MxCommandReply reply, string sessionId)
|
||||
{
|
||||
@@ -1166,11 +1166,6 @@ public static class MxGatewayClientCli
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
// Client.Dotnet-020: never swallow OperationCanceledException
|
||||
// here. A bare `catch` would let Ctrl+C / parent CTS /
|
||||
// wall-clock timeouts keep spinning until --duration-seconds
|
||||
// elapsed, burning CPU and skewing the p99/max latency numbers
|
||||
// with hundreds of immediate-OCE iterations.
|
||||
sw.Stop();
|
||||
failedCalls++;
|
||||
latencyMillis.Add(sw.Elapsed.TotalMilliseconds);
|
||||
@@ -1397,8 +1392,6 @@ public static class MxGatewayClientCli
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Client.Dotnet-017: graceful end-of-window completion mode for a
|
||||
// finite-window event collector. Emit aggregate JSON below and exit 0.
|
||||
}
|
||||
|
||||
if (json && !jsonLines)
|
||||
@@ -1468,10 +1461,6 @@ public static class MxGatewayClientCli
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
// Mirrors stream-events (Client.Dotnet-017): the supplied token covers
|
||||
// the user's --timeout wall-clock budget and external Ctrl+C / parent
|
||||
// CTS cancellation. All are graceful completion modes for a
|
||||
// finite-window alarm-feed collector: emit what arrived and exit 0.
|
||||
}
|
||||
|
||||
if (json && !jsonLines)
|
||||
|
||||
@@ -14,6 +14,7 @@ public sealed class BrowseChildrenSmokeTests
|
||||
/// Verifies that BrowseChildren returns a non-zero cache sequence and
|
||||
/// a consistent children/child-has-children count from a live gateway.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact(Skip = "Set MXGATEWAY_API_KEY and MXGATEWAY_ENDPOINT to enable.")]
|
||||
public async Task BrowseChildren_LiveGateway_ReturnsRootsWithCacheSequence()
|
||||
{
|
||||
|
||||
@@ -8,14 +8,10 @@ namespace ZB.MOM.WW.MxGateway.Client.Tests;
|
||||
/// </summary>
|
||||
internal sealed class FakeGalaxyRepositoryTransport(MxGatewayClientOptions options) : IGalaxyRepositoryClientTransport
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the gateway client options.
|
||||
/// </summary>
|
||||
/// <inheritdoc />
|
||||
public MxGatewayClientOptions Options { get; } = options;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the raw gRPC client; always null for the fake.
|
||||
/// </summary>
|
||||
/// <inheritdoc />
|
||||
public GalaxyRepository.GalaxyRepositoryClient? RawClient => null;
|
||||
|
||||
/// <summary>
|
||||
@@ -66,11 +62,7 @@ internal sealed class FakeGalaxyRepositoryTransport(MxGatewayClientOptions optio
|
||||
/// </summary>
|
||||
public Queue<Exception> DiscoverHierarchyExceptions { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Records the request and either throws a queued exception or returns the configured reply.
|
||||
/// </summary>
|
||||
/// <param name="request">The TestConnectionRequest to process.</param>
|
||||
/// <param name="callOptions">Call options specifying RPC behavior.</param>
|
||||
/// <inheritdoc />
|
||||
public Task<TestConnectionReply> TestConnectionAsync(
|
||||
TestConnectionRequest request,
|
||||
CallOptions callOptions)
|
||||
@@ -84,11 +76,7 @@ internal sealed class FakeGalaxyRepositoryTransport(MxGatewayClientOptions optio
|
||||
return Task.FromResult(TestConnectionReply);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records the request and either throws a queued exception or returns the configured reply.
|
||||
/// </summary>
|
||||
/// <param name="request">The GetLastDeployTimeRequest to process.</param>
|
||||
/// <param name="callOptions">Call options specifying RPC behavior.</param>
|
||||
/// <inheritdoc />
|
||||
public Task<GetLastDeployTimeReply> GetLastDeployTimeAsync(
|
||||
GetLastDeployTimeRequest request,
|
||||
CallOptions callOptions)
|
||||
@@ -102,11 +90,7 @@ internal sealed class FakeGalaxyRepositoryTransport(MxGatewayClientOptions optio
|
||||
return Task.FromResult(GetLastDeployTimeReply);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records the request and either throws a queued exception or returns the configured reply.
|
||||
/// </summary>
|
||||
/// <param name="request">The DiscoverHierarchyRequest to process.</param>
|
||||
/// <param name="callOptions">Call options specifying RPC behavior.</param>
|
||||
/// <inheritdoc />
|
||||
public Task<DiscoverHierarchyReply> DiscoverHierarchyAsync(
|
||||
DiscoverHierarchyRequest request,
|
||||
CallOptions callOptions)
|
||||
@@ -141,11 +125,7 @@ internal sealed class FakeGalaxyRepositoryTransport(MxGatewayClientOptions optio
|
||||
/// </summary>
|
||||
public Func<Task>? BrowseChildrenGate { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Records the request and either throws a queued exception or returns the configured reply.
|
||||
/// </summary>
|
||||
/// <param name="request">The BrowseChildrenRequest to process.</param>
|
||||
/// <param name="callOptions">Call options specifying RPC behavior.</param>
|
||||
/// <inheritdoc />
|
||||
public async Task<BrowseChildrenReply> BrowseChildrenAsync(
|
||||
BrowseChildrenRequest request,
|
||||
CallOptions callOptions)
|
||||
@@ -187,11 +167,7 @@ internal sealed class FakeGalaxyRepositoryTransport(MxGatewayClientOptions optio
|
||||
/// </summary>
|
||||
public Func<CancellationToken, Task>? WatchDeployEventsBeforeYield { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Records the request and streams events, checking for queued exceptions and calling WatchDeployEventsBeforeYield before each event.
|
||||
/// </summary>
|
||||
/// <param name="request">The WatchDeployEventsRequest to process.</param>
|
||||
/// <param name="callOptions">Call options specifying RPC behavior.</param>
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<DeployEvent> WatchDeployEventsAsync(
|
||||
WatchDeployEventsRequest request,
|
||||
CallOptions callOptions)
|
||||
|
||||
@@ -11,14 +11,10 @@ internal sealed class FakeGatewayTransport(MxGatewayClientOptions options) : IMx
|
||||
private readonly Queue<MxCommandReply> _invokeReplies = new();
|
||||
private readonly List<MxEvent> _events = [];
|
||||
|
||||
/// <summary>
|
||||
/// Gets the gateway client options.
|
||||
/// </summary>
|
||||
/// <inheritdoc />
|
||||
public MxGatewayClientOptions Options { get; } = options;
|
||||
|
||||
/// <summary>
|
||||
/// Gets null, since this is a test fake without a real gRPC client.
|
||||
/// </summary>
|
||||
/// <inheritdoc />
|
||||
public MxAccessGateway.MxAccessGatewayClient? RawClient => null;
|
||||
|
||||
/// <summary>
|
||||
@@ -102,11 +98,7 @@ internal sealed class FakeGatewayTransport(MxGatewayClientOptions options) : IMx
|
||||
/// </summary>
|
||||
public Queue<Exception> InvokeExceptions { get; } = new();
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the OpenSessionAsync call is recorded and returns the configured reply.
|
||||
/// </summary>
|
||||
/// <param name="request">The OpenSessionRequest to process.</param>
|
||||
/// <param name="callOptions">Call options specifying RPC behavior.</param>
|
||||
/// <inheritdoc />
|
||||
public Task<OpenSessionReply> OpenSessionAsync(
|
||||
OpenSessionRequest request,
|
||||
CallOptions callOptions)
|
||||
@@ -120,11 +112,7 @@ internal sealed class FakeGatewayTransport(MxGatewayClientOptions options) : IMx
|
||||
return Task.FromResult(OpenSessionReply);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the CloseSessionAsync call is recorded and returns the configured reply.
|
||||
/// </summary>
|
||||
/// <param name="request">The CloseSessionRequest to process.</param>
|
||||
/// <param name="callOptions">Call options specifying RPC behavior.</param>
|
||||
/// <inheritdoc />
|
||||
public Task<CloseSessionReply> CloseSessionAsync(
|
||||
CloseSessionRequest request,
|
||||
CallOptions callOptions)
|
||||
@@ -138,11 +126,7 @@ internal sealed class FakeGatewayTransport(MxGatewayClientOptions options) : IMx
|
||||
return Task.FromResult(CloseSessionReply);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the InvokeAsync call is recorded and returns the next enqueued reply.
|
||||
/// </summary>
|
||||
/// <param name="request">The MxCommandRequest to process.</param>
|
||||
/// <param name="callOptions">Call options specifying RPC behavior.</param>
|
||||
/// <inheritdoc />
|
||||
public Task<MxCommandReply> InvokeAsync(
|
||||
MxCommandRequest request,
|
||||
CallOptions callOptions)
|
||||
@@ -156,11 +140,7 @@ internal sealed class FakeGatewayTransport(MxGatewayClientOptions options) : IMx
|
||||
return Task.FromResult(_invokeReplies.Dequeue());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the StreamEventsAsync call is recorded and yields all enqueued events.
|
||||
/// </summary>
|
||||
/// <param name="request">The StreamEventsRequest to process.</param>
|
||||
/// <param name="callOptions">Call options specifying RPC behavior.</param>
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync(
|
||||
StreamEventsRequest request,
|
||||
CallOptions callOptions)
|
||||
@@ -193,11 +173,7 @@ internal sealed class FakeGatewayTransport(MxGatewayClientOptions options) : IMx
|
||||
_events.Add(gatewayEvent);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records the acknowledge call and returns the next enqueued reply (or default).
|
||||
/// </summary>
|
||||
/// <param name="request">The acknowledge alarm request.</param>
|
||||
/// <param name="callOptions">Call options specifying RPC behavior.</param>
|
||||
/// <inheritdoc />
|
||||
public Task<AcknowledgeAlarmReply> AcknowledgeAlarmAsync(
|
||||
AcknowledgeAlarmRequest request,
|
||||
CallOptions callOptions)
|
||||
@@ -218,11 +194,7 @@ internal sealed class FakeGatewayTransport(MxGatewayClientOptions options) : IMx
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records the query call and yields each enqueued snapshot.
|
||||
/// </summary>
|
||||
/// <param name="request">The query active alarms request.</param>
|
||||
/// <param name="callOptions">Call options specifying RPC behavior.</param>
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<ActiveAlarmSnapshot> QueryActiveAlarmsAsync(
|
||||
QueryActiveAlarmsRequest request,
|
||||
CallOptions callOptions)
|
||||
@@ -251,11 +223,7 @@ internal sealed class FakeGatewayTransport(MxGatewayClientOptions options) : IMx
|
||||
_activeAlarmSnapshots.Add(snapshot);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records the stream-alarms call and yields each enqueued feed message.
|
||||
/// </summary>
|
||||
/// <param name="request">The stream alarms request.</param>
|
||||
/// <param name="callOptions">Call options specifying RPC behavior.</param>
|
||||
/// <inheritdoc />
|
||||
public async IAsyncEnumerable<AlarmFeedMessage> StreamAlarmsAsync(
|
||||
StreamAlarmsRequest request,
|
||||
CallOptions callOptions)
|
||||
|
||||
@@ -9,6 +9,7 @@ public sealed class GalaxyRepositoryClientTests
|
||||
/// <summary>
|
||||
/// Verifies that TestConnectionAsync attaches the API key in request metadata and returns the Ok flag.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task TestConnectionAsync_AttachesApiKeyMetadataAndReturnsOkFlag()
|
||||
{
|
||||
@@ -27,6 +28,7 @@ public sealed class GalaxyRepositoryClientTests
|
||||
/// <summary>
|
||||
/// Verifies that TestConnectionAsync returns false when the server reports NotOk.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task TestConnectionAsync_ReturnsFalseWhenServerReportsNotOk()
|
||||
{
|
||||
@@ -42,6 +44,7 @@ public sealed class GalaxyRepositoryClientTests
|
||||
/// <summary>
|
||||
/// Verifies that GetLastDeployTimeAsync returns null when the server reports not present.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task GetLastDeployTimeAsync_ReturnsNullWhenNotPresent()
|
||||
{
|
||||
@@ -58,6 +61,7 @@ public sealed class GalaxyRepositoryClientTests
|
||||
/// <summary>
|
||||
/// Verifies that GetLastDeployTimeAsync returns the timestamp when the server reports it present.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task GetLastDeployTimeAsync_ReturnsTimestampWhenPresent()
|
||||
{
|
||||
@@ -79,6 +83,7 @@ public sealed class GalaxyRepositoryClientTests
|
||||
/// <summary>
|
||||
/// Verifies that DiscoverHierarchyAsync returns the objects from the server reply.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task DiscoverHierarchyAsync_ReturnsObjectsFromReply()
|
||||
{
|
||||
@@ -141,6 +146,7 @@ public sealed class GalaxyRepositoryClientTests
|
||||
/// <summary>
|
||||
/// Verifies that DiscoverHierarchyAsync propagates cancellation tokens to the transport.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task DiscoverHierarchyAsync_PropagatesCancellationToTransport()
|
||||
{
|
||||
@@ -161,6 +167,7 @@ public sealed class GalaxyRepositoryClientTests
|
||||
/// <summary>
|
||||
/// Verifies that TestConnectionAsync retries on transient gRPC failures.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task DiscoverHierarchyAsync_WithRepeatedPageToken_ThrowsProtocolError()
|
||||
{
|
||||
@@ -184,6 +191,7 @@ public sealed class GalaxyRepositoryClientTests
|
||||
/// <summary>
|
||||
/// Verifies that DiscoverHierarchyAsync maps typed filter options correctly to the request.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task DiscoverHierarchyAsync_WithOptions_MapsTypedFilters()
|
||||
{
|
||||
@@ -218,6 +226,7 @@ public sealed class GalaxyRepositoryClientTests
|
||||
/// <summary>
|
||||
/// Verifies that TestConnectionAsync retries on transient gRPC failures.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task TestConnectionAsync_RetriesOnTransientGrpcFailure()
|
||||
{
|
||||
@@ -235,6 +244,7 @@ public sealed class GalaxyRepositoryClientTests
|
||||
/// <summary>
|
||||
/// Verifies that DiscoverHierarchyAsync retries on transient gRPC failures.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task DiscoverHierarchyAsync_RetriesOnTransientGrpcFailure()
|
||||
{
|
||||
@@ -251,6 +261,7 @@ public sealed class GalaxyRepositoryClientTests
|
||||
/// <summary>
|
||||
/// Verifies that WatchDeployEventsAsync delivers the bootstrap event.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WatchDeployEventsAsync_DeliversBootstrapEvent()
|
||||
{
|
||||
@@ -287,6 +298,7 @@ public sealed class GalaxyRepositoryClientTests
|
||||
/// <summary>
|
||||
/// Verifies that WatchDeployEventsAsync delivers multiple events in order.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WatchDeployEventsAsync_DeliversMultipleEventsInOrder()
|
||||
{
|
||||
@@ -325,6 +337,7 @@ public sealed class GalaxyRepositoryClientTests
|
||||
/// <summary>
|
||||
/// Verifies that WatchDeployEventsAsync stops iteration cleanly when cancelled.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WatchDeployEventsAsync_CancellationStopsIterationCleanly()
|
||||
{
|
||||
@@ -369,6 +382,7 @@ public sealed class GalaxyRepositoryClientTests
|
||||
/// <summary>
|
||||
/// Verifies that WatchDeployEventsAsync throws ObjectDisposedException after the client is disposed.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WatchDeployEventsAsync_ThrowsAfterDisposal()
|
||||
{
|
||||
@@ -384,6 +398,7 @@ public sealed class GalaxyRepositoryClientTests
|
||||
/// <summary>
|
||||
/// Verifies that TestConnectionAsync throws ObjectDisposedException after the client is disposed.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task TestConnectionAsync_ThrowsAfterDisposal()
|
||||
{
|
||||
|
||||
@@ -12,6 +12,7 @@ public sealed class LazyBrowseNodeTests
|
||||
/// Verifies that calling BrowseAsync with no parent returns the root nodes
|
||||
/// from the first BrowseChildren reply and surfaces the per-child has-children hint.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Browse_NoParent_ReturnsRoots()
|
||||
{
|
||||
@@ -36,6 +37,7 @@ public sealed class LazyBrowseNodeTests
|
||||
/// <summary>
|
||||
/// Verifies that ExpandAsync populates Children and marks the node expanded after one RPC.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Expand_PopulatesChildrenAndMarksExpanded()
|
||||
{
|
||||
@@ -62,6 +64,7 @@ public sealed class LazyBrowseNodeTests
|
||||
/// <summary>
|
||||
/// Verifies that a second ExpandAsync call is a no-op and issues no additional RPC.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Expand_CalledTwice_NoSecondRpc()
|
||||
{
|
||||
@@ -86,6 +89,7 @@ public sealed class LazyBrowseNodeTests
|
||||
/// <summary>
|
||||
/// Verifies that an RPC failure (NotFound) during expand is wrapped in MxGatewayException.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Expand_UnknownParent_ThrowsMxGatewayException()
|
||||
{
|
||||
@@ -113,6 +117,7 @@ public sealed class LazyBrowseNodeTests
|
||||
/// <summary>
|
||||
/// Verifies that ExpandAsync drains multi-page sibling replies and forwards the page token.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Expand_MultiPageSiblings_GathersAllPages()
|
||||
{
|
||||
@@ -147,6 +152,7 @@ public sealed class LazyBrowseNodeTests
|
||||
/// <summary>
|
||||
/// Verifies that ten concurrent ExpandAsync calls issue exactly one RPC, not ten.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Expand_CalledConcurrently_OnlyFiresOneRpc()
|
||||
{
|
||||
@@ -179,8 +185,9 @@ public sealed class LazyBrowseNodeTests
|
||||
/// Verifies that reading Children/IsExpanded concurrently with an in-flight ExpandAsync
|
||||
/// never throws (no torn enumeration of a mid-append list) and, once IsExpanded flips to
|
||||
/// true, the published Children snapshot is fully populated. Pins the safe-publication
|
||||
/// contract on the lock-free readers (Client.Dotnet-025).
|
||||
/// contract on the lock-free readers.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Expand_ConcurrentReadOfChildren_NeverTearsAndPublishesAtomically()
|
||||
{
|
||||
@@ -268,6 +275,7 @@ public sealed class LazyBrowseNodeTests
|
||||
/// <summary>
|
||||
/// Verifies that BrowseChildrenOptions filter fields are forwarded to the BrowseChildren request.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Browse_WithFilter_ForwardsToRequest()
|
||||
{
|
||||
|
||||
@@ -12,6 +12,7 @@ namespace ZB.MOM.WW.MxGateway.Client.Tests;
|
||||
public sealed class MxGatewayClientAlarmsTests
|
||||
{
|
||||
/// <summary>AcknowledgeAlarmAsync records request and returns reply.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task AcknowledgeAlarmAsync_RecordsRequestShapeAndReturnsReply()
|
||||
{
|
||||
@@ -48,6 +49,7 @@ public sealed class MxGatewayClientAlarmsTests
|
||||
}
|
||||
|
||||
/// <summary>AcknowledgeAlarmAsync honors cancellation.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task AcknowledgeAlarmAsync_HonorsCancellation()
|
||||
{
|
||||
@@ -72,6 +74,7 @@ public sealed class MxGatewayClientAlarmsTests
|
||||
}
|
||||
|
||||
/// <summary>AcknowledgeAlarmAsync maps unauthenticated RPC exception to typed exception.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task AcknowledgeAlarmAsync_MapsUnauthenticated_RpcException_ToTypedException()
|
||||
{
|
||||
@@ -97,6 +100,7 @@ public sealed class MxGatewayClientAlarmsTests
|
||||
}
|
||||
|
||||
/// <summary>QueryActiveAlarmsAsync streams enqueued snapshots.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task QueryActiveAlarmsAsync_StreamsEnqueuedSnapshots()
|
||||
{
|
||||
@@ -122,6 +126,7 @@ public sealed class MxGatewayClientAlarmsTests
|
||||
}
|
||||
|
||||
/// <summary>QueryActiveAlarmsAsync passes filter prefix.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task QueryActiveAlarmsAsync_PassesFilterPrefix()
|
||||
{
|
||||
@@ -142,6 +147,7 @@ public sealed class MxGatewayClientAlarmsTests
|
||||
}
|
||||
|
||||
/// <summary>QueryActiveAlarmsAsync honors cancellation during enumeration.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task QueryActiveAlarmsAsync_HonorsCancellationDuringEnumeration()
|
||||
{
|
||||
|
||||
@@ -24,6 +24,7 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the version command with --json flag prints JSON protocol versions.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_VersionJson_PrintsJsonProtocolVersions()
|
||||
{
|
||||
@@ -38,6 +39,7 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that the write command builds a write request and prints JSON reply.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_Write_BuildsWriteCommandAndPrintsJsonReply()
|
||||
{
|
||||
@@ -83,7 +85,7 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Client.Dotnet-030: <c>advise-supervisory</c> was present in the command
|
||||
/// <c>advise-supervisory</c> was present in the command
|
||||
/// dispatch table but absent from <see cref="MxGatewayClientCli"/>'s
|
||||
/// <c>IsKnownGatewayCommand</c> guard, so the guard intercepted it first and
|
||||
/// returned exit code 2 "Unknown command" before dispatch could run. This
|
||||
@@ -91,6 +93,7 @@ public sealed class MxGatewayClientCliTests
|
||||
/// "Unknown command") and reaches the dispatch handler (exit 0, reply written
|
||||
/// to stdout).
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_AdviseSupervisory_IsRecognizedAndReachesDispatch()
|
||||
{
|
||||
@@ -244,6 +247,7 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that error output redacts sensitive API key values.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_ErrorOutput_RedactsApiKey()
|
||||
{
|
||||
@@ -268,12 +272,13 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Client.Dotnet-028: when the API key is sourced from the env var
|
||||
/// When the API key is sourced from the env var
|
||||
/// (<c>--api-key-env</c> path, no <c>--api-key</c> flag), the error-redaction
|
||||
/// catch block must still resolve and redact the effective key. Regression
|
||||
/// guard for the catch block reverting to <c>GetOptional("api-key")</c> only,
|
||||
/// which is null on the env-var path and leaves the key unredacted.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_ErrorOutput_RedactsApiKey_WhenSourcedFromEnvironmentVariable()
|
||||
{
|
||||
@@ -310,6 +315,7 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that stream-events with max-events limit stops output in non-JSON format.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_StreamEvents_WithMaxEventsStopsNonJsonOutput()
|
||||
{
|
||||
@@ -352,6 +358,7 @@ public sealed class MxGatewayClientCliTests
|
||||
|
||||
|
||||
/// <summary>Verifies that stream-alarms with --max-events stops output and distinguishes payload cases.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_StreamAlarms_WithMaxEventsStopsAndDistinguishesPayloadCases()
|
||||
{
|
||||
@@ -391,6 +398,7 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that acknowledge-alarm builds a request and prints the JSON reply.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_AcknowledgeAlarm_BuildsRequestAndPrintsJsonReply()
|
||||
{
|
||||
@@ -433,6 +441,7 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that smoke command closes opened session when a command fails.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_Smoke_WhenCommandFails_ClosesOpenedSession()
|
||||
{
|
||||
@@ -464,6 +473,7 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that galaxy-test-connection command prints JSON reply.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_GalaxyTestConnection_PrintsJsonReply()
|
||||
{
|
||||
@@ -494,6 +504,7 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that galaxy-discover command prints hierarchy summary.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_GalaxyDiscover_PrintsHierarchySummary()
|
||||
{
|
||||
@@ -567,6 +578,7 @@ public sealed class MxGatewayClientCliTests
|
||||
/// Verifies galaxy-browse walks root objects and eagerly expands one further
|
||||
/// level when --depth 1 is passed, printing an indented tree.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_GalaxyBrowse_TextTreeExpandsToDepth()
|
||||
{
|
||||
@@ -623,6 +635,7 @@ public sealed class MxGatewayClientCliTests
|
||||
/// Verifies galaxy-browse --json emits a nested JSON document and forwards
|
||||
/// the filter flags onto the BrowseChildren request.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_GalaxyBrowse_JsonForwardsFilters()
|
||||
{
|
||||
@@ -667,6 +680,7 @@ public sealed class MxGatewayClientCliTests
|
||||
/// Verifies galaxy-browse --parent fetches exactly one level of children for
|
||||
/// the supplied gobject id via a parent-scoped BrowseChildren request.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_GalaxyBrowse_ParentFetchesSingleLevel()
|
||||
{
|
||||
@@ -704,6 +718,7 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that galaxy-watch command prints text output for deploy events.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_GalaxyWatch_PrintsTextOutputForEvents()
|
||||
{
|
||||
@@ -758,6 +773,7 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that galaxy-watch with --json emits one JSON object per event.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_GalaxyWatch_JsonEmitsOneObjectPerEvent()
|
||||
{
|
||||
@@ -793,6 +809,7 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that batch mode dispatches a single version command and emits the EOR sentinel.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_Batch_DispatchesVersionAndWritesEndOfRecord()
|
||||
{
|
||||
@@ -819,6 +836,7 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that batch mode routes per-command errors to stdout as JSON between EOR markers.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_Batch_WritesErrorsToStdoutAsJson()
|
||||
{
|
||||
@@ -853,7 +871,7 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Client.Dotnet-018: the README CLI examples for the alarm subcommands at
|
||||
/// The README CLI examples for the alarm subcommands at
|
||||
/// `clients/dotnet/README.md` must drive cleanly through the production
|
||||
/// CLI argument parser. The previous text used non-existent flags
|
||||
/// (`--session-id`, `--max-messages`, `--alarm-reference`) that would
|
||||
@@ -863,6 +881,7 @@ public sealed class MxGatewayClientCliTests
|
||||
/// against exit code 0.
|
||||
/// </summary>
|
||||
/// <param name="command">The alarm subcommand to validate (e.g. "stream-alarms", "acknowledge-alarm").</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Theory]
|
||||
[InlineData("stream-alarms")]
|
||||
[InlineData("acknowledge-alarm")]
|
||||
@@ -911,12 +930,13 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Client.Dotnet-019: `BenchReadBulkAsync` previously fell back to
|
||||
/// `BenchReadBulkAsync` previously fell back to
|
||||
/// <c>reply.ReturnValue.Int32Value</c> when the register reply had no
|
||||
/// typed <c>Register</c> payload, silently driving the rest of the bench
|
||||
/// against a zero server handle. The fix must fail loudly with a
|
||||
/// descriptive <see cref="MxGatewayException"/>.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_BenchReadBulk_WhenRegisterReplyMissingTypedPayload_FailsLoudly()
|
||||
{
|
||||
@@ -924,9 +944,6 @@ public sealed class MxGatewayClientCliTests
|
||||
using var error = new StringWriter();
|
||||
FakeCliClient fakeClient = new();
|
||||
// Successful protocol + MX status but no typed `Register` payload.
|
||||
// Before the Client.Dotnet-019 fix this silently became serverHandle=0
|
||||
// and the bench proceeded through SubscribeBulk / warmup / steady-state
|
||||
// against an invalid handle, producing a misleading zero-result summary.
|
||||
fakeClient.InvokeReplies.Enqueue(new MxCommandReply
|
||||
{
|
||||
SessionId = "session-fixture",
|
||||
@@ -961,12 +978,13 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Client.Dotnet-020: the steady-state loop in `BenchReadBulkAsync` had a
|
||||
/// The steady-state loop in `BenchReadBulkAsync` had a
|
||||
/// bare `catch { failedCalls++; continue; }` that swallowed
|
||||
/// <see cref="OperationCanceledException"/>, so token-driven cancellation
|
||||
/// kept spinning until <c>--duration-seconds</c> elapsed. After the fix
|
||||
/// the bench must exit promptly when the supplied token cancels.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_BenchReadBulk_WhenSteadyStateLoopReceivesCancellation_ExitsPromptly()
|
||||
{
|
||||
@@ -1055,12 +1073,13 @@ public sealed class MxGatewayClientCliTests
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Client.Dotnet-021: both `ReadBulkAsync` and `BenchReadBulkAsync` cast
|
||||
/// Both `ReadBulkAsync` and `BenchReadBulkAsync` cast
|
||||
/// the user-supplied <c>--timeout-ms</c> to <see cref="uint"/> without
|
||||
/// bounds checking, so a negative value (e.g. <c>-1</c>) silently wraps
|
||||
/// to ~49.7 days. The fix must reject negatives with a clear error.
|
||||
/// </summary>
|
||||
/// <param name="command">The bulk-read subcommand to validate (e.g. "read-bulk", "bench-read-bulk").</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Theory]
|
||||
[InlineData("read-bulk")]
|
||||
[InlineData("bench-read-bulk")]
|
||||
@@ -1223,7 +1242,8 @@ public sealed class MxGatewayClientCliTests
|
||||
/// <summary>Optional per-call handler that overrides queue-based behaviour.</summary>
|
||||
public Func<MxCommandRequest, CancellationToken, Task<MxCommandReply>>? InvokeHandler { get; init; }
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>Disposes the fake client. No-op; there are no unmanaged resources to release.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
return ValueTask.CompletedTask;
|
||||
|
||||
@@ -7,6 +7,7 @@ namespace ZB.MOM.WW.MxGateway.Client.Tests;
|
||||
public sealed class MxGatewayClientSessionTests
|
||||
{
|
||||
/// <summary>Verifies that open session attaches API key metadata and cancellation token.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task OpenSessionRawAsync_AttachesApiKeyMetadataAndCancellation()
|
||||
{
|
||||
@@ -22,6 +23,7 @@ public sealed class MxGatewayClientSessionTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that open session returns a session with the raw open reply.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task OpenSessionAsync_ReturnsSessionWithRawOpenReply()
|
||||
{
|
||||
@@ -37,6 +39,7 @@ public sealed class MxGatewayClientSessionTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that register builds a register command and returns server handle.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RegisterAsync_BuildsRegisterCommandAndReturnsServerHandle()
|
||||
{
|
||||
@@ -62,6 +65,7 @@ public sealed class MxGatewayClientSessionTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that add item 2 builds a command with the specified context.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task AddItem2Async_BuildsAddItem2CommandWithContext()
|
||||
{
|
||||
@@ -87,6 +91,7 @@ public sealed class MxGatewayClientSessionTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that write raw builds a write command with the raw value.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WriteRawAsync_BuildsWriteCommandWithRawValue()
|
||||
{
|
||||
@@ -118,6 +123,7 @@ public sealed class MxGatewayClientSessionTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that write 2 raw builds a write 2 command with value and timestamp.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Write2RawAsync_BuildsWrite2CommandWithValueAndTimestamp()
|
||||
{
|
||||
@@ -146,6 +152,7 @@ public sealed class MxGatewayClientSessionTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that subscribe bulk builds one command and returns per-item results.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task SubscribeBulkAsync_BuildsOneBulkCommandAndReturnsPerItemResults()
|
||||
{
|
||||
@@ -185,6 +192,7 @@ public sealed class MxGatewayClientSessionTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that stream events yields events in the order received from the gateway.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StreamEventsAsync_YieldsEventsInGatewayOrder()
|
||||
{
|
||||
@@ -265,6 +273,7 @@ public sealed class MxGatewayClientSessionTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that close is explicit and idempotent.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CloseAsync_IsExplicitAndIdempotent()
|
||||
{
|
||||
@@ -281,6 +290,7 @@ public sealed class MxGatewayClientSessionTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that invoke retries safe diagnostic commands on transient RPC failure.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task InvokeAsync_RetriesSafeDiagnosticCommandOnTransientGrpcFailure()
|
||||
{
|
||||
@@ -305,6 +315,7 @@ public sealed class MxGatewayClientSessionTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that open session does not retry on transient RPC failure.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task OpenSessionAsync_DoesNotRetryTransientGrpcFailure()
|
||||
{
|
||||
@@ -318,6 +329,7 @@ public sealed class MxGatewayClientSessionTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that invoke does not retry write commands on transient RPC failure.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task InvokeAsync_DoesNotRetryWriteCommand()
|
||||
{
|
||||
@@ -333,6 +345,7 @@ public sealed class MxGatewayClientSessionTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that invoke helpers pass cancellation token to the transport.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task InvokeHelpers_PassCancellationTokenToTransport()
|
||||
{
|
||||
@@ -379,6 +392,7 @@ public sealed class MxGatewayClientSessionTests
|
||||
}
|
||||
|
||||
/// <summary>Verifies that WriteArrayElementsAsync builds a write command whose value is a SparseArrayValue.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task WriteArrayElementsAsync_BuildsWriteCommandWithSparseArrayValue()
|
||||
{
|
||||
|
||||
@@ -3,6 +3,7 @@ namespace ZB.MOM.WW.MxGateway.Client.Tests;
|
||||
public sealed class MxGatewayGeneratedContractTests
|
||||
{
|
||||
/// <summary>Verifies that the generated gRPC client can be instantiated from the client factory.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task GeneratedGrpcClient_CanBeConstructedFromClientFactory()
|
||||
{
|
||||
|
||||
@@ -337,6 +337,9 @@ public sealed class GalaxyRepositoryClient : IAsyncDisposable
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
/// <summary>Builds a BrowseChildren request from the caller-supplied filter options.</summary>
|
||||
/// <param name="options">The browse-children filter options.</param>
|
||||
/// <returns>The populated request.</returns>
|
||||
internal static BrowseChildrenRequest BuildBrowseChildrenRequest(BrowseChildrenOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
@@ -424,6 +427,7 @@ public sealed class GalaxyRepositoryClient : IAsyncDisposable
|
||||
/// <summary>
|
||||
/// Closes the gRPC channel and releases resources.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed)
|
||||
@@ -493,6 +497,9 @@ public sealed class GalaxyRepositoryClient : IAsyncDisposable
|
||||
private static HttpMessageHandler CreateHttpHandler(MxGatewayClientOptions options) =>
|
||||
CreateHttpHandlerForTests(options);
|
||||
|
||||
/// <summary>Creates the <see cref="SocketsHttpHandler"/> used for gRPC calls, exposed for test configuration.</summary>
|
||||
/// <param name="options">The client options supplying TLS and timeout settings.</param>
|
||||
/// <returns>The configured HTTP handler.</returns>
|
||||
internal static SocketsHttpHandler CreateHttpHandlerForTests(MxGatewayClientOptions options)
|
||||
{
|
||||
SocketsHttpHandler handler = new()
|
||||
|
||||
@@ -10,9 +10,7 @@ internal sealed class GrpcGalaxyRepositoryClientTransport(
|
||||
MxGatewayClientOptions options,
|
||||
GalaxyRepository.GalaxyRepositoryClient rawClient) : IGalaxyRepositoryClientTransport
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the gateway client options.
|
||||
/// </summary>
|
||||
/// <inheritdoc />
|
||||
public MxGatewayClientOptions Options { get; } = options;
|
||||
|
||||
/// <summary>
|
||||
@@ -91,7 +89,16 @@ internal sealed class GrpcGalaxyRepositoryClientTransport(
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>
|
||||
/// Watches for deployment events from the Galaxy Repository server.
|
||||
/// </summary>
|
||||
/// <param name="request">The watch deploy events request.</param>
|
||||
/// <param name="callOptions">gRPC call options (timeout, cancellation, etc.).</param>
|
||||
/// <param name="cancellationToken">
|
||||
/// A cancellation token distinct from <paramref name="callOptions"/>'s; when cancelable, it
|
||||
/// takes precedence over the token carried by <paramref name="callOptions"/>.
|
||||
/// </param>
|
||||
/// <returns>An asynchronous stream of deploy events.</returns>
|
||||
public async IAsyncEnumerable<DeployEvent> WatchDeployEventsAsync(
|
||||
WatchDeployEventsRequest request,
|
||||
CallOptions callOptions,
|
||||
|
||||
@@ -10,9 +10,7 @@ internal sealed class GrpcMxGatewayClientTransport(
|
||||
MxGatewayClientOptions options,
|
||||
MxAccessGateway.MxAccessGatewayClient rawClient) : IMxGatewayClientTransport
|
||||
{
|
||||
/// <summary>
|
||||
/// Gets the gateway client options.
|
||||
/// </summary>
|
||||
/// <inheritdoc />
|
||||
public MxGatewayClientOptions Options { get; } = options;
|
||||
|
||||
/// <summary>
|
||||
@@ -74,7 +72,13 @@ internal sealed class GrpcMxGatewayClientTransport(
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>
|
||||
/// Streams events from the session.
|
||||
/// </summary>
|
||||
/// <param name="request">The stream events request.</param>
|
||||
/// <param name="callOptions">gRPC call options.</param>
|
||||
/// <param name="cancellationToken">Cancellation token; falls back to <paramref name="callOptions"/>'s token when not cancelable.</param>
|
||||
/// <returns>An async enumerable of events.</returns>
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync(
|
||||
StreamEventsRequest request,
|
||||
CallOptions callOptions,
|
||||
@@ -133,7 +137,14 @@ internal sealed class GrpcMxGatewayClientTransport(
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>
|
||||
/// Streams a snapshot of all alarms currently in Active or ActiveAcked state — the
|
||||
/// ConditionRefresh equivalent for the gateway.
|
||||
/// </summary>
|
||||
/// <param name="request">The query request, optionally scoped by alarm-reference prefix.</param>
|
||||
/// <param name="callOptions">gRPC call options.</param>
|
||||
/// <param name="cancellationToken">Cancellation token; falls back to <paramref name="callOptions"/>'s token when not cancelable.</param>
|
||||
/// <returns>An async enumerable of active-alarm snapshots.</returns>
|
||||
public async IAsyncEnumerable<ActiveAlarmSnapshot> QueryActiveAlarmsAsync(
|
||||
QueryActiveAlarmsRequest request,
|
||||
CallOptions callOptions,
|
||||
@@ -175,7 +186,14 @@ internal sealed class GrpcMxGatewayClientTransport(
|
||||
return QueryActiveAlarmsAsync(request, callOptions);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>
|
||||
/// Attaches to the gateway's central alarm feed — the current active-alarm
|
||||
/// snapshot followed by live transitions.
|
||||
/// </summary>
|
||||
/// <param name="request">The stream request, optionally scoped by alarm-reference prefix.</param>
|
||||
/// <param name="callOptions">gRPC call options.</param>
|
||||
/// <param name="cancellationToken">Cancellation token; falls back to <paramref name="callOptions"/>'s token when not cancelable.</param>
|
||||
/// <returns>An async enumerable of alarm feed messages.</returns>
|
||||
public async IAsyncEnumerable<AlarmFeedMessage> StreamAlarmsAsync(
|
||||
StreamAlarmsRequest request,
|
||||
CallOptions callOptions,
|
||||
|
||||
@@ -15,6 +15,7 @@ internal interface IGalaxyRepositoryClientTransport
|
||||
/// <summary>Tests the connection to the Galaxy Repository server.</summary>
|
||||
/// <param name="request">The test connection request.</param>
|
||||
/// <param name="callOptions">gRPC call options (timeout, cancellation, etc.).</param>
|
||||
/// <returns>The test connection reply.</returns>
|
||||
Task<TestConnectionReply> TestConnectionAsync(
|
||||
TestConnectionRequest request,
|
||||
CallOptions callOptions);
|
||||
@@ -22,6 +23,7 @@ internal interface IGalaxyRepositoryClientTransport
|
||||
/// <summary>Gets the last deploy time from the Galaxy Repository server.</summary>
|
||||
/// <param name="request">The get last deploy time request.</param>
|
||||
/// <param name="callOptions">gRPC call options (timeout, cancellation, etc.).</param>
|
||||
/// <returns>The last deploy time reply.</returns>
|
||||
Task<GetLastDeployTimeReply> GetLastDeployTimeAsync(
|
||||
GetLastDeployTimeRequest request,
|
||||
CallOptions callOptions);
|
||||
@@ -29,6 +31,7 @@ internal interface IGalaxyRepositoryClientTransport
|
||||
/// <summary>Discovers the object hierarchy in the Galaxy Repository.</summary>
|
||||
/// <param name="request">The discover hierarchy request.</param>
|
||||
/// <param name="callOptions">gRPC call options (timeout, cancellation, etc.).</param>
|
||||
/// <returns>The discovered hierarchy reply.</returns>
|
||||
Task<DiscoverHierarchyReply> DiscoverHierarchyAsync(
|
||||
DiscoverHierarchyRequest request,
|
||||
CallOptions callOptions);
|
||||
@@ -36,6 +39,7 @@ internal interface IGalaxyRepositoryClientTransport
|
||||
/// <summary>Returns direct children of a parent in the Galaxy hierarchy.</summary>
|
||||
/// <param name="request">The browse children request.</param>
|
||||
/// <param name="callOptions">gRPC call options (timeout, cancellation, etc.).</param>
|
||||
/// <returns>The browse children reply.</returns>
|
||||
Task<BrowseChildrenReply> BrowseChildrenAsync(
|
||||
BrowseChildrenRequest request,
|
||||
CallOptions callOptions);
|
||||
@@ -43,6 +47,7 @@ internal interface IGalaxyRepositoryClientTransport
|
||||
/// <summary>Watches for deployment events from the Galaxy Repository server.</summary>
|
||||
/// <param name="request">The watch deploy events request.</param>
|
||||
/// <param name="callOptions">gRPC call options (timeout, cancellation, etc.).</param>
|
||||
/// <returns>An asynchronous stream of deploy events.</returns>
|
||||
IAsyncEnumerable<DeployEvent> WatchDeployEventsAsync(
|
||||
WatchDeployEventsRequest request,
|
||||
CallOptions callOptions);
|
||||
|
||||
@@ -12,7 +12,7 @@ public sealed class LazyBrowseNode
|
||||
{
|
||||
private readonly GalaxyRepositoryClient _client;
|
||||
private readonly BrowseChildrenOptions _options;
|
||||
// Client.Dotnet-027 (Won't Fix): this gate is used only via WaitAsync/Release and
|
||||
// Won't Fix: this gate is used only via WaitAsync/Release and
|
||||
// never via AvailableWaitHandle, so SemaphoreSlim allocates no kernel wait handle —
|
||||
// it holds no unmanaged/OS handle to leak. It is pure managed memory whose lifetime
|
||||
// is the node's, so the type is intentionally not IDisposable: making it disposable
|
||||
@@ -27,6 +27,11 @@ public sealed class LazyBrowseNode
|
||||
private IReadOnlyList<LazyBrowseNode> _children = [];
|
||||
private volatile bool _isExpanded;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="LazyBrowseNode"/> class.</summary>
|
||||
/// <param name="client">The Galaxy Repository client used to fetch children on expansion.</param>
|
||||
/// <param name="object">The underlying Galaxy object for this node.</param>
|
||||
/// <param name="hasChildrenHint">Whether the server reports this node has at least one matching descendant.</param>
|
||||
/// <param name="options">The browse options to apply when fetching children.</param>
|
||||
internal LazyBrowseNode(
|
||||
GalaxyRepositoryClient client,
|
||||
GalaxyObject @object,
|
||||
@@ -66,6 +71,7 @@ public sealed class LazyBrowseNode
|
||||
/// partially-built list.
|
||||
/// </remarks>
|
||||
/// <param name="cancellationToken">Token to observe for cancellation.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async Task ExpandAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (_isExpanded)
|
||||
|
||||
@@ -7,6 +7,7 @@ public static class MxCommandReplyExtensions
|
||||
{
|
||||
/// <summary>Validates that the reply has a successful protocol status (Ok or MxAccessFailure), throwing a gateway exception if not.</summary>
|
||||
/// <param name="reply">The command reply to check.</param>
|
||||
/// <returns>The same reply, for chaining.</returns>
|
||||
public static MxCommandReply EnsureProtocolSuccess(this MxCommandReply reply)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(reply);
|
||||
@@ -24,6 +25,7 @@ public static class MxCommandReplyExtensions
|
||||
|
||||
/// <summary>Validates that the reply indicates MXAccess success (no HResult or status failures), throwing MxAccessException if not.</summary>
|
||||
/// <param name="reply">The command reply to check.</param>
|
||||
/// <returns>The same reply, for chaining.</returns>
|
||||
public static MxCommandReply EnsureMxAccessSuccess(this MxCommandReply reply)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(reply);
|
||||
|
||||
@@ -249,6 +249,7 @@ public sealed class MxGatewayClient : IAsyncDisposable
|
||||
/// <summary>
|
||||
/// Disposes the client and releases all resources.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed)
|
||||
@@ -318,6 +319,11 @@ public sealed class MxGatewayClient : IAsyncDisposable
|
||||
private static HttpMessageHandler CreateHttpHandler(MxGatewayClientOptions options) =>
|
||||
CreateHttpHandlerForTests(options);
|
||||
|
||||
/// <summary>
|
||||
/// Creates the <see cref="SocketsHttpHandler"/> used for gRPC calls; exposed internally so tests can inspect it.
|
||||
/// </summary>
|
||||
/// <param name="options">The client options that configure connect timeout and TLS behavior.</param>
|
||||
/// <returns>A configured <see cref="SocketsHttpHandler"/>.</returns>
|
||||
internal static SocketsHttpHandler CreateHttpHandlerForTests(MxGatewayClientOptions options)
|
||||
{
|
||||
SocketsHttpHandler handler = new()
|
||||
|
||||
@@ -12,6 +12,7 @@ internal static class MxGatewayClientRetryPolicy
|
||||
/// <summary>Creates a Polly ResiliencePipeline that retries transient gRPC failures with exponential backoff.</summary>
|
||||
/// <param name="options">Retry configuration (max attempts, delay bounds, jitter).</param>
|
||||
/// <param name="logger">Optional logger for retry diagnostics.</param>
|
||||
/// <returns>A configured resilience pipeline that retries transient gRPC failures.</returns>
|
||||
public static ResiliencePipeline Create(
|
||||
MxGatewayClientRetryOptions options,
|
||||
ILogger? logger)
|
||||
@@ -42,6 +43,7 @@ internal static class MxGatewayClientRetryPolicy
|
||||
|
||||
/// <summary>Returns whether a command kind is eligible for automatic retry on transient failures.</summary>
|
||||
/// <param name="kind">The command kind to check.</param>
|
||||
/// <returns><see langword="true"/> if the command kind may be automatically retried; otherwise, <see langword="false"/>.</returns>
|
||||
public static bool IsRetryableCommand(MxCommandKind kind)
|
||||
{
|
||||
return kind is MxCommandKind.Ping
|
||||
|
||||
@@ -211,6 +211,7 @@ public sealed class MxGatewaySession : IAsyncDisposable
|
||||
/// <param name="serverHandle">The ServerHandle from register.</param>
|
||||
/// <param name="itemHandle">The ItemHandle from add-item.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async Task AdviseAsync(
|
||||
int serverHandle,
|
||||
int itemHandle,
|
||||
@@ -252,6 +253,7 @@ public sealed class MxGatewaySession : IAsyncDisposable
|
||||
/// <param name="serverHandle">The ServerHandle from register.</param>
|
||||
/// <param name="itemHandle">The ItemHandle from add-item.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async Task UnAdviseAsync(
|
||||
int serverHandle,
|
||||
int itemHandle,
|
||||
@@ -293,6 +295,7 @@ public sealed class MxGatewaySession : IAsyncDisposable
|
||||
/// <param name="serverHandle">The ServerHandle from register.</param>
|
||||
/// <param name="itemHandle">The ItemHandle from add-item.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async Task RemoveItemAsync(
|
||||
int serverHandle,
|
||||
int itemHandle,
|
||||
@@ -675,6 +678,7 @@ public sealed class MxGatewaySession : IAsyncDisposable
|
||||
/// <param name="value">The value to write.</param>
|
||||
/// <param name="userId">User ID context for the write.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async Task WriteAsync(
|
||||
int serverHandle,
|
||||
int itemHandle,
|
||||
@@ -704,6 +708,7 @@ public sealed class MxGatewaySession : IAsyncDisposable
|
||||
/// <param name="elements">Map of zero-based array index to scalar <see cref="MxValue"/>.</param>
|
||||
/// <param name="userId">User ID context for the write.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public Task WriteArrayElementsAsync(
|
||||
int serverHandle,
|
||||
int itemHandle,
|
||||
@@ -786,6 +791,7 @@ public sealed class MxGatewaySession : IAsyncDisposable
|
||||
/// <param name="timestampValue">The timestamp to write with the value.</param>
|
||||
/// <param name="userId">User ID context for the write.</param>
|
||||
/// <param name="cancellationToken">Cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async Task Write2Async(
|
||||
int serverHandle,
|
||||
int itemHandle,
|
||||
@@ -1425,6 +1431,7 @@ public sealed class MxGatewaySession : IAsyncDisposable
|
||||
/// <summary>
|
||||
/// Closes the session and releases resources.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
await CloseAsync().ConfigureAwait(false);
|
||||
|
||||
@@ -7,6 +7,7 @@ public static class MxStatusProxyExtensions
|
||||
{
|
||||
/// <summary>Returns whether the status indicates success (success flag set and category is Ok).</summary>
|
||||
/// <param name="status">The status to check.</param>
|
||||
/// <returns><see langword="true"/> if the status indicates success; otherwise <see langword="false"/>.</returns>
|
||||
public static bool IsSuccess(this MxStatusProxy status)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(status);
|
||||
@@ -17,6 +18,7 @@ public static class MxStatusProxyExtensions
|
||||
|
||||
/// <summary>Returns a formatted summary of the status for diagnostic output.</summary>
|
||||
/// <param name="status">The status to summarize.</param>
|
||||
/// <returns>A formatted diagnostic summary string.</returns>
|
||||
public static string ToDiagnosticSummary(this MxStatusProxy status)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(status);
|
||||
|
||||
@@ -14,6 +14,7 @@ public static class MxValueExtensions
|
||||
/// Converts a boolean value to an MxValue with MxDataType.Boolean.
|
||||
/// </summary>
|
||||
/// <param name="value">Scalar boolean value to wrap.</param>
|
||||
/// <returns>An <see cref="MxValue"/> carrying the wrapped boolean.</returns>
|
||||
public static MxValue ToMxValue(this bool value)
|
||||
{
|
||||
return new MxValue
|
||||
@@ -28,6 +29,7 @@ public static class MxValueExtensions
|
||||
/// Converts a 32-bit integer value to an MxValue with MxDataType.Integer.
|
||||
/// </summary>
|
||||
/// <param name="value">32-bit integer value to wrap.</param>
|
||||
/// <returns>An <see cref="MxValue"/> carrying the wrapped integer.</returns>
|
||||
public static MxValue ToMxValue(this int value)
|
||||
{
|
||||
return new MxValue
|
||||
@@ -42,6 +44,7 @@ public static class MxValueExtensions
|
||||
/// Converts a 64-bit integer value to an MxValue with MxDataType.Integer.
|
||||
/// </summary>
|
||||
/// <param name="value">64-bit integer value to wrap.</param>
|
||||
/// <returns>An <see cref="MxValue"/> carrying the wrapped integer.</returns>
|
||||
public static MxValue ToMxValue(this long value)
|
||||
{
|
||||
return new MxValue
|
||||
@@ -56,6 +59,7 @@ public static class MxValueExtensions
|
||||
/// Converts a single-precision floating-point value to an MxValue with MxDataType.Float.
|
||||
/// </summary>
|
||||
/// <param name="value">Single-precision floating-point value to wrap.</param>
|
||||
/// <returns>An <see cref="MxValue"/> carrying the wrapped value.</returns>
|
||||
public static MxValue ToMxValue(this float value)
|
||||
{
|
||||
return new MxValue
|
||||
@@ -70,6 +74,7 @@ public static class MxValueExtensions
|
||||
/// Converts a double-precision floating-point value to an MxValue with MxDataType.Double.
|
||||
/// </summary>
|
||||
/// <param name="value">Double-precision floating-point value to wrap.</param>
|
||||
/// <returns>An <see cref="MxValue"/> carrying the wrapped value.</returns>
|
||||
public static MxValue ToMxValue(this double value)
|
||||
{
|
||||
return new MxValue
|
||||
@@ -84,6 +89,7 @@ public static class MxValueExtensions
|
||||
/// Converts a string value to an MxValue with MxDataType.String.
|
||||
/// </summary>
|
||||
/// <param name="value">String value to wrap.</param>
|
||||
/// <returns>An <see cref="MxValue"/> carrying the wrapped string.</returns>
|
||||
public static MxValue ToMxValue(this string value)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(value);
|
||||
@@ -100,6 +106,7 @@ public static class MxValueExtensions
|
||||
/// Converts a DateTimeOffset value to an MxValue with MxDataType.Time.
|
||||
/// </summary>
|
||||
/// <param name="value">DateTimeOffset value to wrap.</param>
|
||||
/// <returns>An <see cref="MxValue"/> carrying the wrapped timestamp.</returns>
|
||||
public static MxValue ToMxValue(this DateTimeOffset value)
|
||||
{
|
||||
return new MxValue
|
||||
@@ -114,6 +121,7 @@ public static class MxValueExtensions
|
||||
/// Converts a DateTime value to an MxValue with MxDataType.Time.
|
||||
/// </summary>
|
||||
/// <param name="value">DateTime value to wrap.</param>
|
||||
/// <returns>An <see cref="MxValue"/> carrying the wrapped timestamp.</returns>
|
||||
public static MxValue ToMxValue(this DateTime value)
|
||||
{
|
||||
return new DateTimeOffset(
|
||||
@@ -127,6 +135,7 @@ public static class MxValueExtensions
|
||||
/// Converts a boolean array to an MxValue with MxDataType.Boolean.
|
||||
/// </summary>
|
||||
/// <param name="values">Array of boolean values to wrap.</param>
|
||||
/// <returns>An <see cref="MxValue"/> carrying the wrapped array.</returns>
|
||||
public static MxValue ToMxValue(this IReadOnlyList<bool> values)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(values);
|
||||
@@ -145,6 +154,7 @@ public static class MxValueExtensions
|
||||
/// Converts a 32-bit integer array to an MxValue with MxDataType.Integer.
|
||||
/// </summary>
|
||||
/// <param name="values">Array of 32-bit integer values to wrap.</param>
|
||||
/// <returns>An <see cref="MxValue"/> carrying the wrapped array.</returns>
|
||||
public static MxValue ToMxValue(this IReadOnlyList<int> values)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(values);
|
||||
@@ -163,6 +173,7 @@ public static class MxValueExtensions
|
||||
/// Converts a 64-bit integer array to an MxValue with MxDataType.Integer.
|
||||
/// </summary>
|
||||
/// <param name="values">Array of 64-bit integer values to wrap.</param>
|
||||
/// <returns>An <see cref="MxValue"/> carrying the wrapped array.</returns>
|
||||
public static MxValue ToMxValue(this IReadOnlyList<long> values)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(values);
|
||||
@@ -181,6 +192,7 @@ public static class MxValueExtensions
|
||||
/// Converts a single-precision floating-point array to an MxValue with MxDataType.Float.
|
||||
/// </summary>
|
||||
/// <param name="values">Array of single-precision floating-point values to wrap.</param>
|
||||
/// <returns>An <see cref="MxValue"/> carrying the wrapped array.</returns>
|
||||
public static MxValue ToMxValue(this IReadOnlyList<float> values)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(values);
|
||||
@@ -199,6 +211,7 @@ public static class MxValueExtensions
|
||||
/// Converts a double-precision floating-point array to an MxValue with MxDataType.Double.
|
||||
/// </summary>
|
||||
/// <param name="values">Array of double-precision floating-point values to wrap.</param>
|
||||
/// <returns>An <see cref="MxValue"/> carrying the wrapped array.</returns>
|
||||
public static MxValue ToMxValue(this IReadOnlyList<double> values)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(values);
|
||||
@@ -217,6 +230,7 @@ public static class MxValueExtensions
|
||||
/// Converts a string array to an MxValue with MxDataType.String.
|
||||
/// </summary>
|
||||
/// <param name="values">Array of string values to wrap.</param>
|
||||
/// <returns>An <see cref="MxValue"/> carrying the wrapped array.</returns>
|
||||
public static MxValue ToMxValue(this IReadOnlyList<string> values)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(values);
|
||||
@@ -235,6 +249,7 @@ public static class MxValueExtensions
|
||||
/// Converts a DateTimeOffset array to an MxValue with MxDataType.Time.
|
||||
/// </summary>
|
||||
/// <param name="values">Array of DateTimeOffset values to wrap.</param>
|
||||
/// <returns>An <see cref="MxValue"/> carrying the wrapped array.</returns>
|
||||
public static MxValue ToMxValue(this IReadOnlyList<DateTimeOffset> values)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(values);
|
||||
@@ -253,6 +268,7 @@ public static class MxValueExtensions
|
||||
/// Gets the projection kind (field name) of the given MxValue's current oneof value.
|
||||
/// </summary>
|
||||
/// <param name="value">The MxValue whose oneof projection kind is returned.</param>
|
||||
/// <returns>The field name of the value's current oneof projection (e.g. "boolValue").</returns>
|
||||
public static string GetProjectionKind(this MxValue value)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(value);
|
||||
@@ -276,6 +292,7 @@ public static class MxValueExtensions
|
||||
/// Converts an MxValue to a CLR object; returns the boxed value or null for null MxValues.
|
||||
/// </summary>
|
||||
/// <param name="value">The MxValue to convert.</param>
|
||||
/// <returns>The boxed CLR value, or <see langword="null"/> if the MxValue is null.</returns>
|
||||
public static object? ToClrValue(this MxValue value)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(value);
|
||||
@@ -299,6 +316,7 @@ public static class MxValueExtensions
|
||||
/// Converts an MxArray to a CLR array; returns null if the array does not have a known element type.
|
||||
/// </summary>
|
||||
/// <param name="array">The MxArray to convert.</param>
|
||||
/// <returns>The CLR array, or <see langword="null"/> if the element type is not known.</returns>
|
||||
public static object? ToClrArrayValue(this MxArray array)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(array);
|
||||
@@ -328,6 +346,7 @@ public static class MxValueExtensions
|
||||
/// <param name="variantType">Variant type string (e.g., "VT_BSTR").</param>
|
||||
/// <param name="rawDiagnostic">Diagnostic string describing the raw value.</param>
|
||||
/// <param name="rawDataType">Optional MXAccess data type override.</param>
|
||||
/// <returns>An <see cref="MxValue"/> with MxDataType.Unknown carrying the raw payload.</returns>
|
||||
public static MxValue ToRawMxValue(
|
||||
byte[] value,
|
||||
string variantType,
|
||||
|
||||
@@ -3789,6 +3789,20 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile {
|
||||
*/
|
||||
com.google.protobuf.ByteString
|
||||
getGatewayVersionBytes();
|
||||
|
||||
/**
|
||||
* <pre>
|
||||
* Maximum worker-frame payload size, in bytes, negotiated by the gateway from its
|
||||
* configured pipe limit. The worker adopts this as its frame-protocol MaxMessageBytes
|
||||
* instead of a hard-coded default; 0 (an older gateway that never set the field) means
|
||||
* "use the worker's built-in default". Sits above the public gRPC cap by an
|
||||
* envelope-overhead margin so an accepted gRPC payload always fits one worker frame.
|
||||
* </pre>
|
||||
*
|
||||
* <code>uint32 max_frame_bytes = 4;</code>
|
||||
* @return The maxFrameBytes.
|
||||
*/
|
||||
int getMaxFrameBytes();
|
||||
}
|
||||
/**
|
||||
* Protobuf type {@code mxaccess_worker.v1.GatewayHello}
|
||||
@@ -3918,6 +3932,25 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile {
|
||||
}
|
||||
}
|
||||
|
||||
public static final int MAX_FRAME_BYTES_FIELD_NUMBER = 4;
|
||||
private int maxFrameBytes_ = 0;
|
||||
/**
|
||||
* <pre>
|
||||
* Maximum worker-frame payload size, in bytes, negotiated by the gateway from its
|
||||
* configured pipe limit. The worker adopts this as its frame-protocol MaxMessageBytes
|
||||
* instead of a hard-coded default; 0 (an older gateway that never set the field) means
|
||||
* "use the worker's built-in default". Sits above the public gRPC cap by an
|
||||
* envelope-overhead margin so an accepted gRPC payload always fits one worker frame.
|
||||
* </pre>
|
||||
*
|
||||
* <code>uint32 max_frame_bytes = 4;</code>
|
||||
* @return The maxFrameBytes.
|
||||
*/
|
||||
@java.lang.Override
|
||||
public int getMaxFrameBytes() {
|
||||
return maxFrameBytes_;
|
||||
}
|
||||
|
||||
private byte memoizedIsInitialized = -1;
|
||||
@java.lang.Override
|
||||
public final boolean isInitialized() {
|
||||
@@ -3941,6 +3974,9 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile {
|
||||
if (!com.google.protobuf.GeneratedMessage.isStringEmpty(gatewayVersion_)) {
|
||||
com.google.protobuf.GeneratedMessage.writeString(output, 3, gatewayVersion_);
|
||||
}
|
||||
if (maxFrameBytes_ != 0) {
|
||||
output.writeUInt32(4, maxFrameBytes_);
|
||||
}
|
||||
getUnknownFields().writeTo(output);
|
||||
}
|
||||
|
||||
@@ -3960,6 +3996,10 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile {
|
||||
if (!com.google.protobuf.GeneratedMessage.isStringEmpty(gatewayVersion_)) {
|
||||
size += com.google.protobuf.GeneratedMessage.computeStringSize(3, gatewayVersion_);
|
||||
}
|
||||
if (maxFrameBytes_ != 0) {
|
||||
size += com.google.protobuf.CodedOutputStream
|
||||
.computeUInt32Size(4, maxFrameBytes_);
|
||||
}
|
||||
size += getUnknownFields().getSerializedSize();
|
||||
memoizedSize = size;
|
||||
return size;
|
||||
@@ -3981,6 +4021,8 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile {
|
||||
.equals(other.getNonce())) return false;
|
||||
if (!getGatewayVersion()
|
||||
.equals(other.getGatewayVersion())) return false;
|
||||
if (getMaxFrameBytes()
|
||||
!= other.getMaxFrameBytes()) return false;
|
||||
if (!getUnknownFields().equals(other.getUnknownFields())) return false;
|
||||
return true;
|
||||
}
|
||||
@@ -3998,6 +4040,8 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile {
|
||||
hash = (53 * hash) + getNonce().hashCode();
|
||||
hash = (37 * hash) + GATEWAY_VERSION_FIELD_NUMBER;
|
||||
hash = (53 * hash) + getGatewayVersion().hashCode();
|
||||
hash = (37 * hash) + MAX_FRAME_BYTES_FIELD_NUMBER;
|
||||
hash = (53 * hash) + getMaxFrameBytes();
|
||||
hash = (29 * hash) + getUnknownFields().hashCode();
|
||||
memoizedHashCode = hash;
|
||||
return hash;
|
||||
@@ -4132,6 +4176,7 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile {
|
||||
supportedProtocolVersion_ = 0;
|
||||
nonce_ = "";
|
||||
gatewayVersion_ = "";
|
||||
maxFrameBytes_ = 0;
|
||||
return this;
|
||||
}
|
||||
|
||||
@@ -4174,6 +4219,9 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile {
|
||||
if (((from_bitField0_ & 0x00000004) != 0)) {
|
||||
result.gatewayVersion_ = gatewayVersion_;
|
||||
}
|
||||
if (((from_bitField0_ & 0x00000008) != 0)) {
|
||||
result.maxFrameBytes_ = maxFrameBytes_;
|
||||
}
|
||||
}
|
||||
|
||||
@java.lang.Override
|
||||
@@ -4201,6 +4249,9 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile {
|
||||
bitField0_ |= 0x00000004;
|
||||
onChanged();
|
||||
}
|
||||
if (other.getMaxFrameBytes() != 0) {
|
||||
setMaxFrameBytes(other.getMaxFrameBytes());
|
||||
}
|
||||
this.mergeUnknownFields(other.getUnknownFields());
|
||||
onChanged();
|
||||
return this;
|
||||
@@ -4242,6 +4293,11 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile {
|
||||
bitField0_ |= 0x00000004;
|
||||
break;
|
||||
} // case 26
|
||||
case 32: {
|
||||
maxFrameBytes_ = input.readUInt32();
|
||||
bitField0_ |= 0x00000008;
|
||||
break;
|
||||
} // case 32
|
||||
default: {
|
||||
if (!super.parseUnknownField(input, extensionRegistry, tag)) {
|
||||
done = true; // was an endgroup tag
|
||||
@@ -4435,6 +4491,62 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile {
|
||||
return this;
|
||||
}
|
||||
|
||||
private int maxFrameBytes_ ;
|
||||
/**
|
||||
* <pre>
|
||||
* Maximum worker-frame payload size, in bytes, negotiated by the gateway from its
|
||||
* configured pipe limit. The worker adopts this as its frame-protocol MaxMessageBytes
|
||||
* instead of a hard-coded default; 0 (an older gateway that never set the field) means
|
||||
* "use the worker's built-in default". Sits above the public gRPC cap by an
|
||||
* envelope-overhead margin so an accepted gRPC payload always fits one worker frame.
|
||||
* </pre>
|
||||
*
|
||||
* <code>uint32 max_frame_bytes = 4;</code>
|
||||
* @return The maxFrameBytes.
|
||||
*/
|
||||
@java.lang.Override
|
||||
public int getMaxFrameBytes() {
|
||||
return maxFrameBytes_;
|
||||
}
|
||||
/**
|
||||
* <pre>
|
||||
* Maximum worker-frame payload size, in bytes, negotiated by the gateway from its
|
||||
* configured pipe limit. The worker adopts this as its frame-protocol MaxMessageBytes
|
||||
* instead of a hard-coded default; 0 (an older gateway that never set the field) means
|
||||
* "use the worker's built-in default". Sits above the public gRPC cap by an
|
||||
* envelope-overhead margin so an accepted gRPC payload always fits one worker frame.
|
||||
* </pre>
|
||||
*
|
||||
* <code>uint32 max_frame_bytes = 4;</code>
|
||||
* @param value The maxFrameBytes to set.
|
||||
* @return This builder for chaining.
|
||||
*/
|
||||
public Builder setMaxFrameBytes(int value) {
|
||||
|
||||
maxFrameBytes_ = value;
|
||||
bitField0_ |= 0x00000008;
|
||||
onChanged();
|
||||
return this;
|
||||
}
|
||||
/**
|
||||
* <pre>
|
||||
* Maximum worker-frame payload size, in bytes, negotiated by the gateway from its
|
||||
* configured pipe limit. The worker adopts this as its frame-protocol MaxMessageBytes
|
||||
* instead of a hard-coded default; 0 (an older gateway that never set the field) means
|
||||
* "use the worker's built-in default". Sits above the public gRPC cap by an
|
||||
* envelope-overhead margin so an accepted gRPC payload always fits one worker frame.
|
||||
* </pre>
|
||||
*
|
||||
* <code>uint32 max_frame_bytes = 4;</code>
|
||||
* @return This builder for chaining.
|
||||
*/
|
||||
public Builder clearMaxFrameBytes() {
|
||||
bitField0_ = (bitField0_ & ~0x00000008);
|
||||
maxFrameBytes_ = 0;
|
||||
onChanged();
|
||||
return this;
|
||||
}
|
||||
|
||||
// @@protoc_insertion_point(builder_scope:mxaccess_worker.v1.GatewayHello)
|
||||
}
|
||||
|
||||
@@ -12552,64 +12664,65 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile {
|
||||
"rker_heartbeat\030\023 \001(\0132#.mxaccess_worker.v" +
|
||||
"1.WorkerHeartbeatH\000\0227\n\014worker_fault\030\024 \001(" +
|
||||
"\0132\037.mxaccess_worker.v1.WorkerFaultH\000B\006\n\004" +
|
||||
"body\"Z\n\014GatewayHello\022\"\n\032supported_protoc" +
|
||||
"body\"s\n\014GatewayHello\022\"\n\032supported_protoc" +
|
||||
"ol_version\030\001 \001(\r\022\r\n\005nonce\030\002 \001(\t\022\027\n\017gatew" +
|
||||
"ay_version\030\003 \001(\t\"i\n\013WorkerHello\022\030\n\020proto" +
|
||||
"col_version\030\001 \001(\r\022\r\n\005nonce\030\002 \001(\t\022\031\n\021work" +
|
||||
"er_process_id\030\003 \001(\005\022\026\n\016worker_version\030\004 " +
|
||||
"\001(\t\"\216\001\n\013WorkerReady\022\031\n\021worker_process_id" +
|
||||
"\030\001 \001(\005\022\027\n\017mxaccess_progid\030\002 \001(\t\022\026\n\016mxacc" +
|
||||
"ess_clsid\030\003 \001(\t\0223\n\017ready_timestamp\030\004 \001(\013" +
|
||||
"2\032.google.protobuf.Timestamp\"w\n\rWorkerCo" +
|
||||
"mmand\022/\n\007command\030\001 \001(\0132\036.mxaccess_gatewa" +
|
||||
"y.v1.MxCommand\0225\n\021enqueue_timestamp\030\002 \001(" +
|
||||
"\0132\032.google.protobuf.Timestamp\"\201\001\n\022Worker" +
|
||||
"CommandReply\0222\n\005reply\030\001 \001(\0132#.mxaccess_g" +
|
||||
"ateway.v1.MxCommandReply\0227\n\023completed_ti" +
|
||||
"mestamp\030\002 \001(\0132\032.google.protobuf.Timestam" +
|
||||
"p\"\036\n\014WorkerCancel\022\016\n\006reason\030\001 \001(\t\"Q\n\016Wor" +
|
||||
"kerShutdown\022/\n\014grace_period\030\001 \001(\0132\031.goog" +
|
||||
"le.protobuf.Duration\022\016\n\006reason\030\002 \001(\t\"H\n\021" +
|
||||
"WorkerShutdownAck\0223\n\006status\030\001 \001(\0132#.mxac" +
|
||||
"cess_gateway.v1.ProtocolStatus\":\n\013Worker" +
|
||||
"Event\022+\n\005event\030\001 \001(\0132\034.mxaccess_gateway." +
|
||||
"v1.MxEvent\"\245\002\n\017WorkerHeartbeat\022\031\n\021worker" +
|
||||
"_process_id\030\001 \001(\005\022.\n\005state\030\002 \001(\0162\037.mxacc" +
|
||||
"ess_worker.v1.WorkerState\022?\n\033last_sta_ac" +
|
||||
"tivity_timestamp\030\003 \001(\0132\032.google.protobuf" +
|
||||
".Timestamp\022\035\n\025pending_command_count\030\004 \001(" +
|
||||
"\r\022\"\n\032outbound_event_queue_depth\030\005 \001(\r\022\033\n" +
|
||||
"\023last_event_sequence\030\006 \001(\004\022&\n\036current_co" +
|
||||
"mmand_correlation_id\030\007 \001(\t\"\364\001\n\013WorkerFau" +
|
||||
"lt\0229\n\010category\030\001 \001(\0162\'.mxaccess_worker.v" +
|
||||
"1.WorkerFaultCategory\022\026\n\016command_method\030" +
|
||||
"\002 \001(\t\022\024\n\007hresult\030\003 \001(\005H\000\210\001\001\022\026\n\016exception" +
|
||||
"_type\030\004 \001(\t\022\032\n\022diagnostic_message\030\005 \001(\t\022" +
|
||||
"<\n\017protocol_status\030\006 \001(\0132#.mxaccess_gate" +
|
||||
"way.v1.ProtocolStatusB\n\n\010_hresult*\227\002\n\013Wo" +
|
||||
"rkerState\022\034\n\030WORKER_STATE_UNSPECIFIED\020\000\022" +
|
||||
"\031\n\025WORKER_STATE_STARTING\020\001\022\034\n\030WORKER_STA" +
|
||||
"TE_HANDSHAKING\020\002\022!\n\035WORKER_STATE_INITIAL" +
|
||||
"IZING_STA\020\003\022\026\n\022WORKER_STATE_READY\020\004\022\"\n\036W" +
|
||||
"ORKER_STATE_EXECUTING_COMMAND\020\005\022\036\n\032WORKE" +
|
||||
"R_STATE_SHUTTING_DOWN\020\006\022\030\n\024WORKER_STATE_" +
|
||||
"STOPPED\020\007\022\030\n\024WORKER_STATE_FAULTED\020\010*\307\004\n\023" +
|
||||
"WorkerFaultCategory\022%\n!WORKER_FAULT_CATE" +
|
||||
"GORY_UNSPECIFIED\020\000\022+\n\'WORKER_FAULT_CATEG" +
|
||||
"ORY_INVALID_ARGUMENTS\020\001\0227\n3WORKER_FAULT_" +
|
||||
"CATEGORY_GATEWAY_AUTHENTICATION_FAILED\020\002" +
|
||||
"\022+\n\'WORKER_FAULT_CATEGORY_PROTOCOL_MISMA" +
|
||||
"TCH\020\003\022,\n(WORKER_FAULT_CATEGORY_PROTOCOL_" +
|
||||
"VIOLATION\020\004\022+\n\'WORKER_FAULT_CATEGORY_PIP" +
|
||||
"E_DISCONNECTED\020\005\0222\n.WORKER_FAULT_CATEGOR" +
|
||||
"Y_MXACCESS_CREATION_FAILED\020\006\0221\n-WORKER_F" +
|
||||
"AULT_CATEGORY_MXACCESS_COMMAND_FAILED\020\007\022" +
|
||||
":\n6WORKER_FAULT_CATEGORY_MXACCESS_EVENT_" +
|
||||
"CONVERSION_FAILED\020\010\022\"\n\036WORKER_FAULT_CATE" +
|
||||
"GORY_STA_HUNG\020\t\022(\n$WORKER_FAULT_CATEGORY" +
|
||||
"_QUEUE_OVERFLOW\020\n\022*\n&WORKER_FAULT_CATEGO" +
|
||||
"RY_SHUTDOWN_TIMEOUT\020\013B&\252\002#ZB.MOM.WW.MxGa" +
|
||||
"teway.Contracts.Protob\006proto3"
|
||||
"ay_version\030\003 \001(\t\022\027\n\017max_frame_bytes\030\004 \001(" +
|
||||
"\r\"i\n\013WorkerHello\022\030\n\020protocol_version\030\001 \001" +
|
||||
"(\r\022\r\n\005nonce\030\002 \001(\t\022\031\n\021worker_process_id\030\003" +
|
||||
" \001(\005\022\026\n\016worker_version\030\004 \001(\t\"\216\001\n\013WorkerR" +
|
||||
"eady\022\031\n\021worker_process_id\030\001 \001(\005\022\027\n\017mxacc" +
|
||||
"ess_progid\030\002 \001(\t\022\026\n\016mxaccess_clsid\030\003 \001(\t" +
|
||||
"\0223\n\017ready_timestamp\030\004 \001(\0132\032.google.proto" +
|
||||
"buf.Timestamp\"w\n\rWorkerCommand\022/\n\007comman" +
|
||||
"d\030\001 \001(\0132\036.mxaccess_gateway.v1.MxCommand\022" +
|
||||
"5\n\021enqueue_timestamp\030\002 \001(\0132\032.google.prot" +
|
||||
"obuf.Timestamp\"\201\001\n\022WorkerCommandReply\0222\n" +
|
||||
"\005reply\030\001 \001(\0132#.mxaccess_gateway.v1.MxCom" +
|
||||
"mandReply\0227\n\023completed_timestamp\030\002 \001(\0132\032" +
|
||||
".google.protobuf.Timestamp\"\036\n\014WorkerCanc" +
|
||||
"el\022\016\n\006reason\030\001 \001(\t\"Q\n\016WorkerShutdown\022/\n\014" +
|
||||
"grace_period\030\001 \001(\0132\031.google.protobuf.Dur" +
|
||||
"ation\022\016\n\006reason\030\002 \001(\t\"H\n\021WorkerShutdownA" +
|
||||
"ck\0223\n\006status\030\001 \001(\0132#.mxaccess_gateway.v1" +
|
||||
".ProtocolStatus\":\n\013WorkerEvent\022+\n\005event\030" +
|
||||
"\001 \001(\0132\034.mxaccess_gateway.v1.MxEvent\"\245\002\n\017" +
|
||||
"WorkerHeartbeat\022\031\n\021worker_process_id\030\001 \001" +
|
||||
"(\005\022.\n\005state\030\002 \001(\0162\037.mxaccess_worker.v1.W" +
|
||||
"orkerState\022?\n\033last_sta_activity_timestam" +
|
||||
"p\030\003 \001(\0132\032.google.protobuf.Timestamp\022\035\n\025p" +
|
||||
"ending_command_count\030\004 \001(\r\022\"\n\032outbound_e" +
|
||||
"vent_queue_depth\030\005 \001(\r\022\033\n\023last_event_seq" +
|
||||
"uence\030\006 \001(\004\022&\n\036current_command_correlati" +
|
||||
"on_id\030\007 \001(\t\"\364\001\n\013WorkerFault\0229\n\010category\030" +
|
||||
"\001 \001(\0162\'.mxaccess_worker.v1.WorkerFaultCa" +
|
||||
"tegory\022\026\n\016command_method\030\002 \001(\t\022\024\n\007hresul" +
|
||||
"t\030\003 \001(\005H\000\210\001\001\022\026\n\016exception_type\030\004 \001(\t\022\032\n\022" +
|
||||
"diagnostic_message\030\005 \001(\t\022<\n\017protocol_sta" +
|
||||
"tus\030\006 \001(\0132#.mxaccess_gateway.v1.Protocol" +
|
||||
"StatusB\n\n\010_hresult*\227\002\n\013WorkerState\022\034\n\030WO" +
|
||||
"RKER_STATE_UNSPECIFIED\020\000\022\031\n\025WORKER_STATE" +
|
||||
"_STARTING\020\001\022\034\n\030WORKER_STATE_HANDSHAKING\020" +
|
||||
"\002\022!\n\035WORKER_STATE_INITIALIZING_STA\020\003\022\026\n\022" +
|
||||
"WORKER_STATE_READY\020\004\022\"\n\036WORKER_STATE_EXE" +
|
||||
"CUTING_COMMAND\020\005\022\036\n\032WORKER_STATE_SHUTTIN" +
|
||||
"G_DOWN\020\006\022\030\n\024WORKER_STATE_STOPPED\020\007\022\030\n\024WO" +
|
||||
"RKER_STATE_FAULTED\020\010*\307\004\n\023WorkerFaultCate" +
|
||||
"gory\022%\n!WORKER_FAULT_CATEGORY_UNSPECIFIE" +
|
||||
"D\020\000\022+\n\'WORKER_FAULT_CATEGORY_INVALID_ARG" +
|
||||
"UMENTS\020\001\0227\n3WORKER_FAULT_CATEGORY_GATEWA" +
|
||||
"Y_AUTHENTICATION_FAILED\020\002\022+\n\'WORKER_FAUL" +
|
||||
"T_CATEGORY_PROTOCOL_MISMATCH\020\003\022,\n(WORKER" +
|
||||
"_FAULT_CATEGORY_PROTOCOL_VIOLATION\020\004\022+\n\'" +
|
||||
"WORKER_FAULT_CATEGORY_PIPE_DISCONNECTED\020" +
|
||||
"\005\0222\n.WORKER_FAULT_CATEGORY_MXACCESS_CREA" +
|
||||
"TION_FAILED\020\006\0221\n-WORKER_FAULT_CATEGORY_M" +
|
||||
"XACCESS_COMMAND_FAILED\020\007\022:\n6WORKER_FAULT" +
|
||||
"_CATEGORY_MXACCESS_EVENT_CONVERSION_FAIL" +
|
||||
"ED\020\010\022\"\n\036WORKER_FAULT_CATEGORY_STA_HUNG\020\t" +
|
||||
"\022(\n$WORKER_FAULT_CATEGORY_QUEUE_OVERFLOW" +
|
||||
"\020\n\022*\n&WORKER_FAULT_CATEGORY_SHUTDOWN_TIM" +
|
||||
"EOUT\020\013B&\252\002#ZB.MOM.WW.MxGateway.Contracts" +
|
||||
".Protob\006proto3"
|
||||
};
|
||||
descriptor = com.google.protobuf.Descriptors.FileDescriptor
|
||||
.internalBuildGeneratedFileFrom(descriptorData,
|
||||
@@ -12629,7 +12742,7 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile {
|
||||
internal_static_mxaccess_worker_v1_GatewayHello_fieldAccessorTable = new
|
||||
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
|
||||
internal_static_mxaccess_worker_v1_GatewayHello_descriptor,
|
||||
new java.lang.String[] { "SupportedProtocolVersion", "Nonce", "GatewayVersion", });
|
||||
new java.lang.String[] { "SupportedProtocolVersion", "Nonce", "GatewayVersion", "MaxFrameBytes", });
|
||||
internal_static_mxaccess_worker_v1_WorkerHello_descriptor =
|
||||
getDescriptor().getMessageType(2);
|
||||
internal_static_mxaccess_worker_v1_WorkerHello_fieldAccessorTable = new
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Shared pytest fixtures for the client test suite.
|
||||
|
||||
pytest-asyncio 1.x no longer installs a current event loop for *synchronous*
|
||||
tests, and it clears the loop after each ``async`` test. Some synchronous tests
|
||||
build a ``grpc.aio.Channel`` (via ``create_channel``), which calls
|
||||
``asyncio.get_event_loop()`` and therefore raises
|
||||
``RuntimeError: There is no current event loop`` on Python 3.12 when a prior
|
||||
async test has left the policy with no current loop. Ensure every test starts
|
||||
with a usable current loop so channel construction in a sync test is
|
||||
order-independent.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _ensure_current_event_loop() -> Iterator[None]:
|
||||
try:
|
||||
asyncio.get_event_loop()
|
||||
except RuntimeError:
|
||||
asyncio.set_event_loop(asyncio.new_event_loop())
|
||||
yield
|
||||
@@ -42,6 +42,12 @@ message GatewayHello {
|
||||
uint32 supported_protocol_version = 1;
|
||||
string nonce = 2;
|
||||
string gateway_version = 3;
|
||||
// Maximum worker-frame payload size, in bytes, negotiated by the gateway from its
|
||||
// configured pipe limit. The worker adopts this as its frame-protocol MaxMessageBytes
|
||||
// instead of a hard-coded default; 0 (an older gateway that never set the field) means
|
||||
// "use the worker's built-in default". Sits above the public gRPC cap by an
|
||||
// envelope-overhead margin so an accepted gRPC payload always fits one worker frame.
|
||||
uint32 max_frame_bytes = 4;
|
||||
}
|
||||
|
||||
message WorkerHello {
|
||||
|
||||
+3
-2
@@ -112,8 +112,9 @@ dotnet build src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csp
|
||||
```
|
||||
|
||||
`scripts/check-codegen.ps1` enforces this in CI (it force-regenerates and fails on any
|
||||
`git diff` against the committed `Generated/`); the Windows CI job's net48 worker build is the
|
||||
definitive guard.
|
||||
`git diff` against the committed `Generated/`) — that regeneration diff in the `portable`
|
||||
job is the primary guard. The SSH-driven `windows-x86` job's net48 worker build is the
|
||||
secondary guard (a stale `Generated/` also breaks the x86 build with `CS0246`).
|
||||
|
||||
Client generation inputs are published through
|
||||
`clients/proto/proto-inputs.json` and the descriptor set under
|
||||
|
||||
@@ -245,7 +245,7 @@ dev/test GLAuth posture (`glauth.md`), not a production posture.
|
||||
| `MxGateway:Ldap:AllowInsecure` | `true` | Permits a plaintext bind. Must be `true` when `Transport` is `None`; set `false` (with `Ldaps`/`StartTls`) in production. |
|
||||
| `MxGateway:Ldap:SearchBase` | `dc=zb,dc=local` | Search base DN. |
|
||||
| `MxGateway:Ldap:ServiceAccountDn` | `cn=serviceaccount,dc=zb,dc=local` | Bind DN for the search account. |
|
||||
| `MxGateway:Ldap:ServiceAccountPassword` | `serviceaccount123` | Search-account password. **Dev-only committed value.** On the deployed hosts, do not ship this in `appsettings.json`; supply it out-of-band via the env-var override `MxGateway__Ldap__ServiceAccountPassword` (double-underscore form, as set on the NSSM-wrapped services). The committed `serviceaccount123` is a shared dev credential for the local GLAuth instance and should be rotated; production must use a secret it does not share with dev. |
|
||||
| `MxGateway:Ldap:ServiceAccountPassword` | `${secret:ldap/mxgateway/bind}` | Search-account password. **No longer a committed plaintext value:** `appsettings.json` ships the reference `${secret:ldap/mxgateway/bind}`, which the pre-host `${secret:}` expander resolves at startup from the encrypted secrets store (the code-side design default is now blank, so a missing/unresolved value fails closed rather than falling back to a leaked credential). Seed the value once with `secret set ldap/mxgateway/bind <value>` (the store's master key must be present via `ZB_SECRETS_MASTER_KEY`); startup aborts with `SecretNotFoundException` if the secret is absent. An operator may instead override it directly with the env var `MxGateway__Ldap__ServiceAccountPassword` (double-underscore form) — a plain literal there is used as-is and the secret lookup is skipped. |
|
||||
| `MxGateway:Ldap:UserNameAttribute` | `cn` | LDAP attribute holding the login user name. |
|
||||
| `MxGateway:Ldap:DisplayNameAttribute` | `cn` | LDAP attribute holding the display name. |
|
||||
| `MxGateway:Ldap:GroupAttribute` | `memberOf` | LDAP attribute enumerating group membership (mapped to dashboard roles via `MxGateway:Dashboard:GroupToRole`). |
|
||||
@@ -255,6 +255,88 @@ When LDAP is enabled, `Server`, `SearchBase`, `ServiceAccountDn`,
|
||||
must be in range. See `glauth.md` for the shared dev instance and the
|
||||
dev→production hardening posture.
|
||||
|
||||
## Secrets Master Key
|
||||
|
||||
`${secret:...}` tokens in configuration — currently just
|
||||
`MxGateway:Ldap:ServiceAccountPassword` (see above) — are resolved at startup
|
||||
from an encrypted local SQLite store, bound from the top-level `Secrets`
|
||||
section (a sibling of `MxGateway`, not nested under it):
|
||||
|
||||
| Option | Default | Description |
|
||||
|--------|---------|-------------|
|
||||
| `Secrets:SqlitePath` | `mxgateway-secrets.db` | Path to the encrypted secrets store, resolved relative to the app content root when not rooted. |
|
||||
| `Secrets:MasterKey:Source` | `Environment` | Key-encryption-key (KEK) provider. `Environment` reads a base64-encoded 32-byte key from an env var; `Dpapi` uses a machine-bound key file instead (see below). |
|
||||
| `Secrets:MasterKey:EnvVarName` | `ZB_SECRETS_MASTER_KEY` | Env var name the `Environment` provider reads the KEK from. |
|
||||
|
||||
The gateway always opens (and, if needed, creates and schema-migrates) this
|
||||
store at startup, even when zero `${secret:}` tokens are configured.
|
||||
|
||||
### Delivering the master key on the NSSM box
|
||||
|
||||
MxAccessGateway is a single Windows/NSSM box with no clustering, so the
|
||||
simplest posture applies: the `Environment` provider delivers
|
||||
`ZB_SECRETS_MASTER_KEY` via the NSSM service environment — the same mechanism
|
||||
already used for `MxGateway__Ldap__ServiceAccountPassword` overrides and the
|
||||
other `AppEnvironmentExtra` entries (see `wonder-app-vd03`/`windev` deploy
|
||||
notes). The key is **never committed** to `appsettings.json` or the repo.
|
||||
|
||||
```text
|
||||
nssm get MxAccessGw AppEnvironmentExtra
|
||||
nssm set MxAccessGw AppEnvironmentExtra ZB_SECRETS_MASTER_KEY=<base64-32-bytes>
|
||||
```
|
||||
|
||||
`AppEnvironmentExtra` is the same newline-separated `KEY=VALUE` block that
|
||||
already carries `ASPNETCORE_ENVIRONMENT`, `MxGateway__Worker__ExecutablePath`,
|
||||
and the API-key pepper (see the `wonder-app-vd03`/`windev` deploy notes) —
|
||||
`nssm set ... AppEnvironmentExtra` **replaces the entire block**, so always
|
||||
`get` first and re-include every existing entry alongside the new key, then
|
||||
restart the service for it to take effect.
|
||||
|
||||
### Seeding a secret
|
||||
|
||||
Once the master key is in place, seed (or update) a secret on the box with
|
||||
the `secret` CLI (`ZB.MOM.WW.Secrets.Cli`). The CLI is config-driven — it
|
||||
reads `Secrets:SqlitePath` and the master-key settings from its own
|
||||
`appsettings.json` plus environment variables, exactly like the gateway — so
|
||||
run it with the same `ZB_SECRETS_MASTER_KEY` set and, if it isn't colocated
|
||||
with a config that already points at the gateway's store, an explicit
|
||||
`Secrets__SqlitePath` override so both processes read/write the same file:
|
||||
|
||||
```text
|
||||
set ZB_SECRETS_MASTER_KEY=<base64-32-bytes>
|
||||
set Secrets__SqlitePath=C:\ProgramData\MxGateway\mxgateway-secrets.db
|
||||
secret set ldap/mxgateway/bind <the real LDAP bind password>
|
||||
```
|
||||
|
||||
A mismatched `Secrets:SqlitePath` between the CLI and the gateway silently
|
||||
seeds a different store, and the gateway then fails closed at startup as if
|
||||
the secret were never set.
|
||||
|
||||
### Fail-closed
|
||||
|
||||
If a referenced `${secret:...}` token cannot be resolved — the secret is
|
||||
missing from the store, or `ZB_SECRETS_MASTER_KEY` is absent or wrong — the
|
||||
gateway refuses to start (`SecretNotFoundException`, or a master-key error
|
||||
raised before that). There is no fallback to a blank or leaked credential.
|
||||
|
||||
### Alternative: DPAPI
|
||||
|
||||
`Secrets:MasterKey:Source = Dpapi` uses a machine-bound key file instead of an
|
||||
env var — stronger at-rest binding (the key never appears in the service
|
||||
environment or process listing) at the cost of tying the store to one
|
||||
machine, which complicates DR restore (the key file does not travel with a
|
||||
backup/restore of the `.db` file alone). Not used on the current NSSM
|
||||
deployments; documented here as the alternative posture if the env-var
|
||||
exposure becomes a concern.
|
||||
|
||||
### Rotation / DR
|
||||
|
||||
Losing the KEK makes the store permanently unrecoverable — there is no
|
||||
recovery path other than re-seeding every secret from scratch. Back the KEK
|
||||
up in the org secret vault, not just on the box. Full KEK rotation
|
||||
(re-wrapping every secret under a new key, `RewrapAll`) is a future item and
|
||||
is **not yet available**.
|
||||
|
||||
## Protocol Options
|
||||
|
||||
| Option | Default | Description |
|
||||
|
||||
+37
-8
@@ -236,10 +236,19 @@ LDAP groups resolve to the `Admin` role succeeds and emits the role claim;
|
||||
the password into `FailureMessage`; an unknown username fails authentication; and
|
||||
an unreachable LDAP server is absorbed into a failed result rather than throwing.
|
||||
|
||||
`appsettings.json` now ships the LDAP bind password as the unexpanded
|
||||
`${secret:ldap/mxgateway/bind}` token (resolved at gateway startup by the
|
||||
pre-host secrets expander, which this suite's bare `ConfigurationBuilder`
|
||||
does not run). Before running the live LDAP suite, set
|
||||
`MxGateway__Ldap__ServiceAccountPassword` to the real GLAuth service-account
|
||||
password (dev value `serviceaccount123` for the shared GLAuth) so the suite
|
||||
binds with the real password instead of the literal token.
|
||||
|
||||
Run the LDAP live tests explicitly:
|
||||
|
||||
```bash
|
||||
$env:MXGATEWAY_RUN_LIVE_LDAP_TESTS = "1"
|
||||
$env:MxGateway__Ldap__ServiceAccountPassword = "serviceaccount123"
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.IntegrationTests/ZB.MOM.WW.MxGateway.IntegrationTests.csproj --filter FullyQualifiedName~DashboardLdapLiveTests
|
||||
```
|
||||
|
||||
@@ -423,18 +432,38 @@ runtime because the x86 Worker cannot build on Linux:
|
||||
`MxaccessGateway.java` with spurious protobuf-runtime-version churn on every build, so
|
||||
when no `.proto` changed the job reverts that one file (`git checkout`) before asserting
|
||||
the generated tree is clean. The dev Mac has no JRE, so Java verification is CI-only.
|
||||
- **`windows`** (self-hosted windev runner) — the only host that builds the **x86 /
|
||||
net48 Worker and Worker.Tests**; these are Windows-only and out of scope for the Linux
|
||||
jobs. This job also proves the net48 build against the committed `Generated/` (the
|
||||
definitive guard for the "regenerate and commit `Generated/`" rule).
|
||||
- **`live-mxaccess`** (self-hosted, MXAccess-installed) — scheduled only
|
||||
(`MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`); never gates a push.
|
||||
- **`windows-x86`** (Linux runner, per push/PR) — builds the **x86 / net48 Worker and
|
||||
Worker.Tests**, which are Windows-only and out of scope for the Linux jobs. It runs on a
|
||||
Linux runner that always schedules and SSHes to windev (`10.100.0.48`), where
|
||||
`scripts/ci/run-windev-ci.sh` fast-forwards the isolated CI clone `C:\build\mxaccessgw-ci`
|
||||
to the SHA under test and runs `scripts/ci/windev-worker-ci.ps1 -Mode test` there; the
|
||||
remote exit code propagates back, so a Worker regression turns the job red. A native
|
||||
Windows runner is not used because act_runner host-mode on Windows is broken and a
|
||||
`runs-on` gate with no runner wedges the queue forever. **A red `windows-x86` can mean
|
||||
the tier is down, not the change**: an unreachable windev fails the job (never skips), so
|
||||
if it goes red without a related code change, check the tier — `ssh windev`, read
|
||||
`C:\build\mxaccessgw-ci\ci.log` — before assuming a real test failure.
|
||||
- **`nightly-windev`** (Linux runner, scheduled `0 6 * * *`) — re-runs the full x86 build +
|
||||
Worker.Tests on windev, then the live-MXAccess smoke (`MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`)
|
||||
and a full `slnx` build (which surfaces Windows-only analyzer diagnostics such as
|
||||
`xUnit1030`). Gated `if: github.event_name == 'schedule'`, so it never gates a push. On
|
||||
failure it opens a Gitea issue via the Actions token, since nobody watches the Actions page.
|
||||
|
||||
The freshness guard `scripts/check-codegen.ps1` fails the build when the committed
|
||||
client descriptor set or the C# `Generated/` no longer matches the current `.proto`
|
||||
sources — the codegen drift class this repo has hit repeatedly (stale client
|
||||
descriptors, net48 `CS0246` on unregenerated protos). See
|
||||
[Client Proto Generation](./ClientProtoGeneration.md) and [Contracts](./Contracts.md).
|
||||
descriptors, net48 `CS0246` on unregenerated protos). The **primary** guard for the
|
||||
"regenerate and commit `Generated/`" rule is that regeneration diff in the `portable` job;
|
||||
the `windows-x86` net48 compile is the **secondary** guard (a stale `Generated/` also breaks
|
||||
the x86 build with `CS0246`). See [Client Proto Generation](./ClientProtoGeneration.md) and
|
||||
[Contracts](./Contracts.md).
|
||||
|
||||
If the SSH-driven Windows tier is unavailable for infrastructure reasons (windev down, CI
|
||||
key/secret rotation in flight), fall back to the manual windev worktree procedure as a
|
||||
degraded mode: on windev, fast-forward an isolated `origin/main` worktree under `C:\build`
|
||||
(never the dirty Desktop checkout), then run the x86 Worker build and `Worker.Tests`
|
||||
(`-p:Platform=x86`) there by hand. Do this per merge for worker-touching changes until the
|
||||
`windows-x86` job is green again.
|
||||
|
||||
## Related Documentation
|
||||
|
||||
|
||||
@@ -0,0 +1,297 @@
|
||||
# Dashboard EventsHub per-session ACL (TST-15 / SEC-25 · session-resilience epic Phase 4)
|
||||
|
||||
Status: **Design** — approved-to-implement pending the schema-touch call in §9.
|
||||
Findings: TST-15 (`Medium`, P2), SEC-25 (`Low`, P2). Epic tasks: 16–19 of
|
||||
`docs/plans/2026-06-15-session-resilience.md`.
|
||||
Depends on: TST-02 (owner-scoped gRPC attach, shipped P0), SEC-25 near-term
|
||||
mitigation (value redaction in the dashboard mirror, shipped P2).
|
||||
|
||||
## 1. Problem
|
||||
|
||||
`EventsHub.SubscribeSession(sessionId)` joins the caller's SignalR connection to
|
||||
the `session:{id}` group with no per-session check. The hub-level
|
||||
`[Authorize(Policy = HubClientsPolicy)]` only requires *any* dashboard role, so
|
||||
**every authenticated Viewer (and, via SEC-02, anonymous localhost) can subscribe
|
||||
to any session's raw `MxEvent` feed** — the one production-source `TODO` left in
|
||||
the repo (`Dashboard/Hubs/EventsHub.cs:42`).
|
||||
|
||||
SEC-25's shipped mitigation already strips tag *values* from the mirrored events
|
||||
when `Dashboard:ShowTagValues` is false (the default), so no payload leaks through
|
||||
this seam today. What remains is **event-metadata visibility**: which sessions'
|
||||
event streams (worker sequence, family, server/item handles, alarm references) a
|
||||
given Viewer may observe. That is what this ACL gates.
|
||||
|
||||
### 1.1 The crux: two disjoint identity domains
|
||||
|
||||
This is *not* a copy of the gRPC owner check (TST-02), and the reason is the whole
|
||||
design problem:
|
||||
|
||||
| | gRPC `StreamEvents` (TST-02) | Dashboard EventsHub (TST-15) |
|
||||
|---|---|---|
|
||||
| Caller identity | API key (`mxgw_<id>_<secret>`) | LDAP user → `Administrator`/`Viewer` role |
|
||||
| Session identity | `GatewaySession.OwnerKeyId` (an API key id) | same session, but caller has **no** key id |
|
||||
| Check | `caller.KeyId == session.OwnerKeyId` — same domain, trivial | caller is an LDAP principal; there is **nothing on the session to compare it to** |
|
||||
|
||||
TST-02's binding is API-key-id ↔ API-key-id. A dashboard Viewer has no API key id,
|
||||
so "sessions they own" does not translate directly. The design must introduce a
|
||||
**bridge concept** that both a *session* and a *dashboard group* can be associated
|
||||
with. That bridge is the **session tag** (epic Task 17, "Session Tag + dashboard
|
||||
group-to-tag config").
|
||||
|
||||
## 2. Decision summary
|
||||
|
||||
1. **Admin sees all.** A connection carrying `DashboardRoles.Admin` bypasses the
|
||||
ACL entirely — it may subscribe to any session. (Admin already reaches every
|
||||
destructive surface; event-metadata visibility is strictly weaker.)
|
||||
2. **Viewer is strict.** A Viewer-only connection may subscribe to a session iff
|
||||
`session.Tags ∩ viewer.GrantedTags ≠ ∅`.
|
||||
3. **A session's tags come from its owning API key**, not from the client wire
|
||||
request. The API key is the tenant principal (TST-02 already treats it as the
|
||||
session's owner); deriving the tag from the key makes the dashboard boundary
|
||||
match the gRPC owner boundary. See §3.1 and the rejected alternative in §10.1.
|
||||
4. **Untagged sessions are Admin-only by default.** If the owning key carries no
|
||||
tag, no Viewer sees the session. A config knob
|
||||
(`Dashboard:UntaggedSessionVisibility`) can relax this to `AllViewers` for a
|
||||
genuinely single-tenant deployment, preserving today's behaviour for operators
|
||||
who want it. Default is the safe `AdminOnly`.
|
||||
5. **Enforcement is server-side and stateless**, carried in the existing
|
||||
short-lived hub bearer token (§4). No new server-side session/among-connection
|
||||
state.
|
||||
|
||||
## 3. The session tag
|
||||
|
||||
A session gains an immutable set of tags, assigned once at `OpenSession` and never
|
||||
changed for the session's life (so no re-check on an already-joined SignalR
|
||||
group is needed).
|
||||
|
||||
```csharp
|
||||
// GatewaySession — new read-only property, set from the owner key at construction.
|
||||
public IReadOnlySet<string> Tags { get; } // empty set == untagged
|
||||
```
|
||||
|
||||
### 3.1 Source: the owning API key
|
||||
|
||||
Sessions inherit the tag set of the API key that opened them. The tag rides in the
|
||||
**existing `ApiKeyConstraints` JSON blob** (`ApiKeyConstraintSerializer`), so there
|
||||
is **no SQLite schema migration** — a new column is not required:
|
||||
|
||||
```csharp
|
||||
// ApiKeyConstraints — add one field (serialized into the existing constraints column).
|
||||
public sealed record ApiKeyConstraints(
|
||||
...,
|
||||
bool ReadHistorizedOnly,
|
||||
IReadOnlyList<string> DashboardTags); // NEW — default empty
|
||||
```
|
||||
|
||||
At `OpenSession`, the resolved `ApiKeyIdentity.EffectiveConstraints.DashboardTags`
|
||||
is copied onto the new `GatewaySession.Tags`. The `apikey` admin CLI gains
|
||||
`--dashboard-tags team-a,team-b` on `create`/`update`.
|
||||
|
||||
Semantic note: `ApiKeyConstraints` today scopes *data-access* authorization
|
||||
(read/write subtrees, globs, classification). A dashboard *visibility* tag is a
|
||||
distinct concern, but piggy-backing on the same serialized blob is the pragmatic
|
||||
choice — it avoids a migration and keeps all per-key policy in one place. The
|
||||
field is documented as visibility-only; it does **not** gate any data-access path.
|
||||
|
||||
### 3.2 Dashboard group → tag grant
|
||||
|
||||
A new config map, mirroring the shape of `Dashboard:GroupToRole`:
|
||||
|
||||
```jsonc
|
||||
"MxGateway": {
|
||||
"Dashboard": {
|
||||
"GroupToRole": { "GwAdmin": "Administrator", "GwViewer": "Viewer" },
|
||||
"GroupToTag": { "GwViewer": ["team-a"], "TeamBViewers": ["team-b"] },
|
||||
"UntaggedSessionVisibility": "AdminOnly" // or "AllViewers"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`GroupToTag` maps an LDAP group (short RDN, matching the `GroupToRole` convention
|
||||
in `DashboardGroupRoleMapping`) to the tags that group is granted. A Viewer's
|
||||
`GrantedTags` is the union over their LDAP groups. Admin's grant is irrelevant
|
||||
(they bypass). Bound on `DashboardOptions`:
|
||||
|
||||
```csharp
|
||||
public Dictionary<string, string[]> GroupToTag { get; init; }
|
||||
= new(StringComparer.OrdinalIgnoreCase);
|
||||
public UntaggedSessionVisibility UntaggedSessionVisibility { get; init; }
|
||||
= UntaggedSessionVisibility.AdminOnly;
|
||||
```
|
||||
|
||||
Validated in `GatewayOptionsValidator` exactly like `GroupToRole` (keys non-empty;
|
||||
`UntaggedSessionVisibility` a known enum). No validation coupling to `GroupToRole`
|
||||
— a group may appear in one, the other, or both.
|
||||
|
||||
## 4. Enforcement path
|
||||
|
||||
The tag grant must reach `SubscribeSession`. A hub connection authenticates by
|
||||
either the dashboard cookie or the 5-minute bearer minted by `HubTokenService`.
|
||||
`HubTokenPayload` today carries `(Name, NameIdentifier, Roles)` — extend it with
|
||||
the resolved granted-tag set:
|
||||
|
||||
```csharp
|
||||
private sealed record HubTokenPayload(
|
||||
string? Name, string? NameIdentifier, string[]? Roles, string[]? Tags);
|
||||
```
|
||||
|
||||
- **Mint (`HubTokenService.Issue`)** — resolve the principal's granted tags from
|
||||
their LDAP-group claims (`mxgateway:ldap_group`) via a `GroupToTag` mapper
|
||||
(sibling of `DashboardGroupRoleMapping`), and stamp them into the payload as a
|
||||
`zb:dashboardtag` claim set. Admin needs no tags stamped.
|
||||
- **Cookie path** — the same tag claims are added in
|
||||
`DashboardAuthenticator.CreatePrincipal` so a cookie-authenticated circuit
|
||||
carries them without a token round-trip.
|
||||
- **Validate (`HubTokenService.Validate`)** — rehydrate the tag claims onto the
|
||||
reconstructed principal, alongside the existing role claims.
|
||||
- The 5-minute lifetime already bounds staleness of a changed grant
|
||||
(documented at `HubTokenService.cs:30-37`) — this now also bounds a changed
|
||||
tag grant, which is the natural place the SEC-25 "tokens gain session binding"
|
||||
note anticipated.
|
||||
|
||||
`SubscribeSession` becomes an authorizing, `SessionManager`-aware method:
|
||||
|
||||
```csharp
|
||||
public async Task SubscribeSession(string sessionId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(sessionId)) return;
|
||||
|
||||
if (!_sessionAcl.CanViewSession(Context.User, sessionId))
|
||||
{
|
||||
// Deny quietly: do NOT join the group. Optionally surface a HubException
|
||||
// so the client can distinguish "denied" from "no events yet".
|
||||
throw new HubException("Not authorized for this session.");
|
||||
}
|
||||
|
||||
await Groups.AddToGroupAsync(Context.ConnectionId, GroupName(sessionId));
|
||||
}
|
||||
```
|
||||
|
||||
`IDashboardSessionAcl.CanViewSession(ClaimsPrincipal, sessionId)`:
|
||||
1. Admin role → `true`.
|
||||
2. Session not found in `SessionManager` → `false` (no group join for a phantom id).
|
||||
3. Session untagged → `UntaggedSessionVisibility == AllViewers`.
|
||||
4. Otherwise → `session.Tags ∩ principal.GrantedTags ≠ ∅`.
|
||||
|
||||
Because the broadcaster only ever publishes to `session:{id}` and only joined
|
||||
connections receive, gating at join is sufficient — there is no second seam to
|
||||
guard. Tags are immutable per session, so a mid-life re-check is unnecessary.
|
||||
|
||||
### 4.1 Interaction with SEC-02 (anonymous localhost)
|
||||
|
||||
Anonymous-localhost satisfies `HubClientsPolicy` but carries no role and no tags.
|
||||
Under this ACL it is treated as a **Viewer with an empty tag grant** → sees only
|
||||
untagged sessions, and only when `UntaggedSessionVisibility == AllViewers`. That
|
||||
tightens SEC-02's current "loopback sees every session" posture without a separate
|
||||
change. `DisableLogin` auto-login (both roles) keeps admin-sees-all — unchanged.
|
||||
|
||||
## 5. End-to-end data flow
|
||||
|
||||
```
|
||||
apikey create --dashboard-tags team-a → key.Constraints.DashboardTags = [team-a]
|
||||
client OpenSession (Bearer mxgw_<id>_..) → session.OwnerKeyId = <id>,
|
||||
session.Tags = [team-a]
|
||||
LDAP login gw-viewer ∈ group GwViewer → GroupToTag[GwViewer] = [team-a]
|
||||
principal grantedTags = {team-a}
|
||||
hub token / cookie → payload.Tags = [team-a]
|
||||
SubscribeSession("s-123") (session.Tags={team-a})→ {team-a} ∩ {team-a} ≠ ∅ → JOIN
|
||||
SubscribeSession("s-999") (session.Tags={team-b})→ {team-a} ∩ {team-b} = ∅ → DENY
|
||||
Admin → bypass → JOIN either
|
||||
```
|
||||
|
||||
## 6. Task breakdown (maps to epic Tasks 16–19)
|
||||
|
||||
- **Task 16 — gRPC all-sessions admin scope (already largely shipped).** The gRPC
|
||||
owner gate is TST-02. This task is the *dashboard* admin bypass in
|
||||
`IDashboardSessionAcl` (§4). No gRPC change required; the "all-sessions admin
|
||||
scope" for gRPC is out of scope for TST-15 and stays with the broader epic.
|
||||
- **Task 17 — session tag + group-to-tag config.** `ApiKeyConstraints.DashboardTags`
|
||||
+ serializer + `apikey` CLI flag; `GatewaySession.Tags` set at `OpenSession`;
|
||||
`DashboardOptions.GroupToTag` + `UntaggedSessionVisibility` + validator.
|
||||
- **Task 18 — EventsHub ACL + hub-token tag claim.** `HubTokenPayload.Tags`,
|
||||
mint/validate, cookie-principal tag claims, `IDashboardSessionAcl` +
|
||||
`SubscribeSession` gate. Remove the `TODO(per-session-acl)`.
|
||||
- **Task 19 — ACL tests, incl. live LDAP users.** §7.
|
||||
|
||||
Sequence: 17 → 18 → 19; 16's dashboard slice folds into 18.
|
||||
|
||||
## 7. Test plan
|
||||
|
||||
Unit / fake-worker (default suite, no LDAP, no COM):
|
||||
- `DashboardSessionAclTests` — the four `CanViewSession` branches: admin bypass;
|
||||
session-not-found deny; untagged under both `UntaggedSessionVisibility` values;
|
||||
tag-intersection match and non-match.
|
||||
- `HubTokenServiceTests` — extend: tags round-trip through `Issue`/`Validate`;
|
||||
a token minted before a grant change still carries the old tags until expiry.
|
||||
- `EventsHubTests` — `SubscribeSession` joins on allow, throws `HubException`
|
||||
and does **not** join on deny (assert via a fake `IGroupManager`).
|
||||
- `GroupToTag` mapper tests — union across multiple groups; unknown group → no tags.
|
||||
- Config: `GatewayOptionsValidator` accepts/ rejects `GroupToTag` /
|
||||
`UntaggedSessionVisibility` shapes.
|
||||
|
||||
Live LDAP (`MXGATEWAY_RUN_LIVE_LDAP_TESTS=1`, `LiveLdapFactAttribute`) — extend
|
||||
`DashboardLdapLiveTests`: `gw-viewer`/`password` (Viewer) is granted `team-a` via
|
||||
`GroupToTag`, subscribes to a `team-a` session (allow) and a `team-b` session
|
||||
(deny); `multi-role`/`password` (Admin) subscribes to both (allow). Uses the
|
||||
shared GLAuth server per `glauth.md`.
|
||||
|
||||
Verification: `dotnet test ... --filter FullyQualifiedName~DashboardSessionAcl`,
|
||||
`~EventsHub`, `~HubTokenService`; `dotnet build src/ZB.MOM.WW.MxGateway.Server`.
|
||||
|
||||
## 8. Docs to update (same change as source)
|
||||
|
||||
- `docs/Sessions.md` — dashboard event visibility / session-tag model.
|
||||
- `gateway.md` dashboard section — the ACL and the tag source.
|
||||
- `docs/GatewayConfiguration.md` — `Dashboard:GroupToTag`,
|
||||
`Dashboard:UntaggedSessionVisibility`; `apikey --dashboard-tags`.
|
||||
- `docs/Authorization.md` — note `DashboardTags` is visibility-only, not a
|
||||
data-access constraint.
|
||||
- `CLAUDE.md` dashboard-auth paragraph — Viewer is tag-scoped; Admin sees all;
|
||||
anonymous-localhost is an empty-grant Viewer.
|
||||
- `glauth.md` — the test-user tag grants used by the live tests.
|
||||
|
||||
## 9. Open call for the implementer
|
||||
|
||||
**Does the tag ride in the existing `ApiKeyConstraints` JSON blob (no migration,
|
||||
recommended) or a new dedicated key column/table?** The blob avoids a SQLite
|
||||
migration and keeps per-key policy in one place; a dedicated column is cleaner
|
||||
semantically (visibility ≠ data-access constraint) but costs a migration + audit
|
||||
touch. Recommendation: **blob**, documented as visibility-only. Settle before
|
||||
Task 17.
|
||||
|
||||
## 10. Rejected alternatives
|
||||
|
||||
### 10.1 Client-supplied session tag (proto field on `OpenSession`)
|
||||
Lighter — one proto field, one session field, no API-key-store touch. Rejected as
|
||||
*primary* because the tag would then be attacker-controlled: a low-trust gRPC
|
||||
client could label its session with any tenant's tag, choosing which dashboard
|
||||
groups observe its event feed. Deriving the tag from the owning API key keeps the
|
||||
boundary aligned with TST-02's trust model (the key is the tenant). (If a future
|
||||
requirement wants client-declared sub-tags *within* an owner's grant, they can be
|
||||
intersected with the key's tags — additive, not a replacement.)
|
||||
|
||||
### 10.2 Map dashboard groups directly to owner key ids (reuse `OwnerKeyId`, no tag)
|
||||
Zero API-key-store change — `Dashboard:GroupToOwnerKeys` maps a group to a set of
|
||||
key ids. Rejected: key ids are opaque and rotate; operators would maintain a map
|
||||
of GUID-like ids, and every key rotation breaks the dashboard grant. A named tag
|
||||
is the right operator-facing abstraction and survives key rotation.
|
||||
|
||||
### 10.3 Full jti-denylist token revocation
|
||||
Out of scope. The 5-minute token lifetime bounds a changed grant; the tokens are
|
||||
data-protection-encrypted and single-purpose. Matches the SEC-25 recommendation
|
||||
to defer heavy revocation.
|
||||
|
||||
## 11. Security considerations
|
||||
|
||||
- **No new value-leak surface.** SEC-25 already redacts values in the mirror; this
|
||||
change only narrows *which sessions'* metadata a Viewer sees. Strictly a
|
||||
tightening.
|
||||
- **Fail closed.** Session-not-found, empty grant, and untagged-under-`AdminOnly`
|
||||
all deny. A misconfigured/empty `GroupToTag` yields Viewers who see nothing
|
||||
(except untagged-under-`AllViewers`) — safe, not open.
|
||||
- **Default preserves single-tenant ergonomics** only if the operator opts into
|
||||
`UntaggedSessionVisibility=AllViewers`; the shipped default (`AdminOnly`) is the
|
||||
strict one, so an upgrade tightens rather than loosens.
|
||||
- **Staleness bound.** A revoked tag grant takes effect within one token lifetime
|
||||
(≤5 min) for token-auth connections and immediately for a fresh cookie login.
|
||||
```
|
||||
@@ -22,6 +22,8 @@
|
||||
<package pattern="ZB.MOM.WW.Audit" />
|
||||
<package pattern="ZB.MOM.WW.Theme" />
|
||||
<package pattern="ZB.MOM.WW.GalaxyRepository" />
|
||||
<package pattern="ZB.MOM.WW.Secrets" />
|
||||
<package pattern="ZB.MOM.WW.Secrets.*" />
|
||||
</packageSource>
|
||||
</packageSourceMapping>
|
||||
</configuration>
|
||||
|
||||
@@ -14,7 +14,8 @@
|
||||
# protos. A drift means a .proto was edited without refreshing the vendored copies, which would
|
||||
# publish a stale wire contract to crate consumers while the in-repo build stays correct.
|
||||
#
|
||||
# The x86 Worker + Worker.Tests are Windows-only and are guarded by the Windows CI job, not here.
|
||||
# The x86 Worker + Worker.Tests are Windows-only and are guarded by the SSH-driven `windows-x86`
|
||||
# CI job (see docs/GatewayTesting.md, Continuous Integration), not here.
|
||||
|
||||
[CmdletBinding()]
|
||||
param()
|
||||
@@ -52,7 +53,9 @@ try {
|
||||
$failures.Add('Contracts project failed to build during codegen check.')
|
||||
}
|
||||
|
||||
$diff = (& git -C $repoRoot status --porcelain -- 'src/ZB.MOM.WW.MxGateway.Contracts/Generated').Trim()
|
||||
# Pipe through Out-String so a clean tree (git emits nothing -> $null) does not throw
|
||||
# "cannot call a method on a null-valued expression"; Out-String yields '' for no output.
|
||||
$diff = (& git -C $repoRoot status --porcelain -- 'src/ZB.MOM.WW.MxGateway.Contracts/Generated' | Out-String).Trim()
|
||||
if (-not [string]::IsNullOrEmpty($diff)) {
|
||||
Write-Host $diff
|
||||
$failures.Add('Contracts/Generated differs from a fresh regeneration. Run `dotnet build` on the contracts project and commit Generated/ (required for the net48 worker build).')
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
# Windows/x86 CI tier (TST-25)
|
||||
|
||||
The x86 / net48 Worker and its tests cannot build on a Linux runner, and a native Windows
|
||||
`act_runner` is broken (host-mode writes each step's script to a path it can't resolve; a
|
||||
`runs-on` gate with no runner wedges the queue forever). Instead, a Linux CI job SSHes to
|
||||
windev (`10.100.0.48`) and runs the Worker build/tests there, propagating the remote exit
|
||||
code back so a regression turns the job red.
|
||||
|
||||
## Pieces
|
||||
|
||||
| File | Runs on | Role |
|
||||
|---|---|---|
|
||||
| `run-windev-ci.sh <build\|test\|live>` | Linux CI job | Writes the SSH key/known-hosts, base64-encodes a bootstrap, SSHes to windev, returns the remote exit code. |
|
||||
| `windev-worker-ci.ps1 -Sha -Mode` | windev | Locks the CI clone, checks out the SHA, runs the x86 build / `Worker.Tests` / live smoke. |
|
||||
| `windev.known_hosts` | committed | Pinned windev host keys so `StrictHostKeyChecking=yes` has no TOFU prompt. Host keys are public; committing is intentional. |
|
||||
|
||||
Modes are cumulative: `build` = x86 Worker build; `test` = build + `Worker.Tests`;
|
||||
`live` = test + full-`slnx` build + live-MXAccess smoke (`MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`).
|
||||
|
||||
CI jobs (`.gitea/workflows/ci.yml`): `windows-x86` runs `test` per push/PR; `nightly-windev`
|
||||
runs `live` on the `0 6 * * *` schedule and opens a Gitea issue on failure.
|
||||
|
||||
## One-time bring-up (operator — required before this lands on `main`)
|
||||
|
||||
Until these are done, `windows-x86` is **red on every push** (no key → SSH fails). Do them
|
||||
first, then merge.
|
||||
|
||||
1. **CI clone** — `C:\build\mxaccessgw-ci`, a fresh clone of the Gitea origin (done during
|
||||
TST-25 verification; recreate with `git clone https://gitea.dohertylan.com/dohertj2/mxaccessgw C:\build\mxaccessgw-ci` if missing). Never the dirty Desktop checkout.
|
||||
2. **Dedicated CI key** — generate a NEW ed25519 keypair for CI (do not reuse the operator's
|
||||
personal windev key): `ssh-keygen -t ed25519 -f ci_windev -N ''`. Append `ci_windev.pub`
|
||||
to the windev CI account's `authorized_keys` (optionally with a forced `command=` limiting
|
||||
it to this bootstrap).
|
||||
3. **Gitea repo secret `WINDEV_SSH_KEY`** — store the private key **base64-encoded on a single
|
||||
line**: `base64 -w0 < ci_windev` (macOS: `base64 < ci_windev | tr -d '\n'`). This is required,
|
||||
not cosmetic — Gitea's secret masker is line-oriented, so a raw multiline PEM is **not**
|
||||
redacted in the step `env:` echo and leaks in cleartext; the single-line base64 masks to `***`.
|
||||
`run-windev-ci.sh` auto-decodes it. Host keys are public and come from the committed
|
||||
`windev.known_hosts` pin, so no `WINDEV_SSH_KNOWN_HOSTS` secret is wired into `ci.yml`.
|
||||
`run-windev-ci.sh` connects as user `ci` by default; set the `WINDEV_SSH_USER` **variable**
|
||||
(not a secret — it is public) to the actual account, and ensure that account exists on windev.
|
||||
4. **Network precheck** — confirm the Gitea runner container (on the `traefik` docker network,
|
||||
host `10.100.0.35`) can reach `10.100.0.48:22`: a one-off `nc -z -w5 10.100.0.48 22` step.
|
||||
The network was created for gitea name resolution, not LAN egress, so verify.
|
||||
5. **Actions token** — the `nightly-windev` on-failure step creates a Gitea issue with the
|
||||
built-in token; confirm Actions tokens have issue-write on this repo.
|
||||
|
||||
## Verifying it end-to-end (design's acceptance checks)
|
||||
|
||||
- Push a branch → `portable`, `java`, `windows-x86` all green; windev's clone sits at the pushed SHA.
|
||||
- Deliberate red: add a failing `[Fact]` to `Worker.Tests`, push, confirm `windows-x86` goes
|
||||
red; revert. (Proves exit-code propagation — the whole point.)
|
||||
- Unreachable-host red: point at a bogus port / stop sshd, confirm the job fails fast, not hangs.
|
||||
- Concurrency: push two branches back-to-back, confirm the second remote run waits on the lock.
|
||||
- Nightly: trigger the schedule path, confirm `live` runs and a forced failure opens an issue.
|
||||
- Confirm no key material appears in job logs.
|
||||
|
||||
## Degraded mode
|
||||
|
||||
If the tier is down for infra reasons, fall back to the manual windev worktree procedure
|
||||
(isolated `C:\build` worktree, `dotnet build`/`test` `-p:Platform=x86` by hand) per merge for
|
||||
worker-touching changes until `windows-x86` is green again. See `docs/GatewayTesting.md`.
|
||||
Executable
+131
@@ -0,0 +1,131 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# TST-25: Linux side of the Windows/x86 Worker CI tier.
|
||||
#
|
||||
# Runs in the `windows-x86` / nightly Gitea jobs on a Linux runner (which always schedules,
|
||||
# unlike a broken Windows act_runner). SSHes to windev (10.100.0.48), where it fetches +
|
||||
# checks out the SHA under test in the isolated CI clone C:\build\mxaccessgw-ci and runs
|
||||
# scripts/ci/windev-worker-ci.ps1 there. The remote exit code propagates through
|
||||
# ssh -> this script -> the CI step, so a Worker regression turns the job red.
|
||||
#
|
||||
# Usage: run-windev-ci.sh <build|test|live>
|
||||
#
|
||||
# Environment:
|
||||
# CI_SHA commit to test (default: `git rev-parse HEAD` in this checkout)
|
||||
# WINDEV_SSH_KEY private key (Gitea secret). Store it BASE64-ENCODED (one line):
|
||||
# Gitea's secret masker is line-oriented, so a raw multiline PEM in
|
||||
# the step `env:` echo is NOT redacted and leaks in cleartext, whereas
|
||||
# the single-line base64 is masked to *** (TST-25 acceptance: no key
|
||||
# material in logs). A raw PEM is still accepted for local hand-testing
|
||||
# (auto-detected). If empty, falls back to the runner's default identity.
|
||||
# WINDEV_SSH_KNOWN_HOSTS pinned host keys (Gitea secret). If empty, uses the committed
|
||||
# scripts/ci/windev.known_hosts.
|
||||
# WINDEV_SSH_USER ssh user on windev (default: ci)
|
||||
# WINDEV_SSH_HOST windev host/IP (default: 10.100.0.48)
|
||||
#
|
||||
# The private key is never echoed (written 0600 to a temp file, removed on exit). git on
|
||||
# windev writes progress to stderr, so the remote bootstrap does NOT use -ErrorActionPreference
|
||||
# Stop around git — it checks $LASTEXITCODE instead (known windev gotcha).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
MODE="${1:-}"
|
||||
case "$MODE" in
|
||||
build|test|live) ;;
|
||||
*) echo "usage: $0 <build|test|live>" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
CI_SHA="${CI_SHA:-$(git -C "$SCRIPT_DIR" rev-parse HEAD)}"
|
||||
WINDEV_SSH_USER="${WINDEV_SSH_USER:-ci}"
|
||||
WINDEV_SSH_HOST="${WINDEV_SSH_HOST:-10.100.0.48}"
|
||||
REMOTE_CLONE='C:\build\mxaccessgw-ci'
|
||||
REMOTE_SCRIPT='scripts\ci\windev-worker-ci.ps1'
|
||||
|
||||
# Validate the SHA shape before it reaches a remote shell command.
|
||||
if ! printf '%s' "$CI_SHA" | grep -Eq '^[0-9a-fA-F]{7,40}$'; then
|
||||
echo "run-windev-ci: refusing to run — CI_SHA '$CI_SHA' is not a hex commit id" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
WORK="$(mktemp -d)"
|
||||
cleanup() { rm -rf "$WORK"; }
|
||||
trap cleanup EXIT
|
||||
|
||||
SSH_OPTS=(-o BatchMode=yes -o StrictHostKeyChecking=yes -o ConnectTimeout=15)
|
||||
|
||||
# known_hosts: secret if provided, else the committed pins.
|
||||
KNOWN_HOSTS="$WORK/known_hosts"
|
||||
if [ -n "${WINDEV_SSH_KNOWN_HOSTS:-}" ]; then
|
||||
printf '%s\n' "$WINDEV_SSH_KNOWN_HOSTS" > "$KNOWN_HOSTS"
|
||||
else
|
||||
cp "$SCRIPT_DIR/windev.known_hosts" "$KNOWN_HOSTS"
|
||||
fi
|
||||
SSH_OPTS+=(-o "UserKnownHostsFile=$KNOWN_HOSTS")
|
||||
|
||||
# Private key: secret if provided, else fall back to the default identity/agent.
|
||||
# CI stores it base64-encoded (single line) so Gitea's line-oriented secret masker redacts it
|
||||
# in the step env echo — a raw multiline PEM leaks in cleartext. Decode when it is valid base64
|
||||
# wrapping a PEM; otherwise treat the value as a raw PEM (local hand-testing convenience).
|
||||
if [ -n "${WINDEV_SSH_KEY:-}" ]; then
|
||||
KEY="$WORK/id_ci"
|
||||
if printf '%s' "$WINDEV_SSH_KEY" | base64 -d 2>/dev/null | grep -q 'PRIVATE KEY'; then
|
||||
( umask 077; printf '%s' "$WINDEV_SSH_KEY" | base64 -d > "$KEY" )
|
||||
else
|
||||
( umask 077; printf '%s\n' "$WINDEV_SSH_KEY" > "$KEY" )
|
||||
fi
|
||||
SSH_OPTS+=(-o IdentitiesOnly=yes -i "$KEY")
|
||||
fi
|
||||
|
||||
echo "run-windev-ci: mode=$MODE sha=$CI_SHA target=${WINDEV_SSH_USER}@${WINDEV_SSH_HOST}"
|
||||
|
||||
# Remote PowerShell bootstrap: fetch + checkout enough to load the versioned CI script, then
|
||||
# hand off to it. Not Stop around git (stderr progress).
|
||||
#
|
||||
# The bootstrap fetch/checkout mutates the shared clone's git index, so it MUST hold the
|
||||
# worktree lock — otherwise two concurrent runs collide on .git/index.lock before either
|
||||
# reaches windev-worker-ci.ps1's lock (the exact race the lock exists to prevent). We take the
|
||||
# same mkdir lock here (atomic; retry to a 20-min deadline) and export MXGW_CI_LOCK_HELD so the
|
||||
# handed-off script re-uses this lock instead of re-acquiring/releasing it. A standalone/manual
|
||||
# run of windev-worker-ci.ps1 (degraded mode) has no such env and self-locks as before.
|
||||
read -r -d '' BOOTSTRAP <<PS || true
|
||||
\$ErrorActionPreference = 'Continue'
|
||||
\$lockDir = '$REMOTE_CLONE.lock'
|
||||
\$deadline = (Get-Date).AddMinutes(20)
|
||||
\$haveLock = \$false
|
||||
while (-not \$haveLock) {
|
||||
try { New-Item -Path \$lockDir -ItemType Directory -ErrorAction Stop | Out-Null; \$haveLock = \$true }
|
||||
catch {
|
||||
if ((Get-Date) -ge \$deadline) { Write-Error "timed out waiting for worktree lock \$lockDir"; exit 75 }
|
||||
Start-Sleep -Seconds 10
|
||||
}
|
||||
}
|
||||
\$env:MXGW_CI_LOCK_HELD = '1'
|
||||
\$rc = 1
|
||||
try {
|
||||
Set-Location '$REMOTE_CLONE'
|
||||
git fetch --prune origin
|
||||
git checkout --force --detach $CI_SHA
|
||||
if (\$LASTEXITCODE -ne 0) { Write-Error "bootstrap checkout failed"; \$rc = \$LASTEXITCODE }
|
||||
else {
|
||||
powershell -NoProfile -NonInteractive -ExecutionPolicy Bypass -File '$REMOTE_SCRIPT' -Sha '$CI_SHA' -Mode '$MODE'
|
||||
\$rc = \$LASTEXITCODE
|
||||
}
|
||||
}
|
||||
finally {
|
||||
Remove-Item -Path \$lockDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
exit \$rc
|
||||
PS
|
||||
|
||||
# PowerShell -EncodedCommand wants UTF-16LE base64 on one line (avoids all ssh/quoting hazards).
|
||||
ENCODED="$(printf '%s' "$BOOTSTRAP" | iconv -t UTF-16LE | base64 | tr -d '\n')"
|
||||
|
||||
set +e
|
||||
ssh "${SSH_OPTS[@]}" "${WINDEV_SSH_USER}@${WINDEV_SSH_HOST}" \
|
||||
"powershell -NoProfile -NonInteractive -EncodedCommand $ENCODED"
|
||||
RC=$?
|
||||
set -e
|
||||
|
||||
echo "run-windev-ci: remote exit code $RC"
|
||||
exit $RC
|
||||
@@ -0,0 +1,156 @@
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Windows/x86 Worker CI stage — runs ON windev (10.100.0.48), invoked over SSH by
|
||||
scripts/ci/run-windev-ci.sh at the exact SHA under test. Restores the x86/net48
|
||||
Worker test tier that the removed `windows` CI job used to cover (TST-25).
|
||||
|
||||
.DESCRIPTION
|
||||
The Linux `windows-x86` / nightly CI jobs cannot build the x86/net48 Worker, so they
|
||||
ssh here and run this script inside the isolated CI clone C:\build\mxaccessgw-ci.
|
||||
|
||||
The script is deliberately self-contained and versioned with the code under test: the
|
||||
SSH bootstrap only fetches + checks out the SHA (enough to load *this* file), then this
|
||||
script re-fetches and re-checks-out $Sha INSIDE the worktree lock so the checkout+build
|
||||
are fully serialized against a concurrent run and can never stomp each other's tree.
|
||||
|
||||
Modes (cumulative):
|
||||
build x86 Worker build only (net48/CS0246/x86 compile guard — always fast).
|
||||
test build + Worker.Tests (x86). The per-push default.
|
||||
live test + the live-MXAccess smoke (MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1) and a full
|
||||
slnx build (surfaces Windows-only analyzer diagnostics, e.g. xUnit1030). Nightly.
|
||||
|
||||
Runs under Windows PowerShell 5.1 (`powershell.exe`), so it avoids PS7-only syntax and
|
||||
checks $LASTEXITCODE after every native command rather than relying on stderr-as-error
|
||||
(git writes progress to stderr — the known windev gotcha).
|
||||
|
||||
Exits nonzero on the first failure so the SSH exit code propagates to the CI job.
|
||||
|
||||
.PARAMETER Sha
|
||||
The commit SHA under test. HEAD is force-checked-out to it and verified.
|
||||
|
||||
.PARAMETER Mode
|
||||
build | test | live.
|
||||
|
||||
.PARAMETER RepoRoot
|
||||
The CI clone root. Defaults to C:\build\mxaccessgw-ci.
|
||||
#>
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)][string]$Sha,
|
||||
[Parameter(Mandatory = $true)][ValidateSet('build', 'test', 'live')][string]$Mode,
|
||||
[string]$RepoRoot = 'C:\build\mxaccessgw-ci'
|
||||
)
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$lockDir = "$RepoRoot.lock"
|
||||
$logFile = Join-Path $RepoRoot 'ci.log'
|
||||
$lockTimeout = [TimeSpan]::FromMinutes(20)
|
||||
|
||||
function Write-Log {
|
||||
param([string]$Message)
|
||||
$line = ('[{0}] {1}' -f (Get-Date -Format 'yyyy-MM-dd HH:mm:ss'), $Message)
|
||||
Write-Host $line
|
||||
# Best-effort file log for the "is the tier down?" diagnosis in docs/GatewayTesting.md.
|
||||
try { Add-Content -Path $logFile -Value $line -ErrorAction SilentlyContinue } catch { }
|
||||
}
|
||||
|
||||
# Run a native command and throw on nonzero exit. Native stderr (git progress) does NOT
|
||||
# set $LASTEXITCODE, so it is not treated as failure — only a real nonzero exit is.
|
||||
function Invoke-Native {
|
||||
param([Parameter(Mandatory = $true)][string]$What, [Parameter(Mandatory = $true)][scriptblock]$Command)
|
||||
Write-Log "RUN: $What"
|
||||
& $Command
|
||||
if ($LASTEXITCODE -ne 0) {
|
||||
throw "$What failed with exit code $LASTEXITCODE"
|
||||
}
|
||||
}
|
||||
|
||||
# --- acquire the worktree lock (mkdir is atomic; retry until timeout) ---
|
||||
# run-windev-ci.sh's bootstrap already holds this lock around its pre-hand-off fetch/checkout
|
||||
# (MXGW_CI_LOCK_HELD=1) so that stage can't race a concurrent run; re-use it rather than
|
||||
# re-acquire (and leave releasing to the bootstrap). A standalone/manual run (degraded mode)
|
||||
# has no such env and self-locks here as before.
|
||||
$deadline = (Get-Date).Add($lockTimeout)
|
||||
$haveLock = $false
|
||||
if ($env:MXGW_CI_LOCK_HELD -eq '1') {
|
||||
Write-Log "Worktree lock already held by bootstrap (mode=$Mode sha=$Sha)"
|
||||
}
|
||||
else {
|
||||
Write-Log "Acquiring worktree lock $lockDir (mode=$Mode sha=$Sha)"
|
||||
while (-not $haveLock) {
|
||||
try {
|
||||
New-Item -Path $lockDir -ItemType Directory -ErrorAction Stop | Out-Null
|
||||
$haveLock = $true
|
||||
}
|
||||
catch {
|
||||
if ((Get-Date) -ge $deadline) {
|
||||
throw "Timed out after $($lockTimeout.TotalMinutes) min waiting for worktree lock $lockDir. A stuck run may have left it; if no CI job is active, remove it by hand."
|
||||
}
|
||||
Start-Sleep -Seconds 10
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
Push-Location $RepoRoot
|
||||
try {
|
||||
# Re-fetch + re-checkout under the lock so a concurrent run's bootstrap checkout
|
||||
# (done before we held the lock) cannot leave us on the wrong tree.
|
||||
Invoke-Native 'git fetch origin' { git fetch --prune origin }
|
||||
Invoke-Native "git checkout --force $Sha" { git checkout --force --detach $Sha }
|
||||
# Drop any stray tracked/untracked residue from a prior run so the build is clean.
|
||||
Invoke-Native 'git reset --hard' { git reset --hard $Sha }
|
||||
Invoke-Native 'git clean -fdx (build outputs)' { git clean -fdx -e ci.log }
|
||||
|
||||
$head = (& git rev-parse HEAD).Trim()
|
||||
if ($head -ne $Sha) {
|
||||
throw "Worktree HEAD $head does not match requested SHA $Sha after checkout."
|
||||
}
|
||||
Write-Log "Worktree at $head"
|
||||
|
||||
$workerProj = 'src/ZB.MOM.WW.MxGateway.Worker/ZB.MOM.WW.MxGateway.Worker.csproj'
|
||||
$workerTests = 'src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj'
|
||||
$integrationTests = 'src/ZB.MOM.WW.MxGateway.IntegrationTests/ZB.MOM.WW.MxGateway.IntegrationTests.csproj'
|
||||
|
||||
# build: always. The x86/net48 compile is the floor guard (catches CS0246 on
|
||||
# unregenerated Generated/, x86-only breaks) even when the test step is demoted.
|
||||
Invoke-Native 'x86 Worker build' { dotnet build $workerProj -c Release -p:Platform=x86 }
|
||||
|
||||
if ($Mode -eq 'test' -or $Mode -eq 'live') {
|
||||
Invoke-Native 'x86 Worker.Tests' { dotnet test $workerTests -c Release -p:Platform=x86 }
|
||||
}
|
||||
|
||||
if ($Mode -eq 'live') {
|
||||
# Full-solution build surfaces Windows-only analyzer diagnostics (xUnit1030 etc.)
|
||||
# that never fire on the macOS/Linux NonWindows build.
|
||||
Invoke-Native 'full slnx build' { dotnet build src/ZB.MOM.WW.MxGateway.slnx -c Release }
|
||||
|
||||
$env:MXGATEWAY_RUN_LIVE_MXACCESS_TESTS = '1'
|
||||
try {
|
||||
Invoke-Native 'live-MXAccess smoke' {
|
||||
dotnet test $integrationTests -c Release --filter 'FullyQualifiedName~WorkerLiveMxAccessSmokeTests'
|
||||
}
|
||||
}
|
||||
finally {
|
||||
Remove-Item Env:\MXGATEWAY_RUN_LIVE_MXACCESS_TESTS -ErrorAction SilentlyContinue
|
||||
}
|
||||
}
|
||||
|
||||
Write-Log "SUCCESS mode=$Mode sha=$Sha"
|
||||
}
|
||||
finally {
|
||||
Pop-Location
|
||||
}
|
||||
}
|
||||
catch {
|
||||
Write-Log "FAILURE mode=$Mode sha=$Sha : $($_.Exception.Message)"
|
||||
throw
|
||||
}
|
||||
finally {
|
||||
if ($haveLock) {
|
||||
Remove-Item -Path $lockDir -Recurse -Force -ErrorAction SilentlyContinue
|
||||
Write-Log "Released worktree lock $lockDir"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
# Pinned SSH host keys for the windev CI target (10.100.0.48), used by the
|
||||
# `windows-x86` / nightly jobs so `StrictHostKeyChecking=yes` can verify the host
|
||||
# without an interactive TOFU prompt. Host keys are public data — committing them is
|
||||
# intentional (greppable, versioned) per TST-25's design.
|
||||
#
|
||||
# Regenerate if windev's host keys rotate:
|
||||
# ssh-keyscan -t ed25519,rsa,ecdsa 10.100.0.48 > scripts/ci/windev.known_hosts
|
||||
# (then re-add this header). run-windev-ci.sh prefers the WINDEV_SSH_KNOWN_HOSTS
|
||||
# secret when set and falls back to this file.
|
||||
10.100.0.48 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHRqxpRq/G0BR5vAomHMNc/H7eyTkdqliHNrFlMQ9HCb
|
||||
10.100.0.48 ecdsa-sha2-nistp384 AAAAE2VjZHNhLXNoYTItbmlzdHAzODQAAAAIbmlzdHAzODQAAABhBIvTHAgf7Q+rK2dCyjSUgkYgos4j2eqyJN/AAfs2TXGJgrpp9AfOMnkbPtmMX3CvsrPoZxzBttIYcBYbf9KXb98JAu6fWqeglI/CeXH4oHw6oqMRtM0K2hl2gm3DicKKCQ==
|
||||
10.100.0.48 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCxKOhjdVawf3ziO9MB4VfLA1zDwnNK028siIzjTHZec4uJumInvSFD5S3tkJvSKGfCzXHtLu9m0zHmvJ4gIvLx4gt60TPABPC/QrdRG8mP4uOlPCciukWNIBhMWFU6oxHpZlQgrjQ3a+6WZxIjZBtaZQt3S5f9zEvOu6NUrLmP9cmrrXtAByMYyjISQf2BswxZ5ugeDHn0rVo3p3qYZ3M8lbXV+CRTwKj0I+VVceffJrYwLT0LyWxi0tDxmHxK1PgkhcIjLDWKhk4UxOA6gdO6MjuHvXZv5+UXJzjgGxom+xqxGhAge0F66p5IEJ/xrJMWoPBtFQ1by7tDZAsRhcbQJVeCbBjEDv7ShCbyh2Gq+PfJhWVVaYnMKcqBWkhPrZH4BE9yhAtSlV8QZsZ1iYUYKNAt9Kcg3ugRYx414KUf2fvUWSuoaRkJnBqQq1Mi0yK1JtTZOoYvN4XEYIr56hXQG1RfA8xWqYzjmbHYTgO8zv6tJJTeXLMqubmDdLP9H+U=
|
||||
@@ -34,10 +34,16 @@
|
||||
</PropertyGroup>
|
||||
</Target>
|
||||
|
||||
<!-- SQLitePCLRaw.lib.e_sqlite3 2.1.11 (transitive via Microsoft.Data.Sqlite) carries GHSA-2m69-gcr7-jv3q,
|
||||
which surfaces as NU1903 (warning-as-error). No patched e_sqlite3 release exists yet (2.1.11 is latest),
|
||||
so this targeted suppression keeps every OTHER transitive package audited. Remove once an upstream fix ships. -->
|
||||
<ItemGroup>
|
||||
<NuGetAuditSuppress Include="https://github.com/advisories/GHSA-2m69-gcr7-jv3q" />
|
||||
</ItemGroup>
|
||||
<!-- GHSA-2m69-gcr7-jv3q (NU1903, high) on the native SQLitePCLRaw.lib.e_sqlite3, pulled in
|
||||
transitively by Microsoft.Data.Sqlite.
|
||||
|
||||
The suppression that used to live here said "No patched e_sqlite3 release exists yet
|
||||
(2.1.11 is latest) ... Remove once an upstream fix ships." That fix HAS shipped: 2.1.12
|
||||
patches this advisory within the 2.1.x line. Suppression removed 2026-07-18, so the
|
||||
advisory is audited again instead of silenced.
|
||||
|
||||
The Server project already resolved 2.1.12, but only incidentally — transitively via
|
||||
ZB.MOM.WW.Auth.ApiKeys. The pin in that project makes the floor intentional, so a change
|
||||
to the Auth dependency graph cannot silently drop it back to the vulnerable 2.1.11.
|
||||
Verified: forced restore with no suppression reports no NU1903. -->
|
||||
</Project>
|
||||
|
||||
@@ -161,12 +161,32 @@ public sealed class DashboardLdapLiveTests
|
||||
|
||||
IConfiguration configuration = new ConfigurationBuilder()
|
||||
.AddJsonFile(appSettingsPath, optional: false)
|
||||
.AddEnvironmentVariables()
|
||||
.Build();
|
||||
|
||||
// Same section production binds in AddZbLdapAuth(configuration, "MxGateway:Ldap").
|
||||
// Get<T> returns null only when the section is absent; appsettings.json always
|
||||
// carries it, so fall back to shared defaults defensively rather than throw.
|
||||
return configuration.GetSection("MxGateway:Ldap").Get<LibraryLdapOptions>()
|
||||
LibraryLdapOptions options = configuration.GetSection("MxGateway:Ldap").Get<LibraryLdapOptions>()
|
||||
?? new LibraryLdapOptions();
|
||||
|
||||
// appsettings.json now ships the bind password as the unexpanded
|
||||
// "${secret:ldap/mxgateway/bind}" token (resolved at gateway startup by the
|
||||
// pre-host secrets expander, which this bare ConfigurationBuilder does not run).
|
||||
// AddEnvironmentVariables() above lets MxGateway__Ldap__ServiceAccountPassword
|
||||
// override the token with the real password. Fail loud rather than silently
|
||||
// binding with the literal token string, which would make every live test in
|
||||
// this file fail with a confusing LDAP bind error instead of an actionable one.
|
||||
if (string.IsNullOrEmpty(options.ServiceAccountPassword)
|
||||
|| options.ServiceAccountPassword.StartsWith("${secret:", StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"Live LDAP tests require the real bind password via the "
|
||||
+ "MxGateway__Ldap__ServiceAccountPassword environment variable "
|
||||
+ "(appsettings now ships a ${secret:} token that this suite does not expand). "
|
||||
+ "Set it before running.");
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -22,8 +22,8 @@
|
||||
(IntegrationTests-028).
|
||||
-->
|
||||
<ItemGroup>
|
||||
<PackageReference Include="ZB.MOM.WW.Auth.Abstractions" Version="0.1.4" />
|
||||
<PackageReference Include="ZB.MOM.WW.Auth.Ldap" Version="0.1.4" />
|
||||
<PackageReference Include="ZB.MOM.WW.Auth.Abstractions" Version="0.1.5" />
|
||||
<PackageReference Include="ZB.MOM.WW.Auth.Ldap" Version="0.1.5" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="10.0.7" />
|
||||
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="10.0.7" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -228,7 +228,7 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
// Consume mapped MxEvents through the session's single distributor pump (as an
|
||||
// internal, non-counted subscriber) rather than opening a second raw drain of the
|
||||
// worker event channel — a second drain would split events with the dashboard
|
||||
// mirror pump and silently lose Acknowledge/mode-change transitions (GWC-01).
|
||||
// mirror pump and silently lose Acknowledge/mode-change transitions.
|
||||
await foreach (MxEvent mxEvent in _sessionManager
|
||||
.ReadAlarmEventsAsync(session.SessionId, linked.Token)
|
||||
.ConfigureAwait(false))
|
||||
|
||||
+6
-2
@@ -15,8 +15,8 @@ public static class GatewayConfigurationServiceCollectionExtensions
|
||||
public static IServiceCollection AddGatewayConfiguration(
|
||||
this IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
// GatewayOptionsValidator depends on IHostEnvironment to gate the production-only rules
|
||||
// (SEC-04/06). The real host and the apikey CLI both build through WebApplicationBuilder,
|
||||
// GatewayOptionsValidator depends on IHostEnvironment to gate the production-only rules.
|
||||
// The real host and the apikey CLI both build through WebApplicationBuilder,
|
||||
// which registers IHostEnvironment before this call — so TryAdd is a no-op there. Minimal
|
||||
// containers (unit tests, tooling) have none; fall back to a non-production environment so
|
||||
// the validator still activates and the production guards stay off outside a real
|
||||
@@ -37,12 +37,16 @@ public static class GatewayConfigurationServiceCollectionExtensions
|
||||
/// </summary>
|
||||
private sealed class FallbackHostEnvironment : IHostEnvironment
|
||||
{
|
||||
/// <summary>Gets or sets the name of the environment; defaults to <see cref="Environments.Development"/>.</summary>
|
||||
public string EnvironmentName { get; set; } = Environments.Development;
|
||||
|
||||
/// <summary>Gets or sets the name of the application.</summary>
|
||||
public string ApplicationName { get; set; } = typeof(FallbackHostEnvironment).Assembly.GetName().Name ?? "MxGateway";
|
||||
|
||||
/// <summary>Gets or sets the absolute path to the content root directory.</summary>
|
||||
public string ContentRootPath { get; set; } = AppContext.BaseDirectory;
|
||||
|
||||
/// <summary>Gets or sets the file provider for the content root.</summary>
|
||||
public IFileProvider ContentRootFileProvider { get; set; } = new NullFileProvider();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -470,7 +470,7 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
||||
|
||||
// The worker-frame (pipe) maximum must stay above the public gRPC cap by the envelope-overhead
|
||||
// reserve, otherwise a maximally-sized accepted gRPC payload does not fit one worker frame once
|
||||
// wrapped in a WorkerEnvelope and the outbound write faults the whole session (IPC-03). Fail fast
|
||||
// wrapped in a WorkerEnvelope and the outbound write faults the whole session. Fail fast
|
||||
// at startup rather than mid-traffic. Only checked when both knobs are themselves in range so the
|
||||
// message is not doubled up with the individual range errors.
|
||||
private static void ValidateFrameSizeHeadroom(
|
||||
|
||||
@@ -58,7 +58,7 @@ public sealed class LdapOptions
|
||||
public string ServiceAccountDn { get; init; } = "cn=serviceaccount,dc=zb,dc=local";
|
||||
|
||||
/// <summary>Gets the service account password.</summary>
|
||||
public string ServiceAccountPassword { get; init; } = "serviceaccount123";
|
||||
public string ServiceAccountPassword { get; init; } = string.Empty;
|
||||
|
||||
/// <summary>Gets the LDAP attribute name for user names.</summary>
|
||||
public string UserNameAttribute { get; init; } = "cn";
|
||||
|
||||
@@ -2,8 +2,8 @@ namespace ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Security hot-path options bound from <c>MxGateway:Security</c>. Groups the API-key verification
|
||||
/// cache and <c>last_used</c> write-coalescing knobs (SEC-08) with the login and API-key
|
||||
/// rate-limiting knobs (SEC-11). Defaults preserve safe, low-overhead behaviour without requiring
|
||||
/// cache and <c>last_used</c> write-coalescing knobs with the login and API-key
|
||||
/// rate-limiting knobs. Defaults preserve safe, low-overhead behaviour without requiring
|
||||
/// operators to configure anything.
|
||||
/// </summary>
|
||||
public sealed class SecurityOptions
|
||||
|
||||
@@ -37,8 +37,8 @@ public sealed class WorkerOptions
|
||||
/// The maximum worker-frame (pipe) message size in bytes. Must stay at least
|
||||
/// <see cref="Workers.WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes"/> above
|
||||
/// <see cref="ProtocolOptions.MaxGrpcMessageBytes"/> so a maximally-sized accepted gRPC payload
|
||||
/// still fits one worker frame (IPC-03); the gateway conveys this value to the worker in the
|
||||
/// handshake (<c>GatewayHello.max_frame_bytes</c>, IPC-02). Default is the 16 MB public gRPC cap
|
||||
/// still fits one worker frame; the gateway conveys this value to the worker in the
|
||||
/// handshake (<c>GatewayHello.max_frame_bytes</c>). Default is the 16 MB public gRPC cap
|
||||
/// plus that reserve.
|
||||
/// </summary>
|
||||
public int MaxMessageBytes { get; init; } =
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
</NavRailSection>
|
||||
<NavRailSection Title="Admin" Key="admin">
|
||||
<NavRailItem Href="/apikeys" Text="API Keys" />
|
||||
<NavRailItem Href="/admin/secrets" Text="Secrets" />
|
||||
<NavRailItem Href="/settings" Text="Settings" />
|
||||
</NavRailSection>
|
||||
</Nav>
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
<Router AppAssembly="@typeof(Routes).Assembly">
|
||||
<Router AppAssembly="@typeof(Routes).Assembly"
|
||||
AdditionalAssemblies="new[]{ typeof(ZB.MOM.WW.Secrets.Ui.SecretsPage).Assembly }">
|
||||
<Found Context="routeData">
|
||||
<RouteView RouteData="@routeData" DefaultLayout="@typeof(MainLayout)" />
|
||||
<FocusOnNavigate RouteData="@routeData" Selector="h1" />
|
||||
|
||||
@@ -101,8 +101,6 @@ public sealed class DashboardApiKeyManagementService(
|
||||
.RevokeKeyAsync(normalizedKeyId, RemoteAddress(), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// SEC-08: drop any cached verification for this key in THIS gateway process so the revoke
|
||||
// takes effect immediately rather than after the verification-cache TTL elapses.
|
||||
cacheInvalidator?.Invalidate(normalizedKeyId);
|
||||
|
||||
await WriteDashboardAuditAsync(
|
||||
@@ -143,8 +141,6 @@ public sealed class DashboardApiKeyManagementService(
|
||||
.RotateKeyAsync(normalizedKeyId, RemoteAddress(), cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// SEC-08: the old secret's cached verification must not outlive the rotate — evict it so
|
||||
// the superseded secret stops authenticating within this process immediately.
|
||||
cacheInvalidator?.Invalidate(normalizedKeyId);
|
||||
|
||||
bool succeeded = rotated.Token is not null;
|
||||
@@ -191,8 +187,6 @@ public sealed class DashboardApiKeyManagementService(
|
||||
.DeleteAsync(normalizedKeyId, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
// SEC-08: a deleted key is only reachable after a prior revoke, but evict defensively so no
|
||||
// cached verification survives the delete.
|
||||
cacheInvalidator?.Invalidate(normalizedKeyId);
|
||||
|
||||
await WriteDashboardAuditAsync(
|
||||
|
||||
@@ -12,7 +12,7 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
public static class DashboardEndpointRouteBuilderExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// The named rate-limiter policy applied to the <c>POST /auth/login</c> route (SEC-11). Registered
|
||||
/// The named rate-limiter policy applied to the <c>POST /auth/login</c> route. Registered
|
||||
/// in <c>GatewayApplication</c> via <see cref="GetLoginRateLimitPartition"/>.
|
||||
/// </summary>
|
||||
internal const string LoginRateLimiterPolicy = "dashboard-login";
|
||||
@@ -40,8 +40,12 @@ public static class DashboardEndpointRouteBuilderExtensions
|
||||
});
|
||||
}
|
||||
|
||||
// Test seam: a standalone partitioned limiter over the production partition logic, so a test can
|
||||
// acquire leases and assert the (permit+1)th is rejected without an HTTP server.
|
||||
/// <summary>
|
||||
/// Test seam: builds a standalone partitioned limiter over the production partition logic, so a
|
||||
/// test can acquire leases and assert the (permit+1)th is rejected without an HTTP server.
|
||||
/// </summary>
|
||||
/// <param name="security">The bound security options carrying the login-limit knobs.</param>
|
||||
/// <returns>A partitioned rate limiter keyed the same way as the production login policy.</returns>
|
||||
internal static PartitionedRateLimiter<HttpContext> CreateLoginRateLimiter(SecurityOptions security)
|
||||
=> PartitionedRateLimiter.Create<HttpContext, string>(
|
||||
ctx => GetLoginRateLimitPartition(ctx, security));
|
||||
@@ -76,7 +80,6 @@ public static class DashboardEndpointRouteBuilderExtensions
|
||||
(HttpContext httpContext, IAntiforgery antiforgery, IDashboardAuthenticator authenticator) =>
|
||||
PostLoginAsync(httpContext, antiforgery, authenticator))
|
||||
.AllowAnonymous()
|
||||
// SEC-11: throttle credential stuffing before the LDAP bind is relayed to the directory.
|
||||
.RequireRateLimiting(LoginRateLimiterPolicy)
|
||||
.WithName("DashboardLoginPost");
|
||||
|
||||
@@ -129,6 +132,7 @@ public static class DashboardEndpointRouteBuilderExtensions
|
||||
// cookie scheme and redirected to /login.
|
||||
endpoints.MapRazorComponents<App>()
|
||||
.AddInteractiveServerRenderMode()
|
||||
.AddAdditionalAssemblies(typeof(ZB.MOM.WW.Secrets.Ui.SecretsPage).Assembly)
|
||||
.RequireAuthorization(DashboardAuthenticationDefaults.ViewerPolicy);
|
||||
|
||||
return endpoints;
|
||||
|
||||
@@ -33,7 +33,7 @@ public sealed class HubTokenService
|
||||
// bounds how long a stale role set survives a role change. Five minutes is transparent to
|
||||
// clients because DashboardHubConnectionFactory mints a fresh token on every (re)connect;
|
||||
// see docs/GatewayDashboardDesign.md. Heavier jti-denylist revocation is deliberately
|
||||
// deferred until per-session hub ACLs land (SEC-25), when tokens gain session binding.
|
||||
// deferred until per-session hub ACLs land, when tokens gain session binding.
|
||||
internal static readonly TimeSpan TokenLifetime = TimeSpan.FromMinutes(5);
|
||||
|
||||
private readonly ITimeLimitedDataProtector _protector;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
using System.Security.Cryptography.X509Certificates;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Hosting.StaticWebAssets;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Configuration;
|
||||
@@ -15,6 +16,11 @@ using ZB.MOM.WW.MxGateway.Server.Security.Authentication;
|
||||
using ZB.MOM.WW.MxGateway.Server.Security.Authorization;
|
||||
using ZB.MOM.WW.MxGateway.Server.Sessions;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Configuration;
|
||||
using ZB.MOM.WW.Secrets.DependencyInjection;
|
||||
using ZB.MOM.WW.Secrets.Sqlite;
|
||||
using ZB.MOM.WW.Secrets.Ui;
|
||||
using ZB.MOM.WW.Telemetry;
|
||||
using ZB.MOM.WW.Telemetry.Serilog;
|
||||
|
||||
@@ -41,7 +47,7 @@ public static class GatewayApplication
|
||||
app.UseStaticFiles();
|
||||
app.UseAuthentication();
|
||||
app.UseAuthorization();
|
||||
// SEC-11: the rate limiter must run after routing has selected the endpoint (so its
|
||||
// The rate limiter must run after routing has selected the endpoint (so its
|
||||
// RequireRateLimiting policy metadata is visible) and before the endpoint executes.
|
||||
app.UseRateLimiter();
|
||||
app.UseAntiforgery();
|
||||
@@ -64,11 +70,35 @@ public static class GatewayApplication
|
||||
});
|
||||
StaticWebAssetsLoader.UseStaticWebAssets(builder.Environment, builder.Configuration);
|
||||
|
||||
// Resolve ${secret:...} references in configuration BEFORE any config consumer (TLS, Kestrel,
|
||||
// GatewayOptions/Ldap/Galaxy validators) reads a value, using a standalone secrets provider
|
||||
// (envelope-decrypted via the master key). A token referencing a missing secret fails fast
|
||||
// here (SecretNotFoundException); config with no tokens is untouched (no-op), so this is safe
|
||||
// to always run. CreateBuilder is synchronous and single-shot at bootstrap, so the two awaits
|
||||
// are driven via GetAwaiter().GetResult() (no sync-context deadlock risk during host startup).
|
||||
#pragma warning disable ASP0000 // deliberate throwaway container, disposed here, shares no singletons
|
||||
using (var secretsProvider = new ServiceCollection()
|
||||
.AddZbSecrets(builder.Configuration, "Secrets")
|
||||
.BuildServiceProvider())
|
||||
#pragma warning restore ASP0000
|
||||
{
|
||||
// Intentionally unconditional: the pre-host expander below needs the secrets
|
||||
// store schema to exist before the first ${secret:} resolve. Secrets:RunMigrationsOnStartup
|
||||
// governs only the runtime SecretsMigrationHostedService, not this bootstrap path.
|
||||
secretsProvider.GetRequiredService<SqliteSecretsStoreMigrator>()
|
||||
.MigrateAsync(default).GetAwaiter().GetResult();
|
||||
var resolver = secretsProvider.GetRequiredService<ISecretResolver>();
|
||||
new SecretReferenceExpander(resolver)
|
||||
.ExpandConfigurationAsync((IConfigurationRoot)builder.Configuration, default)
|
||||
.GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
ConfigureSelfSignedTls(builder);
|
||||
|
||||
builder.AddZbSerilog(o => o.ServiceName = "mxgateway");
|
||||
|
||||
builder.Services.AddGatewayConfiguration(builder.Configuration);
|
||||
builder.Services.AddZbSecrets(builder.Configuration, "Secrets");
|
||||
builder.Services.AddSqliteAuthStore(builder.Configuration);
|
||||
builder.Services.AddGatewayGrpcAuthorization();
|
||||
AddLoginRateLimiter(builder);
|
||||
@@ -96,6 +126,10 @@ public static class GatewayApplication
|
||||
builder.Services.AddGatewaySessions();
|
||||
builder.Services.AddGatewayAlarms();
|
||||
builder.Services.AddGatewayDashboard(builder.Configuration);
|
||||
// Register the shared Secrets UI authorization policies (secrets:manage + secrets:reveal)
|
||||
// additively so they compose with the dashboard's existing AddAuthorization block. The
|
||||
// mounted /admin/secrets Blazor page carries [Authorize(Policy = "secrets:manage")].
|
||||
builder.Services.Configure<AuthorizationOptions>(o => o.AddSecretsAuthorization());
|
||||
// Register the gateway's browse-scope provider before AddZbGalaxyRepository so the
|
||||
// library's TryAddSingleton default (NullGalaxyBrowseScopeProvider) does not win.
|
||||
builder.Services.AddSingleton<ZB.MOM.WW.GalaxyRepository.Grpc.IGalaxyBrowseScopeProvider,
|
||||
@@ -105,7 +139,7 @@ public static class GatewayApplication
|
||||
return builder;
|
||||
}
|
||||
|
||||
// SEC-11: register the named fixed-window rate-limiter policy applied to POST /auth/login. The
|
||||
// Registers the named fixed-window rate-limiter policy applied to POST /auth/login. The
|
||||
// limit knobs are read from MxGateway:Security at startup; the per-request partition is keyed on
|
||||
// the remote IP (see DashboardEndpointRouteBuilderExtensions.GetLoginRateLimitPartition).
|
||||
private static void AddLoginRateLimiter(WebApplicationBuilder builder)
|
||||
|
||||
@@ -14,43 +14,33 @@ public sealed class EventStreamService(
|
||||
GatewayMetrics metrics) : IEventStreamService
|
||||
{
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// This reads the subscriber's lease channel fed by the session's single
|
||||
/// <see cref="SessionEventDistributor"/> pump. The pump owns the single drain of
|
||||
/// the worker event stream and the worker→public mapping (mirroring the former
|
||||
/// <c>ProduceEventsAsync</c>); this loop is the per-subscriber boundary that
|
||||
/// applies the per-RPC filter (<c>AfterWorkerSequence</c>), queue-depth metrics,
|
||||
/// and the backpressure/overflow policy.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The dashboard mirror runs OFF this per-RPC loop. The dashboard is a
|
||||
/// first-class internal subscriber on the session's
|
||||
/// <see cref="SessionEventDistributor"/> (see <c>GatewaySession.StartDashboardMirror</c>),
|
||||
/// so it receives session events even when no gRPC client is streaming. This loop
|
||||
/// does not mirror to the dashboard. One deliberate consequence: the dashboard sees
|
||||
/// RAW session events, not the per-gRPC-subscriber <c>AfterWorkerSequence</c>-filtered
|
||||
/// view this loop applies — the dashboard is a separate LDAP-authenticated monitoring
|
||||
/// view that should see the session's full event activity.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Overflow handling: the distributor's per-subscriber channel is bounded
|
||||
/// and the pump writes non-blocking. When this subscriber's channel is full the pump
|
||||
/// applies the per-subscriber backpressure policy and completes this subscriber's
|
||||
/// channel with a <see cref="SessionManagerException"/>
|
||||
/// (<see cref="SessionManagerErrorCode.EventQueueOverflow"/>). That terminal fault
|
||||
/// surfaces here when the reader's <c>MoveNextAsync</c> throws, and it propagates to
|
||||
/// the gRPC client unchanged. The overflow metric, and (in the legacy
|
||||
/// single-subscriber FailFast case) the session fault + fault metric, are recorded by
|
||||
/// the distributor's overflow handler so the session, the pump, and other subscribers
|
||||
/// are isolated from this subscriber's slowness.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public async IAsyncEnumerable<MxEvent> StreamEventsAsync(
|
||||
StreamEventsRequest request,
|
||||
string? callerKeyId,
|
||||
[EnumeratorCancellation] CancellationToken cancellationToken)
|
||||
{
|
||||
// This reads the subscriber's lease channel fed by the session's single
|
||||
// SessionEventDistributor pump. The pump owns the single drain of the worker event
|
||||
// stream and the worker->public mapping (mirroring the former ProduceEventsAsync); this
|
||||
// loop is the per-subscriber boundary that applies the per-RPC filter (AfterWorkerSequence),
|
||||
// queue-depth metrics, and the backpressure/overflow policy.
|
||||
//
|
||||
// The dashboard mirror runs OFF this per-RPC loop. The dashboard is a first-class internal
|
||||
// subscriber on the session's SessionEventDistributor (see GatewaySession.StartDashboardMirror),
|
||||
// so it receives session events even when no gRPC client is streaming. This loop does not
|
||||
// mirror to the dashboard. One deliberate consequence: the dashboard sees RAW session events,
|
||||
// not the per-gRPC-subscriber AfterWorkerSequence-filtered view this loop applies — the
|
||||
// dashboard is a separate LDAP-authenticated monitoring view that should see the session's
|
||||
// full event activity.
|
||||
//
|
||||
// Overflow handling: the distributor's per-subscriber channel is bounded and the pump writes
|
||||
// non-blocking. When this subscriber's channel is full the pump applies the per-subscriber
|
||||
// backpressure policy and completes this subscriber's channel with a SessionManagerException
|
||||
// (SessionManagerErrorCode.EventQueueOverflow). That terminal fault surfaces here when the
|
||||
// reader's MoveNextAsync throws, and it propagates to the gRPC client unchanged. The overflow
|
||||
// metric, and (in the legacy single-subscriber FailFast case) the session fault + fault metric,
|
||||
// are recorded by the distributor's overflow handler so the session, the pump, and other
|
||||
// subscribers are isolated from this subscriber's slowness.
|
||||
if (!sessionManager.TryGetSession(request.SessionId, out GatewaySession? session) || session is null)
|
||||
{
|
||||
throw new SessionManagerException(
|
||||
@@ -58,7 +48,7 @@ public sealed class EventStreamService(
|
||||
$"Session {request.SessionId} was not found.");
|
||||
}
|
||||
|
||||
// Owner-scoped attach (TST-02, security control): a session's event stream may be
|
||||
// Owner-scoped attach (security control): a session's event stream may be
|
||||
// attached or reattached ONLY by the API key that opened the session. The detach-grace
|
||||
// and fan-out retention windows are on by default, so without this check any event-scoped
|
||||
// key that learns a session id could attach to another key's retained session and receive
|
||||
|
||||
@@ -7,7 +7,7 @@ public sealed class MxAccessGrpcRequestValidator
|
||||
{
|
||||
// Upper bound on a single DrainEvents request. DrainEvents is a diagnostics RPC that returns
|
||||
// buffered events in one non-streaming reply, so an unbounded max_events could pack the whole
|
||||
// queue into a session-killing frame (IPC-04). The worker independently caps each reply at its
|
||||
// queue into a session-killing frame. The worker independently caps each reply at its
|
||||
// own MaxDrainEventsPerReply; this public bound rejects an obviously-abusive request loudly at
|
||||
// the boundary. max_events = 0 is allowed and means "the worker's default batch cap".
|
||||
private const uint MaxDrainEventsPerRequest = 10_000;
|
||||
|
||||
+1
-1
@@ -236,7 +236,7 @@ public static class ApiKeyAdminCommandLineParser
|
||||
ReadHistorizedOnly: HasFlag(options, "read-historized-only"));
|
||||
}
|
||||
|
||||
// Parses the optional --expires value into an absolute UTC expiry (SEC-10). Accepts a relative
|
||||
// Parses the optional --expires value into an absolute UTC expiry. Accepts a relative
|
||||
// "<N>d"/"<N>h" duration from now (operator-friendly) or an absolute ISO-8601 instant/date
|
||||
// (assumed UTC). Null/blank means no expiry — expiry stays opt-in, preserving prior behaviour.
|
||||
private static DateTimeOffset? ParseExpiry(string? value)
|
||||
|
||||
+2
-2
@@ -69,7 +69,7 @@ public static class AuthStoreServiceCollectionExtensions
|
||||
// migrator and the migration hosted service.
|
||||
services.AddZbApiKeyAuth(effectiveConfig, AuthenticationSectionPath);
|
||||
|
||||
// SEC-08 hot-path decorators. Every gRPC call previously did a SQLite read plus a
|
||||
// Hot-path decorators. Every gRPC call previously did a SQLite read plus a
|
||||
// last_used_utc WRITE via IApiKeyVerifier.VerifyAsync (the library verifier couples the
|
||||
// mark into verification). Two gateway-side decorators cut that cost without editing the
|
||||
// external library:
|
||||
@@ -138,7 +138,7 @@ public static class AuthStoreServiceCollectionExtensions
|
||||
/// <summary>
|
||||
/// Replaces the last registration of <typeparamref name="TService"/> with a singleton that wraps
|
||||
/// it. The wrapped (inner) service is created once, preserving singleton semantics. Used to layer
|
||||
/// the SEC-08 store decorator over the external library's registration without editing it.
|
||||
/// the store decorator over the external library's registration without editing it.
|
||||
/// </summary>
|
||||
private static void DecorateSingleton<TService>(
|
||||
IServiceCollection services,
|
||||
|
||||
@@ -30,7 +30,7 @@ public interface IApiKeyCacheInvalidator
|
||||
/// <para>
|
||||
/// Only successful verifications are cached. Failures and unparseable headers always fall through
|
||||
/// to the inner verifier — caching a failure risks pinning a transiently-wrong negative, and the
|
||||
/// SEC-11 per-peer failure counter (not this cache) is what bounds brute-force cost.
|
||||
/// per-peer failure counter (not this cache) is what bounds brute-force cost.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The cache key is the hex SHA-256 of the presented token (which embeds both the key id and the
|
||||
@@ -71,7 +71,11 @@ public sealed class CachingApiKeyVerifier : IApiKeyVerifier, IApiKeyCacheInvalid
|
||||
{
|
||||
}
|
||||
|
||||
// Test/explicit-TTL seam.
|
||||
/// <summary>Initializes a new instance of the <see cref="CachingApiKeyVerifier"/> class with an explicit TTL.</summary>
|
||||
/// <param name="inner">The wrapped verifier (the library verifier) reached on a cache miss.</param>
|
||||
/// <param name="cache">The shared memory cache.</param>
|
||||
/// <param name="ttl">The verification-cache TTL; a zero or negative value disables caching.</param>
|
||||
/// <remarks>Test/explicit-TTL seam.</remarks>
|
||||
internal CachingApiKeyVerifier(IApiKeyVerifier inner, IMemoryCache cache, TimeSpan ttl)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(inner);
|
||||
@@ -81,7 +85,14 @@ public sealed class CachingApiKeyVerifier : IApiKeyVerifier, IApiKeyCacheInvalid
|
||||
_ttl = ttl;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>
|
||||
/// Verifies the given authorization header, returning a cached successful result when one is
|
||||
/// still within TTL, or falling through to the inner verifier on a cache miss or non-cacheable
|
||||
/// header.
|
||||
/// </summary>
|
||||
/// <param name="authorizationHeader">The raw <c>Authorization</c> header value to verify.</param>
|
||||
/// <param name="ct">Token to cancel the asynchronous operation.</param>
|
||||
/// <returns>The verification result, cached or freshly computed.</returns>
|
||||
public async Task<ApiKeyVerification> VerifyAsync(string authorizationHeader, CancellationToken ct)
|
||||
{
|
||||
if (_ttl <= TimeSpan.Zero || !TryComputeCacheKey(authorizationHeader, out string cacheKey))
|
||||
|
||||
+17
-4
@@ -41,7 +41,10 @@ public sealed class CoalescingMarkApiKeyStore : IApiKeyStore
|
||||
{
|
||||
}
|
||||
|
||||
// Test/explicit-window seam.
|
||||
/// <summary>Initializes a new instance of the <see cref="CoalescingMarkApiKeyStore"/> class with an explicit coalescing window (test/explicit-window seam).</summary>
|
||||
/// <param name="inner">The wrapped store.</param>
|
||||
/// <param name="window">The coalescing window; marks within this window of the last forwarded mark for a key are dropped.</param>
|
||||
/// <param name="clock">The time provider.</param>
|
||||
internal CoalescingMarkApiKeyStore(IApiKeyStore inner, TimeSpan window, TimeProvider clock)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(inner);
|
||||
@@ -51,15 +54,25 @@ public sealed class CoalescingMarkApiKeyStore : IApiKeyStore
|
||||
_clock = clock;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>Looks up an API key record by key id, delegating to the wrapped store unchanged.</summary>
|
||||
/// <param name="keyId">The API key id to look up.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>The matching <see cref="ApiKeyRecord"/>, or <see langword="null"/> if none exists.</returns>
|
||||
public Task<ApiKeyRecord?> FindByKeyIdAsync(string keyId, CancellationToken ct)
|
||||
=> _inner.FindByKeyIdAsync(keyId, ct);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>Looks up an active API key record by key id, delegating to the wrapped store unchanged.</summary>
|
||||
/// <param name="keyId">The API key id to look up.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>The matching active <see cref="ApiKeyRecord"/>, or <see langword="null"/> if none exists or is inactive.</returns>
|
||||
public Task<ApiKeyRecord?> FindActiveByKeyIdAsync(string keyId, CancellationToken ct)
|
||||
=> _inner.FindActiveByKeyIdAsync(keyId, ct);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>Marks the key as used, coalescing writes so at most one reaches the wrapped store per key per window.</summary>
|
||||
/// <param name="keyId">The API key id being marked as used.</param>
|
||||
/// <param name="whenUtc">The UTC timestamp of the use.</param>
|
||||
/// <param name="ct">The cancellation token.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public Task MarkUsedAsync(string keyId, DateTimeOffset whenUtc, CancellationToken ct)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(keyId);
|
||||
|
||||
@@ -19,11 +19,6 @@ namespace ZB.MOM.WW.MxGateway.Server.Security.Authentication;
|
||||
/// </remarks>
|
||||
public static class GatewayApiKeyIdentityMapper
|
||||
{
|
||||
// SEC-08: memoize the constraints deserialization keyed by the raw JSON blob so the per-call
|
||||
// JSON parse on the auth hot path collapses to a dictionary lookup. Distinct blobs are bounded
|
||||
// by the number of API keys (small); ApiKeyConstraints is an immutable record, so a parsed
|
||||
// instance is safe to share across callers. The cap is a defensive backstop against a pathological
|
||||
// spread of distinct blobs — past it, we simply parse without caching rather than grow unbounded.
|
||||
private const int MaxCachedConstraintBlobs = 1024;
|
||||
private static readonly ConcurrentDictionary<string, ApiKeyConstraints> ConstraintCache =
|
||||
new(StringComparer.Ordinal);
|
||||
|
||||
@@ -4,7 +4,7 @@ using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Security.Authorization;
|
||||
|
||||
/// <summary>
|
||||
/// Cheap, in-process per-peer sliding-window failure counter for the gRPC auth path (SEC-11). It is
|
||||
/// Cheap, in-process per-peer sliding-window failure counter for the gRPC auth path. It is
|
||||
/// checked BEFORE the API-key verification store read and short-circuits a peer that has exceeded
|
||||
/// <see cref="SecurityOptions.ApiKeyFailureLimit"/> failed attempts within
|
||||
/// <see cref="SecurityOptions.ApiKeyFailureWindowSeconds"/>; a successful verification resets the
|
||||
@@ -45,7 +45,11 @@ public sealed class ApiKeyFailureLimiter
|
||||
{
|
||||
}
|
||||
|
||||
// Test/explicit seam.
|
||||
/// <summary>Initializes a new instance of the <see cref="ApiKeyFailureLimiter"/> class. Test/explicit seam.</summary>
|
||||
/// <param name="limit">The maximum number of failures allowed within <paramref name="window"/>.</param>
|
||||
/// <param name="window">The sliding window over which failures are counted.</param>
|
||||
/// <param name="maxPeers">The maximum number of tracked peers before least-recently-active eviction kicks in.</param>
|
||||
/// <param name="clock">The time provider.</param>
|
||||
internal ApiKeyFailureLimiter(int limit, TimeSpan window, int maxPeers, TimeProvider clock)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(clock);
|
||||
@@ -145,6 +149,7 @@ public sealed class ApiKeyFailureLimiter
|
||||
|
||||
private sealed class PeerState
|
||||
{
|
||||
/// <summary>Timestamps (in ticks) of failures still within the sliding window.</summary>
|
||||
public Queue<long> FailureTicks { get; } = new();
|
||||
|
||||
public long LastActivityTicks;
|
||||
|
||||
+1
-1
@@ -64,7 +64,7 @@ public sealed class GatewayGrpcAuthorizationInterceptor(
|
||||
|
||||
string? authorizationHeader = context.RequestHeaders.GetValue("authorization");
|
||||
|
||||
// SEC-11: short-circuit a peer that has already failed too many times inside the sliding
|
||||
// Short-circuit a peer that has already failed too many times inside the sliding
|
||||
// window BEFORE the verification store read, so online guessing cannot spend a SQLite read
|
||||
// per attempt. The peer key prefers the presented key id over the transport address (NAT
|
||||
// caveat). ResourceExhausted signals throttling without revealing whether any particular
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ public static class GrpcAuthorizationServiceCollectionExtensions
|
||||
services.AddSingleton<GatewayGrpcScopeResolver>();
|
||||
services.AddSingleton<IGatewayRequestIdentityAccessor, GatewayRequestIdentityAccessor>();
|
||||
services.AddSingleton<IConstraintEnforcer, ConstraintEnforcer>();
|
||||
// SEC-11 per-peer failure counter, checked before the verification store read. Bind the knobs
|
||||
// Per-peer failure counter, checked before the verification store read. Bind the knobs
|
||||
// from IConfiguration directly (not IOptions<GatewayOptions>) to avoid coupling this
|
||||
// registration to the whole-options validation pipeline.
|
||||
services.AddSingleton(sp => new ApiKeyFailureLimiter(
|
||||
|
||||
@@ -222,11 +222,6 @@ public sealed class SessionManager : ISessionManager
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Mirrors the registry/metrics cleanup that <see cref="CloseSessionCoreAsync"/>
|
||||
/// performs after a successful close, but skips the <c>WorkerClient.ShutdownAsync</c>
|
||||
/// step that <see cref="GatewaySession.CloseAsync"/> would otherwise attempt.
|
||||
/// </remarks>
|
||||
public async Task<SessionCloseResult> KillWorkerAsync(
|
||||
string sessionId,
|
||||
string reason,
|
||||
|
||||
@@ -27,7 +27,7 @@ public sealed class WorkerClient : IWorkerClient
|
||||
|
||||
// Staging hand-off between the read loop and the dedicated event writer. The read loop writes
|
||||
// here with a non-blocking TryWrite so a full consumer channel (_events) can never stall the read
|
||||
// loop behind an event — replies and heartbeats keep flowing (GWC-04). Unbounded, but only fills
|
||||
// loop behind an event — replies and heartbeats keep flowing. Unbounded, but only fills
|
||||
// during the bounded EventChannelFullModeTimeout window before EventWriteLoopAsync faults on a
|
||||
// sustained backlog, after which the read loop stops.
|
||||
private readonly Channel<WorkerEvent> _eventStaging;
|
||||
@@ -208,7 +208,7 @@ public sealed class WorkerClient : IWorkerClient
|
||||
|
||||
// Reject an oversized command at the enqueue boundary so only this correlation fails
|
||||
// (ResourceExhausted) rather than the frame reaching the write loop and faulting the whole
|
||||
// session (IPC-03). Command envelopes are the only gateway-authored outbound payload whose
|
||||
// session. Command envelopes are the only gateway-authored outbound payload whose
|
||||
// size the caller controls; checking here keeps a MessageTooLarge in the write loop a
|
||||
// genuine desync signal.
|
||||
int envelopeSize = commandEnvelope.CalculateSize();
|
||||
@@ -269,7 +269,7 @@ public sealed class WorkerClient : IWorkerClient
|
||||
// The event channel is SingleReader: only one enumerator may ever drain it, otherwise
|
||||
// the two readers would each receive a random subset of events. Claim the reader at CALL
|
||||
// time (not lazily on first MoveNext) and fail loudly on a second consumer rather than
|
||||
// silently splitting the stream (see GWC-01). The distributor pump is the only intended
|
||||
// silently splitting the stream. The distributor pump is the only intended
|
||||
// caller; the alarm monitor and dashboard mirror attach to the distributor instead.
|
||||
if (Interlocked.CompareExchange(ref _eventsReaderClaimed, 1, 0) != 0)
|
||||
{
|
||||
@@ -519,7 +519,7 @@ public sealed class WorkerClient : IWorkerClient
|
||||
/// Routes a received envelope to its handler. Every branch dispatches synchronously and
|
||||
/// immediately — the event branch only stages the event for the dedicated writer — so a full
|
||||
/// event channel can never delay a command reply, heartbeat, fault, or shutdown ack behind an
|
||||
/// event backlog (GWC-04).
|
||||
/// event backlog.
|
||||
/// </summary>
|
||||
/// <param name="envelope">The envelope to dispatch.</param>
|
||||
private void DispatchEnvelope(WorkerEnvelope envelope)
|
||||
@@ -559,7 +559,7 @@ public sealed class WorkerClient : IWorkerClient
|
||||
/// succeeds unless the channel has been completed during shutdown — in which case the event is
|
||||
/// safely dropped because the client is closing. Backpressure and the sustained-overflow fault
|
||||
/// are applied by <see cref="EventWriteLoopAsync"/> against the bounded consumer channel,
|
||||
/// off the read loop (GWC-04).
|
||||
/// off the read loop.
|
||||
/// </summary>
|
||||
/// <param name="workerEvent">The event received from the worker.</param>
|
||||
private void StageWorkerEvent(WorkerEvent workerEvent)
|
||||
@@ -575,7 +575,7 @@ public sealed class WorkerClient : IWorkerClient
|
||||
/// <summary>
|
||||
/// Drains staged worker events and applies the bounded-channel backpressure (and
|
||||
/// sustained-overflow fault) on a dedicated task, so the timed <see cref="Channel"/> write
|
||||
/// never runs on the read loop (GWC-04). Mirrors <see cref="WriteLoopAsync"/> for events.
|
||||
/// never runs on the read loop. Mirrors <see cref="WriteLoopAsync"/> for events.
|
||||
/// </summary>
|
||||
private async Task EventWriteLoopAsync()
|
||||
{
|
||||
@@ -984,7 +984,7 @@ public sealed class WorkerClient : IWorkerClient
|
||||
GatewayVersion = typeof(GatewayContractInfo).Assembly.GetName().Version?.ToString() ?? GatewayVersionFallback,
|
||||
|
||||
// Convey the negotiated worker-frame maximum so the worker adopts it instead of a
|
||||
// hard-coded default (IPC-02). Sits above the public gRPC cap by the envelope reserve.
|
||||
// hard-coded default. Sits above the public gRPC cap by the envelope reserve.
|
||||
MaxFrameBytes = (uint)_connection.FrameOptions.MaxMessageBytes,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -15,6 +15,6 @@ public enum WorkerClientErrorCode
|
||||
|
||||
// The serialized command envelope exceeds the negotiated worker-frame maximum. Rejected at the
|
||||
// enqueue boundary so only the offending command fails (mapped to ResourceExhausted) instead of
|
||||
// the oversized frame reaching the write loop and faulting the whole session (IPC-03).
|
||||
// the oversized frame reaching the write loop and faulting the whole session.
|
||||
CommandTooLarge,
|
||||
}
|
||||
|
||||
@@ -15,7 +15,7 @@ public sealed class WorkerFrameProtocolOptions
|
||||
/// gRPC payload accepted at the public boundary always fits inside one worker frame once wrapped
|
||||
/// in a <c>WorkerEnvelope</c> (correlation id, timestamps, oneof framing). Without this headroom
|
||||
/// the pipe max equals the gRPC max and a maximally-sized accepted request faults the whole
|
||||
/// session on the outbound write (IPC-03). 64 KiB is far larger than the fixed envelope overhead.
|
||||
/// session on the outbound write. 64 KiB is far larger than the fixed envelope overhead.
|
||||
/// </summary>
|
||||
public const int EnvelopeOverheadReserveBytes = 64 * 1024;
|
||||
|
||||
|
||||
@@ -6,10 +6,10 @@
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Grpc.AspNetCore" Version="2.76.0" />
|
||||
<PackageReference Include="ZB.MOM.WW.Auth.Abstractions" Version="0.1.4" />
|
||||
<PackageReference Include="ZB.MOM.WW.Auth.Ldap" Version="0.1.4" />
|
||||
<PackageReference Include="ZB.MOM.WW.Auth.ApiKeys" Version="0.1.4" />
|
||||
<PackageReference Include="ZB.MOM.WW.Auth.AspNetCore" Version="0.1.4" />
|
||||
<PackageReference Include="ZB.MOM.WW.Auth.Abstractions" Version="0.1.5" />
|
||||
<PackageReference Include="ZB.MOM.WW.Auth.Ldap" Version="0.1.5" />
|
||||
<PackageReference Include="ZB.MOM.WW.Auth.ApiKeys" Version="0.1.5" />
|
||||
<PackageReference Include="ZB.MOM.WW.Auth.AspNetCore" Version="0.1.5" />
|
||||
<PackageReference Include="ZB.MOM.WW.Audit" Version="0.1.0" />
|
||||
<PackageReference Include="ZB.MOM.WW.Theme" Version="0.3.1" />
|
||||
<PackageReference Include="ZB.MOM.WW.Configuration" Version="0.1.0" />
|
||||
@@ -17,11 +17,17 @@
|
||||
<PackageReference Include="ZB.MOM.WW.Telemetry" Version="0.1.0" />
|
||||
<PackageReference Include="ZB.MOM.WW.Telemetry.Serilog" Version="0.1.0" />
|
||||
<PackageReference Include="ZB.MOM.WW.GalaxyRepository" Version="0.2.0" />
|
||||
<PackageReference Include="ZB.MOM.WW.Secrets" Version="0.2.0" />
|
||||
<PackageReference Include="ZB.MOM.WW.Secrets.Abstractions" Version="0.2.0" />
|
||||
<PackageReference Include="ZB.MOM.WW.Secrets.Ui" Version="0.2.0" />
|
||||
<PackageReference Include="Serilog.AspNetCore" Version="10.0.0" />
|
||||
<PackageReference Include="Serilog.Sinks.Console" Version="6.1.1" />
|
||||
<PackageReference Include="Serilog.Sinks.File" Version="7.0.0" />
|
||||
<PackageReference Include="Microsoft.AspNetCore.SignalR.Client" Version="10.0.0" />
|
||||
<PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.7" />
|
||||
<!-- Security pin: GHSA-2m69-gcr7-jv3q. Microsoft.Data.Sqlite pulls the vulnerable
|
||||
native 2.1.11; 2.1.12 patches it. Bumping Sqlite does not clear it. -->
|
||||
<PackageReference Include="SQLitePCLRaw.lib.e_sqlite3" Version="2.1.12" />
|
||||
<PackageReference Include="Microsoft.Data.SqlClient" Version="6.0.2" />
|
||||
<PackageReference Include="Polly.Core" Version="8.6.6" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -11,6 +11,12 @@
|
||||
]
|
||||
},
|
||||
"AllowedHosts": "*",
|
||||
"Secrets": {
|
||||
"SqlitePath": "mxgateway-secrets.db",
|
||||
"MasterKey": { "Source": "Environment", "EnvVarName": "ZB_SECRETS_MASTER_KEY" },
|
||||
"RunMigrationsOnStartup": true,
|
||||
"ResolveCacheTtl": "00:00:30"
|
||||
},
|
||||
"MxGateway": {
|
||||
"Authentication": {
|
||||
"Mode": "ApiKey",
|
||||
@@ -26,7 +32,7 @@
|
||||
"AllowInsecure": true,
|
||||
"SearchBase": "dc=zb,dc=local",
|
||||
"ServiceAccountDn": "cn=serviceaccount,dc=zb,dc=local",
|
||||
"ServiceAccountPassword": "serviceaccount123",
|
||||
"ServiceAccountPassword": "${secret:ldap/mxgateway/bind}",
|
||||
"UserNameAttribute": "cn",
|
||||
"DisplayNameAttribute": "cn",
|
||||
"GroupAttribute": "memberOf"
|
||||
|
||||
@@ -11,10 +11,19 @@ public sealed class GatewayOptionsTests
|
||||
[Fact]
|
||||
public void OptionsBinding_UsesDesignDefaults()
|
||||
{
|
||||
GatewayOptions options = BindOptions(new Dictionary<string, string?>());
|
||||
// The LDAP bind password now defaults to blank (sourced from ${secret:ldap/mxgateway/bind}
|
||||
// at runtime), so enabled-LDAP options fail validation until a value is supplied. Seed one
|
||||
// so binding validates; the blank design default itself is asserted below.
|
||||
GatewayOptions options = BindOptions(new Dictionary<string, string?>
|
||||
{
|
||||
["MxGateway:Ldap:ServiceAccountPassword"] = "test-bind-password",
|
||||
});
|
||||
|
||||
// The bind password is no longer a leaked plaintext literal; its design default is blank.
|
||||
Assert.Equal(string.Empty, new LdapOptions().ServiceAccountPassword);
|
||||
|
||||
Assert.Equal(AuthenticationMode.ApiKey, options.Authentication.Mode);
|
||||
// SEC-01: the default is derived from CommonApplicationData (C:\ProgramData on Windows,
|
||||
// The default is derived from CommonApplicationData (C:\ProgramData on Windows,
|
||||
// /usr/share on Unix) rather than a Windows literal, so assert against the same derivation
|
||||
// to stay platform-correct.
|
||||
Assert.Equal(
|
||||
@@ -35,7 +44,7 @@ public sealed class GatewayOptionsTests
|
||||
Assert.Equal(10, options.Worker.ShutdownTimeoutSeconds);
|
||||
Assert.Equal(5, options.Worker.HeartbeatIntervalSeconds);
|
||||
Assert.Equal(15, options.Worker.HeartbeatGraceSeconds);
|
||||
// 16 MiB public gRPC cap plus the 64 KiB worker-frame envelope reserve (IPC-03 headroom).
|
||||
// 16 MiB public gRPC cap plus the 64 KiB worker-frame envelope reserve for headroom.
|
||||
Assert.Equal((16 * 1024 * 1024) + (64 * 1024), options.Worker.MaxMessageBytes);
|
||||
|
||||
Assert.Equal(30, options.Sessions.DefaultCommandTimeoutSeconds);
|
||||
@@ -75,7 +84,9 @@ public sealed class GatewayOptionsTests
|
||||
["MxGateway:Sessions:DefaultLeaseSeconds"] = "900",
|
||||
["MxGateway:Events:QueueCapacity"] = "256",
|
||||
["MxGateway:Dashboard:Enabled"] = "false",
|
||||
["MxGateway:Protocol:MaxGrpcMessageBytes"] = "8388608"
|
||||
["MxGateway:Protocol:MaxGrpcMessageBytes"] = "8388608",
|
||||
// Blank-by-default bind password must be supplied so enabled LDAP validates on bind.
|
||||
["MxGateway:Ldap:ServiceAccountPassword"] = "test-bind-password"
|
||||
});
|
||||
|
||||
Assert.Equal(AuthenticationMode.Disabled, options.Authentication.Mode);
|
||||
@@ -118,7 +129,9 @@ public sealed class GatewayOptionsTests
|
||||
using ServiceProvider services = BuildServices(
|
||||
new Dictionary<string, string?>
|
||||
{
|
||||
["MxGateway:Authentication:PepperSecretName"] = "RawPepperSecretName"
|
||||
["MxGateway:Authentication:PepperSecretName"] = "RawPepperSecretName",
|
||||
// Blank-by-default bind password must be supplied so enabled LDAP validates on bind.
|
||||
["MxGateway:Ldap:ServiceAccountPassword"] = "test-bind-password"
|
||||
});
|
||||
|
||||
IGatewayConfigurationProvider provider = services.GetRequiredService<IGatewayConfigurationProvider>();
|
||||
|
||||
@@ -6,17 +6,27 @@ namespace ZB.MOM.WW.MxGateway.Tests.Configuration;
|
||||
|
||||
public sealed class GatewayOptionsValidatorTests
|
||||
{
|
||||
// A non-blank LDAP bind password for tests. The shipped design default is now blank
|
||||
// (string.Empty) so the secret-sourced value fails closed if unresolved; enabled-LDAP test
|
||||
// options must therefore supply an explicit password to pass the required-field check.
|
||||
private const string TestBindPassword = "test-bind-password";
|
||||
|
||||
// Constructs the minimal valid GatewayOptions by relying on each sub-option's
|
||||
// design-default values; those defaults are validated separately in GatewayOptionsTests.
|
||||
private static GatewayOptions ValidOptions() => new();
|
||||
// The one exception is the LDAP bind password (blank by design default), which we supply.
|
||||
private static GatewayOptions ValidOptions() => new() { Ldap = ValidLdapOptions() };
|
||||
|
||||
// Enabled LDAP options with all class defaults plus the non-blank bind password.
|
||||
private static LdapOptions ValidLdapOptions() => new() { ServiceAccountPassword = TestBindPassword };
|
||||
|
||||
// Returns enabled LDAP options that pass all checks except Port.
|
||||
// The class defaults already satisfy the blank-field checks; we only
|
||||
// override Enabled (must be true to exercise the port check) and Port.
|
||||
// The class defaults already satisfy the remaining blank-field checks; we override Enabled
|
||||
// (must be true to exercise the port check), Port, and the now-blank-by-default bind password.
|
||||
private static LdapOptions LdapOptionsWithPort(int port) => new()
|
||||
{
|
||||
Enabled = true,
|
||||
Port = port,
|
||||
ServiceAccountPassword = TestBindPassword,
|
||||
};
|
||||
|
||||
private static GatewayOptions CloneWithLdap(GatewayOptions source, LdapOptions ldap)
|
||||
@@ -550,10 +560,6 @@ public sealed class GatewayOptionsValidatorTests
|
||||
Assert.True(result.Succeeded);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// SEC-01: security-sensitive paths must be rooted (absolute)
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
private static GatewayOptions CloneWithAuthentication(GatewayOptions source, AuthenticationOptions authentication)
|
||||
=> new()
|
||||
{
|
||||
@@ -606,10 +612,6 @@ public sealed class GatewayOptionsValidatorTests
|
||||
f => f.Contains("MxGateway:Tls:SelfSignedCertPath") && f.Contains("rooted"));
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// SEC-04: DisableLogin production guard
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
private static GatewayOptions CloneWithDashboard(GatewayOptions source, DashboardOptions dashboard)
|
||||
=> new()
|
||||
{
|
||||
@@ -649,10 +651,6 @@ public sealed class GatewayOptionsValidatorTests
|
||||
Assert.True(result.Succeeded);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// SEC-06: LDAP plaintext transport production guard
|
||||
// -------------------------------------------------------------------------
|
||||
|
||||
/// <summary>Verifies plaintext LDAP transport (None) aborts startup in Production.</summary>
|
||||
[Fact]
|
||||
public void Validate_Fails_WhenLdapTransportNoneInProduction()
|
||||
@@ -679,14 +677,38 @@ public sealed class GatewayOptionsValidatorTests
|
||||
{
|
||||
GatewayOptions options = CloneWithLdap(
|
||||
ValidOptions(),
|
||||
new LdapOptions { Enabled = true, Transport = LdapTransport.Ldaps, AllowInsecure = false });
|
||||
new LdapOptions
|
||||
{
|
||||
Enabled = true,
|
||||
Transport = LdapTransport.Ldaps,
|
||||
AllowInsecure = false,
|
||||
ServiceAccountPassword = TestBindPassword,
|
||||
});
|
||||
ValidateOptionsResult result = new GatewayOptionsValidator(isProduction: true).Validate(null, options);
|
||||
Assert.True(result.Succeeded);
|
||||
}
|
||||
|
||||
// ---- SEC-11: MxGateway:Security validation ----
|
||||
/// <summary>
|
||||
/// Verifies enabled LDAP with a blank <see cref="LdapOptions.ServiceAccountPassword"/> fails
|
||||
/// validation. Locks in the fail-closed design default (blank) introduced when the bind password
|
||||
/// moved to the encrypted secrets store (<c>${secret:ldap/mxgateway/bind}</c>): a blanked/unresolved
|
||||
/// password must abort startup rather than silently binding with an empty credential.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Validate_Fails_WhenLdapEnabledAndServiceAccountPasswordBlank()
|
||||
{
|
||||
GatewayOptions options = CloneWithLdap(
|
||||
ValidOptions(),
|
||||
new LdapOptions { Enabled = true, ServiceAccountPassword = string.Empty });
|
||||
ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, options);
|
||||
Assert.True(result.Failed);
|
||||
Assert.Contains(
|
||||
result.Failures!,
|
||||
f => f.Contains("MxGateway:Ldap:ServiceAccountPassword is required when LDAP login is enabled."));
|
||||
}
|
||||
|
||||
private static GatewayOptions WithSecurity(SecurityOptions security) => new() { Security = security };
|
||||
private static GatewayOptions WithSecurity(SecurityOptions security)
|
||||
=> new() { Security = security, Ldap = ValidLdapOptions() };
|
||||
|
||||
/// <summary>Verifies the default security options pass validation.</summary>
|
||||
[Fact]
|
||||
@@ -791,7 +813,7 @@ public sealed class GatewayOptionsValidatorTests
|
||||
|
||||
/// <summary>
|
||||
/// Verifies the default worker-frame maximum keeps the required envelope-overhead reserve above
|
||||
/// the default public gRPC cap, so a stock configuration passes the IPC-03 headroom check.
|
||||
/// the default public gRPC cap, so a stock configuration passes the headroom check.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Validate_Succeeds_WhenWorkerFrameMaxHasEnvelopeHeadroom()
|
||||
@@ -803,7 +825,7 @@ public sealed class GatewayOptionsValidatorTests
|
||||
/// <summary>
|
||||
/// Verifies that a worker-frame maximum equal to the gRPC cap (zero headroom) fails validation:
|
||||
/// a maximally-sized accepted gRPC payload would not fit one worker frame once wrapped in a
|
||||
/// WorkerEnvelope, faulting the whole session on the outbound write (IPC-03).
|
||||
/// WorkerEnvelope, faulting the whole session on the outbound write.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void Validate_Fails_WhenWorkerFrameMaxEqualsGrpcMaxWithoutHeadroom()
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
using System.Security.Cryptography;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using ZB.MOM.WW.Secrets.Abstractions;
|
||||
using ZB.MOM.WW.Secrets.Configuration;
|
||||
using ZB.MOM.WW.Secrets.DependencyInjection;
|
||||
using ZB.MOM.WW.Secrets.Sqlite;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Exercises the exact pre-host <c>${secret:}</c> expansion flow that
|
||||
/// <see cref="ZB.MOM.WW.MxGateway.Server.GatewayApplication.CreateBuilder"/> runs before the host is
|
||||
/// built: a throwaway <c>AddZbSecrets</c> container migrates the SQLite store, then a
|
||||
/// <see cref="SecretReferenceExpander"/> rewrites configuration tokens in place. A seeded reference
|
||||
/// resolves to its plaintext; a reference to an absent secret is fail-closed
|
||||
/// (<see cref="SecretNotFoundException"/>).
|
||||
/// </summary>
|
||||
public sealed class PreHostSecretExpansionTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task ExpandConfiguration_SeededReference_ResolvesToPlaintext()
|
||||
{
|
||||
using var fixture = SecretsFixture.Create();
|
||||
const string plaintext = "s3cr3t-plaintext-value";
|
||||
await fixture.SeedAsync("test/value", plaintext);
|
||||
|
||||
IConfigurationRoot config = fixture.BuildConfig(("MxGateway:Ldap:ServiceAccountPassword", "${secret:test/value}"));
|
||||
|
||||
await new SecretReferenceExpander(fixture.Resolver).ExpandConfigurationAsync(config, default);
|
||||
|
||||
Assert.Equal(plaintext, config["MxGateway:Ldap:ServiceAccountPassword"]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// G-4: the shipped <c>appsettings.json</c> sources the LDAP bind password from
|
||||
/// <c>${secret:ldap/mxgateway/bind}</c>. Proves that exact reference at that exact key resolves to
|
||||
/// the seeded plaintext through the same pre-host expander the host runs.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public async Task ExpandConfiguration_LdapBindReference_ResolvesToSeededPassword()
|
||||
{
|
||||
using var fixture = SecretsFixture.Create();
|
||||
const string bindPassword = "seeded-bind-password";
|
||||
await fixture.SeedAsync("ldap/mxgateway/bind", bindPassword);
|
||||
|
||||
IConfigurationRoot config = fixture.BuildConfig(
|
||||
("MxGateway:Ldap:ServiceAccountPassword", "${secret:ldap/mxgateway/bind}"));
|
||||
|
||||
await new SecretReferenceExpander(fixture.Resolver).ExpandConfigurationAsync(config, default);
|
||||
|
||||
Assert.Equal(bindPassword, config["MxGateway:Ldap:ServiceAccountPassword"]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ExpandConfiguration_MissingReference_ThrowsFailClosed()
|
||||
{
|
||||
using var fixture = SecretsFixture.Create();
|
||||
|
||||
IConfigurationRoot config = fixture.BuildConfig(("MxGateway:Ldap:ServiceAccountPassword", "${secret:missing}"));
|
||||
|
||||
await Assert.ThrowsAsync<SecretNotFoundException>(
|
||||
() => new SecretReferenceExpander(fixture.Resolver).ExpandConfigurationAsync(config, default));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A self-contained secrets store: a unique temp SQLite path + a per-test master-key env var
|
||||
/// (so parallel tests never collide), wired through the real <c>AddZbSecrets</c> DI graph the
|
||||
/// host uses. Disposal removes the temp file and clears the env var.
|
||||
/// </summary>
|
||||
private sealed class SecretsFixture : IDisposable
|
||||
{
|
||||
private readonly ServiceProvider _provider;
|
||||
private readonly string _sqlitePath;
|
||||
private readonly string _envVarName;
|
||||
|
||||
private SecretsFixture(ServiceProvider provider, string sqlitePath, string envVarName)
|
||||
{
|
||||
_provider = provider;
|
||||
_sqlitePath = sqlitePath;
|
||||
_envVarName = envVarName;
|
||||
}
|
||||
|
||||
public ISecretResolver Resolver => _provider.GetRequiredService<ISecretResolver>();
|
||||
|
||||
public static SecretsFixture Create()
|
||||
{
|
||||
string unique = Guid.NewGuid().ToString("N");
|
||||
string sqlitePath = Path.Combine(Path.GetTempPath(), $"mxgw-secrets-{unique}.db");
|
||||
string envVarName = $"ZB_SECRETS_MASTER_KEY_TEST_{unique}";
|
||||
Environment.SetEnvironmentVariable(envVarName, Convert.ToBase64String(RandomNumberGenerator.GetBytes(32)));
|
||||
|
||||
IConfigurationRoot secretsConfig = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["Secrets:SqlitePath"] = sqlitePath,
|
||||
["Secrets:MasterKey:Source"] = "Environment",
|
||||
["Secrets:MasterKey:EnvVarName"] = envVarName,
|
||||
["Secrets:RunMigrationsOnStartup"] = "true",
|
||||
["Secrets:ResolveCacheTtl"] = "00:00:30",
|
||||
})
|
||||
.Build();
|
||||
|
||||
ServiceProvider provider = new ServiceCollection()
|
||||
.AddZbSecrets(secretsConfig, "Secrets")
|
||||
.BuildServiceProvider();
|
||||
|
||||
// Mirror the pre-host path: migrate the store explicitly before the first resolve.
|
||||
provider.GetRequiredService<SqliteSecretsStoreMigrator>().MigrateAsync(default).GetAwaiter().GetResult();
|
||||
|
||||
return new SecretsFixture(provider, sqlitePath, envVarName);
|
||||
}
|
||||
|
||||
// Seals a plaintext value under a fresh DEK and upserts it — the same store/cipher pair the
|
||||
// CLI 'secret set' verb uses.
|
||||
public async Task SeedAsync(string name, string plaintext)
|
||||
{
|
||||
var secretName = new SecretName(name);
|
||||
ISecretCipher cipher = _provider.GetRequiredService<ISecretCipher>();
|
||||
ISecretStore store = _provider.GetRequiredService<ISecretStore>();
|
||||
StoredSecret sealed_ = cipher.Encrypt(secretName, plaintext, SecretContentType.Text);
|
||||
await store.UpsertAsync(sealed_, default);
|
||||
}
|
||||
|
||||
public IConfigurationRoot BuildConfig(params (string Key, string Value)[] entries) =>
|
||||
new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(entries.ToDictionary(e => e.Key, e => (string?)e.Value))
|
||||
.Build();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_provider.Dispose();
|
||||
Environment.SetEnvironmentVariable(_envVarName, null);
|
||||
|
||||
// The SQLite store runs in WAL mode with connection pooling, so a pooled handle can
|
||||
// outlive _provider.Dispose() and keep the .db (plus its -wal/-shm sidecars) open. Clear
|
||||
// the pool first, then delete all three files so no temp artifact leaks between tests.
|
||||
Microsoft.Data.Sqlite.SqliteConnection.ClearAllPools();
|
||||
foreach (string path in new[] { _sqlitePath, _sqlitePath + "-wal", _sqlitePath + "-shm" })
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(path))
|
||||
{
|
||||
File.Delete(path);
|
||||
}
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
// Best-effort cleanup of the temp store; a locked file must not fail the test.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,7 +9,7 @@ namespace ZB.MOM.WW.MxGateway.Tests.Contracts;
|
||||
public sealed class ClientProtoInputTests
|
||||
{
|
||||
/// <summary>
|
||||
/// Guards the published client descriptor set against silent staleness (IPC-01). Every message
|
||||
/// Guards the published client descriptor set against silent staleness. Every message
|
||||
/// and field compiled into the in-process contract (which the build regenerates from the current
|
||||
/// <c>.proto</c> sources) must appear in the committed protoset. A missing symbol means the
|
||||
/// descriptor was not regenerated after a proto change; run
|
||||
|
||||
@@ -7,7 +7,7 @@ using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard;
|
||||
|
||||
/// <summary>
|
||||
/// SEC-11: the <c>POST /auth/login</c> fixed-window per-IP rate limiter. Exercised through the same
|
||||
/// Tests the <c>POST /auth/login</c> fixed-window per-IP rate limiter. Exercised through the same
|
||||
/// partition factory the production policy registers, so the test pins the real permit limit and
|
||||
/// per-IP partitioning without standing up an HTTP server.
|
||||
/// </summary>
|
||||
|
||||
+7
-4
@@ -203,7 +203,7 @@ public sealed class DashboardSessionAdminServiceTests
|
||||
/// <summary>
|
||||
/// Verifies that a successful close writes a canonical <c>dashboard-close-session</c>
|
||||
/// <see cref="AuditEvent"/> — actor, session-id target, and Success outcome — to the
|
||||
/// audit store, not only the operational log (SEC-12).
|
||||
/// audit store, not only the operational log.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
@@ -229,7 +229,7 @@ public sealed class DashboardSessionAdminServiceTests
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a successful kill writes a canonical <c>dashboard-kill-worker</c>
|
||||
/// <see cref="AuditEvent"/> to the audit store (SEC-12).
|
||||
/// <see cref="AuditEvent"/> to the audit store.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
@@ -253,7 +253,7 @@ public sealed class DashboardSessionAdminServiceTests
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that an unauthorized (viewer) close attempt still writes a <c>Denied</c>
|
||||
/// audit row, so rejected destructive attempts are durably recorded (SEC-12).
|
||||
/// audit row, so rejected destructive attempts are durably recorded.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
@@ -305,7 +305,10 @@ public sealed class DashboardSessionAdminServiceTests
|
||||
/// <summary>Gets the audit events written through this writer, in order.</summary>
|
||||
public IReadOnlyList<AuditEvent> Events => _events.ToArray();
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <summary>Records the given audit event for later inspection by the test.</summary>
|
||||
/// <param name="evt">The audit event to record.</param>
|
||||
/// <param name="ct">Token to cancel the asynchronous operation.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public Task WriteAsync(AuditEvent evt, CancellationToken ct = default)
|
||||
{
|
||||
_events.Enqueue(evt);
|
||||
|
||||
@@ -145,7 +145,7 @@ public sealed class HubTokenServiceTests
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The default token lifetime is the short (5-minute) window mandated by SEC-05, not the
|
||||
/// The default token lifetime is the short (5-minute) window, not the
|
||||
/// former 30-minute window. Pins the value so a regression that widens the exposure window
|
||||
/// of an irrevocable, query-string-carried token is caught in CI.
|
||||
/// </summary>
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
using System.Security.Claims;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.Builder;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using ZB.MOM.WW.MxGateway.Server;
|
||||
using ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway;
|
||||
|
||||
/// <summary>
|
||||
/// Verifies the shared Secrets UI is wired into the dashboard: the
|
||||
/// <c>AddSecretsAuthorization</c> policies are registered so the mounted
|
||||
/// <c>/admin/secrets</c> Blazor page (<c>[Authorize(Policy = "secrets:manage")]</c>) and its
|
||||
/// reveal action resolve, AND that the dashboard's real Administrator principal shape actually
|
||||
/// satisfies them (the claim-type-compatibility risk class that bit HistorianGateway). The
|
||||
/// browser reveal/redirect behaviour is verified separately on the box.
|
||||
/// </summary>
|
||||
public sealed class DashboardSecretsMountTests
|
||||
{
|
||||
private const string ManagePolicy = "secrets:manage";
|
||||
private const string RevealPolicy = "secrets:reveal";
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that Build registers the shared Secrets authorization policies
|
||||
/// (<c>secrets:manage</c> + <c>secrets:reveal</c>) added additively via
|
||||
/// <c>Configure<AuthorizationOptions>(o => o.AddSecretsAuthorization())</c>.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task Build_RegistersSecretsAuthorizationPolicies()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
|
||||
IAuthorizationPolicyProvider policyProvider =
|
||||
app.Services.GetRequiredService<IAuthorizationPolicyProvider>();
|
||||
|
||||
AuthorizationPolicy? managePolicy = await policyProvider.GetPolicyAsync(ManagePolicy);
|
||||
AuthorizationPolicy? revealPolicy = await policyProvider.GetPolicyAsync(RevealPolicy);
|
||||
|
||||
Assert.NotNull(managePolicy);
|
||||
Assert.NotNull(revealPolicy);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that the dashboard's Administrator principal — constructed exactly the way
|
||||
/// <see cref="DashboardAutoLoginAuthenticationHandler.CreatePrincipal(string?)"/> mints it
|
||||
/// (role claim on <c>ZbClaimTypes.Role</c>, carrying <c>Administrator</c>, authenticated) —
|
||||
/// satisfies BOTH secrets policies. This locks claim-type compatibility: if roles were ever
|
||||
/// issued on a divergent claim type, <c>RequireRole("Administrator")</c> would no longer match
|
||||
/// and this test would fail.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task AdministratorPrincipal_SatisfiesSecretsPolicies()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
IAuthorizationService authorization =
|
||||
app.Services.GetRequiredService<IAuthorizationService>();
|
||||
ClaimsPrincipal administrator = DashboardAutoLoginAuthenticationHandler.CreatePrincipal("multi-role");
|
||||
|
||||
AuthorizationResult manage = await authorization.AuthorizeAsync(administrator, resource: null, ManagePolicy);
|
||||
AuthorizationResult reveal = await authorization.AuthorizeAsync(administrator, resource: null, RevealPolicy);
|
||||
|
||||
Assert.True(manage.Succeeded);
|
||||
Assert.True(reveal.Succeeded);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that an anonymous, role-less principal is DENIED both secrets policies, proving the
|
||||
/// policies actually gate on the Administrator role rather than admitting everyone.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task AnonymousPrincipal_IsDeniedSecretsPolicies()
|
||||
{
|
||||
await using WebApplication app = GatewayApplication.Build([]);
|
||||
IAuthorizationService authorization =
|
||||
app.Services.GetRequiredService<IAuthorizationService>();
|
||||
ClaimsPrincipal anonymous = new(new ClaimsIdentity());
|
||||
|
||||
AuthorizationResult manage = await authorization.AuthorizeAsync(anonymous, resource: null, ManagePolicy);
|
||||
AuthorizationResult reveal = await authorization.AuthorizeAsync(anonymous, resource: null, RevealPolicy);
|
||||
|
||||
Assert.False(manage.Succeeded);
|
||||
Assert.False(reveal.Succeeded);
|
||||
}
|
||||
}
|
||||
@@ -39,7 +39,7 @@ public sealed class EventStreamServiceTests
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// TST-02 (owner-scoped attach): the API key that opened a session may attach its event
|
||||
/// Owner-scoped attach: the API key that opened a session may attach its event
|
||||
/// stream — the caller key equals the session owner, so streaming proceeds normally.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
@@ -64,7 +64,7 @@ public sealed class EventStreamServiceTests
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// TST-02 (owner-scoped attach, security control): a caller whose API key differs from
|
||||
/// Owner-scoped attach, security control: a caller whose API key differs from
|
||||
/// the key that opened the session is rejected with a <see cref="SessionManagerErrorCode.PermissionDenied"/>
|
||||
/// fault before any events are streamed — closing the reconnect/fan-out trust-boundary hole.
|
||||
/// </summary>
|
||||
|
||||
@@ -18,8 +18,9 @@ public sealed class MxAccessGrpcRequestValidatorTests
|
||||
|
||||
/// <summary>
|
||||
/// Verifies a DrainEvents request within the per-request ceiling passes validation, including the
|
||||
/// <c>max_events = 0</c> "worker default cap" sentinel (IPC-04).
|
||||
/// <c>max_events = 0</c> "worker default cap" sentinel.
|
||||
/// </summary>
|
||||
/// <param name="maxEvents">The requested drain-events ceiling to validate.</param>
|
||||
[Theory]
|
||||
[InlineData(0u)]
|
||||
[InlineData(1u)]
|
||||
@@ -32,7 +33,7 @@ public sealed class MxAccessGrpcRequestValidatorTests
|
||||
|
||||
/// <summary>
|
||||
/// Verifies a DrainEvents request above the per-request ceiling is rejected with InvalidArgument
|
||||
/// so one accepted request cannot pack an unbounded reply frame (IPC-04).
|
||||
/// so one accepted request cannot pack an unbounded reply frame.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void ValidateInvoke_RejectsDrainEvents_AboveCeiling()
|
||||
|
||||
+1
-1
@@ -108,7 +108,7 @@ public sealed class GatewaySessionDashboardMirrorTests
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// GWC-01 regression: with the internal dashboard mirror active, a second internal
|
||||
/// With the internal dashboard mirror active, a second internal
|
||||
/// subscriber (the alarm monitor's feed, attached via
|
||||
/// <see cref="GatewaySession.AttachInternalEventSubscriber"/>) receives EVERY event —
|
||||
/// including the alarm <c>Acknowledge</c> transition — rather than the two consumers
|
||||
|
||||
@@ -420,10 +420,16 @@ public sealed class SessionManagerTests
|
||||
[Fact]
|
||||
public async Task InvokeAsync_WhenWorkerFaulted_FailsFastWithBothStates()
|
||||
{
|
||||
// Use a deliberately large ready-wait timeout so fail-fast is unambiguous: a terminal
|
||||
// worker must surface immediately instead of burning it. The timing assertion below is
|
||||
// anchored to a small fraction of this timeout (not an absolute ~100ms wall-clock bound,
|
||||
// which flaked under CI load) so a regression that waited out the timeout is caught
|
||||
// while ordinary scheduling jitter never trips it.
|
||||
const int readyWaitTimeoutMs = 5000;
|
||||
FakeWorkerClient workerClient = new() { State = WorkerClientState.Faulted };
|
||||
SessionManager manager = CreateManager(
|
||||
new FakeSessionWorkerClientFactory(workerClient),
|
||||
options: CreateOptions(workerReadyWaitTimeoutMs: 500));
|
||||
options: CreateOptions(workerReadyWaitTimeoutMs: readyWaitTimeoutMs));
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
Assert.Equal(SessionState.Ready, session.State);
|
||||
|
||||
@@ -435,7 +441,9 @@ public sealed class SessionManagerTests
|
||||
CancellationToken.None));
|
||||
stopwatch.Stop();
|
||||
|
||||
Assert.True(stopwatch.ElapsedMilliseconds < 100, $"Expected immediate fail-fast but took {stopwatch.ElapsedMilliseconds}ms.");
|
||||
Assert.True(
|
||||
stopwatch.ElapsedMilliseconds < readyWaitTimeoutMs / 3,
|
||||
$"Expected fail-fast well under the {readyWaitTimeoutMs}ms ready-wait timeout but took {stopwatch.ElapsedMilliseconds}ms.");
|
||||
Assert.Equal(SessionManagerErrorCode.SessionNotReady, exception.ErrorCode);
|
||||
Assert.Contains("Session state is Ready", exception.Message);
|
||||
Assert.Contains("worker state is Faulted", exception.Message);
|
||||
@@ -487,15 +495,17 @@ public sealed class SessionManagerTests
|
||||
GatewaySession session = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None);
|
||||
Assert.Equal(SessionState.Ready, session.State);
|
||||
|
||||
Stopwatch stopwatch = Stopwatch.StartNew();
|
||||
// A zero timeout has no wait window to burn, so there is no wall-clock timing to assert
|
||||
// (the previous absolute ~100ms bound only measured host load and flaked under CI). The
|
||||
// error code, the byte-for-byte both-states message, and InvokeCount == 0 fully pin the
|
||||
// immediate fail-fast — a regression that started polling would still surface the same
|
||||
// outcome, not a timing difference worth a flaky assertion.
|
||||
SessionManagerException exception = await Assert.ThrowsAsync<SessionManagerException>(
|
||||
async () => await manager.InvokeAsync(
|
||||
session.SessionId,
|
||||
CreateCommand(MxCommandKind.Ping),
|
||||
CancellationToken.None));
|
||||
stopwatch.Stop();
|
||||
|
||||
Assert.True(stopwatch.ElapsedMilliseconds < 100, $"Expected immediate fail-fast but took {stopwatch.ElapsedMilliseconds}ms.");
|
||||
Assert.Equal(SessionManagerErrorCode.SessionNotReady, exception.ErrorCode);
|
||||
Assert.Contains("Session state is Ready", exception.Message);
|
||||
Assert.Contains("worker state is Handshaking", exception.Message);
|
||||
|
||||
+9
-2
@@ -15,6 +15,13 @@ public sealed class SessionWorkerClientFactoryFakeWorkerTests : IAsyncDisposable
|
||||
{
|
||||
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
// Anti-hang guard for the CreateAsync failure/timeout tests. It only exists to stop a genuine
|
||||
// hang from wedging the run, so it must be far larger than any in-test semantic timeout (the
|
||||
// factory startup timeout being exercised) — otherwise, under CI load, this WaitAsync net can
|
||||
// trip before the factory's own timeout propagates and the test observes .NET's generic
|
||||
// "The operation has timed out." instead of the asserted "did not complete startup".
|
||||
private static readonly TimeSpan HangGuardTimeout = TimeSpan.FromSeconds(30);
|
||||
|
||||
private readonly List<IWorkerTaskLauncher> _launchers = [];
|
||||
|
||||
/// <summary>
|
||||
@@ -79,7 +86,7 @@ public sealed class SessionWorkerClientFactoryFakeWorkerTests : IAsyncDisposable
|
||||
GatewaySession session = CreateSession();
|
||||
|
||||
WorkerClientException exception = await Assert.ThrowsAsync<WorkerClientException>(
|
||||
async () => await factory.CreateAsync(session, CancellationToken.None).WaitAsync(TestTimeout));
|
||||
async () => await factory.CreateAsync(session, CancellationToken.None).WaitAsync(HangGuardTimeout));
|
||||
|
||||
Assert.Equal(WorkerClientErrorCode.ProtocolViolation, exception.ErrorCode);
|
||||
Assert.True(launcher.Process.IsDisposed);
|
||||
@@ -100,7 +107,7 @@ public sealed class SessionWorkerClientFactoryFakeWorkerTests : IAsyncDisposable
|
||||
GatewaySession session = CreateSession(startupTimeout: TimeSpan.FromSeconds(1));
|
||||
|
||||
TimeoutException exception = await Assert.ThrowsAsync<TimeoutException>(
|
||||
async () => await factory.CreateAsync(session, CancellationToken.None).WaitAsync(TestTimeout));
|
||||
async () => await factory.CreateAsync(session, CancellationToken.None).WaitAsync(HangGuardTimeout));
|
||||
|
||||
Assert.Contains("did not complete startup", exception.Message);
|
||||
Assert.Equal(1, launcher.Process.KillCount);
|
||||
|
||||
@@ -13,7 +13,19 @@ namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Workers;
|
||||
/// </summary>
|
||||
public sealed class OrphanWorkerTerminatorTests
|
||||
{
|
||||
private const string WorkerExecutablePath = @"C:\app\src\ZB.MOM.WW.MxGateway.Worker\bin\x86\Release\ZB.MOM.WW.MxGateway.Worker.exe";
|
||||
// An OS-appropriate, already-normalized absolute path so Path.GetFullPath (used by the
|
||||
// terminator to resolve the configured path) is idempotent and the exact-match compare holds
|
||||
// on every platform. A hard-coded Windows literal (C:\...) is not rooted on Linux, so
|
||||
// Path.GetFullPath there prepends the cwd and the match fails — the terminator is a
|
||||
// Windows-only feature in production, but its matching logic must still be testable on the
|
||||
// Linux CI runner.
|
||||
private static readonly string WorkerExecutablePath = Path.GetFullPath(Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"MxGatewayOrphanTests",
|
||||
"bin",
|
||||
"x86",
|
||||
"Release",
|
||||
"ZB.MOM.WW.MxGateway.Worker.exe"));
|
||||
|
||||
/// <summary>Verifies that orphan worker processes matching the configured executable path are killed.</summary>
|
||||
[Fact]
|
||||
@@ -59,7 +71,10 @@ public sealed class OrphanWorkerTerminatorTests
|
||||
// not our worker and must be left alone.
|
||||
FakeProcessInspector inspector = new(
|
||||
[
|
||||
new RunningProcessInfo(301, @"C:\other\place\ZB.MOM.WW.MxGateway.Worker.exe"),
|
||||
new RunningProcessInfo(301, Path.GetFullPath(Path.Combine(
|
||||
Path.GetTempPath(),
|
||||
"MxGatewayOrphanTests-other",
|
||||
"ZB.MOM.WW.MxGateway.Worker.exe"))),
|
||||
]);
|
||||
OrphanWorkerTerminator terminator = CreateTerminator(inspector);
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ public sealed class WorkerClientTests
|
||||
/// <summary>
|
||||
/// Verifies that a command whose serialized envelope exceeds the negotiated worker-frame maximum
|
||||
/// fails only that command with <see cref="WorkerClientErrorCode.CommandTooLarge"/> at the enqueue
|
||||
/// boundary, leaving the client ready for subsequent commands (IPC-03). Without the pre-check the
|
||||
/// boundary, leaving the client ready for subsequent commands. Without the pre-check the
|
||||
/// oversized frame would reach the write loop and fault the whole session.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
@@ -170,7 +170,7 @@ public sealed class WorkerClientTests
|
||||
|
||||
/// <summary>
|
||||
/// The worker event channel is single-reader: a second <see cref="WorkerClient.ReadEventsAsync"/>
|
||||
/// enumerator must throw rather than silently split events between two consumers (GWC-01).
|
||||
/// enumerator must throw rather than silently split events between two consumers.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
@@ -221,7 +221,7 @@ public sealed class WorkerClientTests
|
||||
/// <summary>
|
||||
/// Verifies that a command reply arriving on the pipe after events is dispatched promptly even
|
||||
/// when the event channel is full and has no consumer — event enqueue is decoupled from the read
|
||||
/// loop, so a blocked event writer cannot delay a reply (GWC-04). The event full-mode timeout is
|
||||
/// loop, so a blocked event writer cannot delay a reply. The event full-mode timeout is
|
||||
/// set far above the command timeout: without the decoupling the read loop would block behind the
|
||||
/// full event channel and the in-flight InvokeAsync would hit CommandTimeout.
|
||||
/// </summary>
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user