Compare commits
79 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5776021970 | |||
| a11786cbea | |||
| aea2c74fdf | |||
| 8b147a32a2 | |||
| cc143b52cf | |||
| ad79238c84 | |||
| 986c6c0ce4 | |||
| 484028e721 | |||
| 1b11e89f29 | |||
| 5cdb3da8c0 | |||
| feadcf5740 | |||
| d6c5a11ae6 | |||
| f3827c71c9 | |||
| 1875426e4b | |||
| 780720165a | |||
| b0156c685e | |||
| f2090fe1b5 | |||
| 92aae28f07 | |||
| 4614080e24 | |||
| 8e30ec5be9 | |||
| 30548da71c | |||
| 914495a9f3 | |||
| fa364ce139 | |||
| f2ab9cc879 | |||
| cb820bb666 | |||
| c6650c156b | |||
| 193221773d | |||
| 06cd8b844c | |||
| cfbc99581b | |||
| ab41362a5c | |||
| b27035b3a6 | |||
| 9f0c3fd9fa | |||
| a71663a2a2 | |||
| 8257a0333e | |||
| 577a0db862 | |||
| 2ef7b93761 | |||
| b177636d7a | |||
| 282a0e70ad | |||
| 3619152b9f | |||
| ee68cc3f72 | |||
| 7200023bcc | |||
| 058d8d529f | |||
| aa07497cba | |||
| f1353dda86 | |||
| 3679e59eed | |||
| f59c34a544 | |||
| 20a3eefb1e | |||
| 22d5d52c14 | |||
| 93edb1b74d | |||
| ddfd41b06d | |||
| 7c5eb062e3 | |||
| b75f8b1704 | |||
| 6c784e3db7 | |||
| f4919ba97c | |||
| 8b3e3b3b45 | |||
| a8a8147a61 | |||
| f82dac1906 | |||
| 4b6f9077bf | |||
| 21903710e7 | |||
| f4b974eec1 | |||
| fb68bdb699 | |||
| 8ae0c2f3d3 | |||
| 753d070535 | |||
| db7b1db947 | |||
| 6b5c737b04 | |||
| 1605f542cc | |||
| 5417222f44 | |||
| aaeb86b132 | |||
| bc22792a36 | |||
| 462850afb7 | |||
| d1ae43d0d3 | |||
| f57a6ae5ff | |||
| d3ac52758c | |||
| 7da52b65b7 | |||
| 7b6dfba654 | |||
| 2b1efb5e50 | |||
| a390fe16e2 | |||
| c94c4d43c4 | |||
| df45cb4a37 |
+23
-6
@@ -4,16 +4,16 @@
|
||||
# .NET Framework 4.8 / x86 and need MXAccess COM installed, so they build ONLY on a Windows host.
|
||||
# They are out of scope for this Linux `portable` job — the `windows-x86` job below builds and
|
||||
# tests them on windev (10.100.0.48) over SSH (see scripts/ci/run-windev-ci.sh), and the nightly
|
||||
# `nightly-windev` job additionally runs the live-MXAccess smoke, which is scheduled-only so it
|
||||
# never gates a push.
|
||||
# `nightly-windev` job additionally runs the full solution build, gateway suite, and live-MXAccess
|
||||
# smoke, which are scheduled-only so they never gate a push.
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
schedule:
|
||||
# 06:00 UTC nightly: the `nightly-windev` job re-runs the x86 Worker build + full Worker.Tests on
|
||||
# windev, then the live-MXAccess smoke (MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1) and a full-slnx build.
|
||||
# 06:00 UTC nightly: x86 Worker build + Worker.Tests, full-slnx build, gateway tests, then the
|
||||
# live-MXAccess smoke (MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1), serialized on windev's host lock.
|
||||
# Push/PR runs skip it — the job is gated `if: github.event_name == 'schedule'`.
|
||||
- cron: '0 6 * * *'
|
||||
|
||||
@@ -23,7 +23,12 @@ jobs:
|
||||
# tests, the codegen/descriptor freshness guards, and the clients that build on Linux.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# fetch-depth: 0 (full history + tags): the CLI-46 client-version-freshness check below
|
||||
# diffs each client's source against its last publish tag (e.g. clients/go/vX.Y.Z), which
|
||||
# a shallow, tagless checkout cannot resolve.
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- uses: actions/setup-dotnet@v4
|
||||
with:
|
||||
@@ -77,6 +82,18 @@ jobs:
|
||||
shell: pwsh
|
||||
run: ./scripts/check-codegen.ps1
|
||||
|
||||
# CLI-46: catches a client whose source changed since its last publish
|
||||
# tag while the version constant still reads that tag's already-shipped
|
||||
# number — the drift the pack-clients.ps1/tag-go-module.ps1 collision
|
||||
# guards only catch at actual publish time. Needs tag history, hence
|
||||
# `fetch-depth: 0` on this job's checkout above.
|
||||
- name: Client version freshness
|
||||
shell: pwsh
|
||||
run: ./scripts/check-client-versions.ps1
|
||||
|
||||
- name: Code-reviews README consistency
|
||||
run: python code-reviews/regen-readme.py --check
|
||||
|
||||
- name: Gateway fake-worker tests
|
||||
run: dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj -c Release --no-build
|
||||
|
||||
@@ -165,7 +182,7 @@ jobs:
|
||||
run: ./scripts/ci/run-windev-ci.sh test
|
||||
|
||||
nightly-windev:
|
||||
# TST-25: scheduled deep run — full Worker.Tests + live-MXAccess smoke + full-slnx build on windev.
|
||||
# TST-35: scheduled deep run — Worker, full-slnx, gateway, and live-smoke tiers on windev.
|
||||
# Gated to the schedule event via `if:` (not an unsatisfiable `runs-on`), so push/PR runs mark it
|
||||
# skipped, not queued. Never gates a push. On failure it opens a Gitea issue so a red nightly is
|
||||
# visible even though nobody watches the Actions page.
|
||||
@@ -181,7 +198,7 @@ jobs:
|
||||
PUBLIC_SERVER_URL: https://gitea.dohertylan.com
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Full Worker.Tests + live-MXAccess smoke on windev
|
||||
- name: Worker + gateway tests + live-MXAccess smoke on windev
|
||||
env:
|
||||
# WINDEV_SSH_KEY is stored base64-encoded (single line) so Gitea's line-oriented secret
|
||||
# masker redacts it in the step env echo; run-windev-ci.sh decodes it. A raw multiline PEM
|
||||
|
||||
@@ -155,3 +155,4 @@ generated-scratch/
|
||||
|
||||
# Agent worktrees (subagent isolation) — never commit
|
||||
.claude/worktrees/
|
||||
.worktrees/
|
||||
|
||||
@@ -157,9 +157,9 @@ External analysis sources referenced by design docs:
|
||||
|
||||
Gateway gRPC clients authenticate with an API key in metadata: `authorization: Bearer mxgw_<key-id>_<secret>`. Keys are stored hashed (with a peppered SHA) in a gateway-owned SQLite DB (default `C:\ProgramData\MxGateway\gateway-auth.db`). Scopes (`session:open`, `session:close`, `invoke:read`, `invoke:write`, `invoke:secure`, `events:read`, `metadata:read`, `admin`) gate specific RPCs; missing → `Unauthenticated`, insufficient → `PermissionDenied`. The `apikey` subcommand on the server exe manages keys; see `src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/`.
|
||||
|
||||
Session event streaming is **owner-scoped**: the API key that opened a session is recorded on the session, and every `StreamEvents` attach/reattach is rejected with `PermissionDenied` unless the caller's key id matches the owner. Possessing the `event` scope and knowing a session id is not sufficient — this closes the reconnect/fan-out trust boundary (detach-grace and replay retention are on by default) so an `event`-scoped key cannot attach to another key's retained session.
|
||||
Session event streaming is **owner-scoped**: the API key that opened a session is recorded on the session, and every `StreamEvents` attach/reattach is rejected with `PermissionDenied` unless the caller's key id matches the owner. Possessing the `events:read` scope and knowing a session id is not sufficient — this closes the reconnect/fan-out trust boundary (detach-grace and replay retention are on by default) so an `events:read`-scoped key cannot attach to another key's retained session.
|
||||
|
||||
Dashboard auth is LDAP-backed (separate from the gRPC API-key model). `/login` binds against `MxGateway:Ldap` and maps the user's LDAP groups to `Admin` or `Viewer` via `MxGateway:Dashboard:GroupToRole`, then issues an HTTP-only secure cookie named `__Host-MxGatewayDashboard` when `Dashboard:RequireHttpsCookie` is true (default) and no `Dashboard:CookieName` override is set, else the plain `MxGatewayDashboard` (the `__Host-` prefix requires a Secure cookie). SignalR hubs at `/hubs/{snapshot,alarms,events}` accept either the cookie or a 5-minute bearer minted at `/hubs/token`. Dashboard event visibility is **tag-scoped per session** (`IDashboardSessionAcl`, gating both the events hub and the session-details page's in-process subscribe): an Administrator sees every session, while any other caller sees a session only when its tags — inherited from the owning API key's `apikey --dashboard-tags`, never from the client's request — intersect the tags their LDAP groups grant via `Dashboard:GroupToTag`; untagged sessions follow `Dashboard:UntaggedSessionVisibility` (default `AdminOnly`), and a principal with no tag claims (anonymous localhost included) is an empty-grant Viewer. `Dashboard:AllowAnonymousLocalhost` (default `true`) grants **read-only** loopback access — it satisfies the Viewer requirement but never the Admin-only requirement, so anonymous localhost can view the dashboard but not reach API-key CRUD or session Close/Kill (`Authentication:Mode=Disabled` is scoped the same way). `Dashboard:DisableLogin` (default `false`) auto-authenticates every dashboard request — including remote browsers — as `Dashboard:AutoLoginUser` (default `multi-role`) with both Admin and Viewer roles; dev/test only, never enable in production.
|
||||
Dashboard auth is LDAP-backed (separate from the gRPC API-key model). `/login` binds against `MxGateway:Ldap` and maps the user's LDAP groups to `Admin` or `Viewer` via `MxGateway:Dashboard:GroupToRole`, then issues an HTTP-only secure cookie named `__Host-MxGatewayDashboard` when `Dashboard:RequireHttpsCookie` is true (default) and no `Dashboard:CookieName` override is set, else the plain `MxGatewayDashboard` (the `__Host-` prefix requires a Secure cookie). SignalR hubs at `/hubs/{snapshot,alarms,events}` accept either the cookie or a 5-minute bearer minted at `/hubs/token`; `/hubs/alarms` and the `/alarms` page additionally require a **real authenticated dashboard role** (`MxGateway.Dashboard.Alarms`, a requirement type the environment-bypass handler does not handle), because the alarm feed is gateway-central with no per-session provenance and so cannot be tag-scoped — its metadata stays galaxy-wide for authenticated Viewers, but anonymous localhost and `Authentication:Mode=Disabled` do not receive it at all. Dashboard event visibility is **tag-scoped per session** (`IDashboardSessionAcl`, gating both the events hub and the session-details page's in-process subscribe): an Administrator sees every session, while any other caller sees a session only when its tags — inherited from the owning API key's `apikey --dashboard-tags`, never from the client's request — intersect the tags their LDAP groups grant via `Dashboard:GroupToTag`. That grant is **re-resolved from the caller's LDAP group claims against current options on every subscribe** (not read from the tag claims stamped at login, which would outlive a config-revoked grant for the cookie's 8-hour sliding life); login-stamped tag claims are the fallback only for a principal carrying no group claims — i.e. the hub-token principal, whose grant the mint already re-resolved. Untagged sessions follow `Dashboard:UntaggedSessionVisibility` (default `AdminOnly`), and a principal with neither claim kind (anonymous localhost included) is an empty-grant Viewer. `Dashboard:AllowAnonymousLocalhost` (default `true`) grants **read-only** loopback access — it satisfies the Viewer requirement but never the Admin-only requirement, so anonymous localhost can view the dashboard but not reach API-key CRUD, session Close/Kill, or the alarm feed (`Authentication:Mode=Disabled` is scoped the same way). `Dashboard:DisableLogin` (default `false`) auto-authenticates every dashboard request — including remote browsers — as `Dashboard:AutoLoginUser` (default `multi-role`) with both Admin and Viewer roles; dev/test only, never enable in production.
|
||||
|
||||
## Process / Platform Notes
|
||||
|
||||
|
||||
+9
-2
@@ -79,6 +79,12 @@ Add one entry per finding to the `## Findings` section of the module's
|
||||
|
||||
- **Finding ID** — `<Module>-NNN`, numbered sequentially within the module and
|
||||
never reused (e.g. `Worker-001`). IDs are permanent even after resolution.
|
||||
The `GWC`/`WRK`/`IPC`/`SEC`/`CLI`/`TST` prefixes are reserved for the
|
||||
`archreview/` domain registers (a separate, whole-repo review series with its
|
||||
own tracking documents) — never assign one of these prefixes to a
|
||||
`code-reviews/` finding, and any other work stream (perf passes, ad hoc
|
||||
remediation plans, etc.) that wants a stable per-item ID must mint its own
|
||||
distinct prefix rather than borrowing one of these six.
|
||||
- **Severity:**
|
||||
- **Critical** — data loss, security breach, crash/deadlock, or outage.
|
||||
- **High** — incorrect behaviour with significant impact; no safe workaround.
|
||||
@@ -127,8 +133,9 @@ finding carries an unrecognised Status value. The PowerShell wrapper
|
||||
`scripts/check-code-reviews-readme.ps1` runs that check and is the intended hook
|
||||
for CI or a pre-commit step.
|
||||
|
||||
> The repo's installed `python` is the real interpreter; the bare `python3`
|
||||
> alias resolves to the Windows Store stub and fails. Use `python`.
|
||||
> On Windows, the installed interpreter may be available only as `python`
|
||||
> because `python3` can resolve to the Windows Store stub. macOS and Linux
|
||||
> normally use `python3`; CI uses the configured `python` executable.
|
||||
|
||||
The per-module `findings.md` files are the source of truth; `README.md` is the
|
||||
aggregated index and must always agree with them — which the script guarantees.
|
||||
|
||||
@@ -79,6 +79,9 @@ Full design + implementation for each row lives in the linked domain doc under i
|
||||
| WRK-26 | Low | P1 | S | WRK-23 (soft); discharges IPC-29 | Done | Write-priority and overflow doc drift from the WRK-07 change |
|
||||
| WRK-27 | Low | — | S | — | Done | Alarm poll bypasses the watchdog's in-flight suppression (15 s vs 75 s) |
|
||||
| WRK-28 | Low | — | S | WRK-21 (same batch) | Done | 10,000 drain cap is a duplicated magic constant |
|
||||
| WRK-29 | — | — | — | — | Reserved (do not reuse) | Interim ID space possibly burned alongside WRK-31 by `e913dab`'s ad hoc numbering; not confirmed in the repo or its history, held back defensively |
|
||||
| WRK-30 | — | — | — | — | Reserved (do not reuse) | Interim ID space possibly burned alongside WRK-31 by `e913dab`'s ad hoc numbering; not confirmed in the repo or its history, held back defensively |
|
||||
| WRK-31 | — | — | — | — | Reserved (do not reuse) | Interim ID burned by `e913dab` (session-owned transport disposal), cited in code at `Ipc/WorkerPipeSession.cs:45,177` and `Ipc/WorkerPipeSessionTests.cs` outside the register; not a finding — registered per cycle-3 `20-worker.md` (~line 9) so it is never reassigned. Cycle-3 new findings start at WRK-32. |
|
||||
|
||||
### Contracts & IPC — [30-contracts-ipc.md](30-contracts-ipc.md)
|
||||
|
||||
@@ -92,7 +95,7 @@ Full design + implementation for each row lives in the linked domain doc under i
|
||||
| IPC-28 | Low | — | S | — | Done | docs/Grpc.md missing CommandTooLarge → ResourceExhausted mapping |
|
||||
| IPC-29 | Low | — | S | WRK-26 (discharged by) | Done (discharged by WRK-26) | WorkerFrameProtocol.md missing write-scheduling/sequencing section |
|
||||
| IPC-30 | Low | P0 | M | WRK-21 (same batch) | Done | Oversized event frame: keep session-fatal, make the death structured |
|
||||
| IPC-31 | Info | — | — | — | N/A | Gateway creation-time sequence stamping accepted; diagnostic-only, decision recorded |
|
||||
| IPC-31 | Info | — | — | — | Done (superseded by GWC-28) | Gateway creation-time sequence stamping — resolved when GWC-28 moved gateway stamping to write time; the two sides now implement the same semantics |
|
||||
| IPC-32 | Info | — | S | IPC-25 (folded in) | Done | check-codegen banner relabel 1/4…4/4 |
|
||||
|
||||
### Security & dashboard — [40-security-dashboard.md](40-security-dashboard.md)
|
||||
@@ -152,6 +155,14 @@ Sequence these together rather than piecemeal — several are one change set spa
|
||||
- ~~When CLI-38 lands, close old **CLI-08** with a pointer here.~~ Done 2026-08-07: CLI-38 landed and old CLI-08 is now `Done` in the first-cycle tracker, pointing at [CLI-38](50-clients.md#cli-38--align-netgojava-on-hresult--0-lands-prior-cli-08-cures-the-doc-drift---medium--p1).
|
||||
- ~~When WRK-26 lands, its doc section also discharges the WorkerFrameProtocol gap~~ Done 2026-08-07: WRK-26 landed and **IPC-29** is discharged, pointing at [WRK-26](20-worker.md#wrk-26--write-priority-and-overflow-doc-drift-from-the-wrk-07-change---low--p1). ~~When TST-25 lands, revisit old **TST-05** (scheduled live smoke) and **TST-24** (client wire tests), which it unlocks.~~ TST-05 revisited 2026-08-10: `Partially done` in the first-cycle tracker — the `nightly-windev` job closes the scheduled-cadence half, but the finding's coverage-audit half stays open (the live suite reaches all six late-added COM commands and none of the five control commands). TST-24 revisited in the same change and closed `Done`: Go/Rust/Java already had real-server wire tests, and the two genuine gaps (.NET, Python) now have them plus a CI step.
|
||||
|
||||
## 2026-08-25 documentation correction
|
||||
|
||||
The 2026-08-07 IPC-23 change-log entry's `docs/Grpc.md` deviation rationale is inaccurate:
|
||||
`DrainEvents` is not private worker IPC. It is a public `MxCommandKind` accepted by the
|
||||
`Invoke` RPC. Cycle-3 IPC-38 adds the missing public count-cap, byte-cap,
|
||||
drain-until-empty, and destructive-stream interaction contract to `docs/Grpc.md` while
|
||||
preserving the historical entry below.
|
||||
|
||||
## Change log
|
||||
|
||||
| Date | Change |
|
||||
|
||||
@@ -24,6 +24,7 @@ members, no positional records). The worker builds and tests only on the Windows
|
||||
| WRK-26 | Low | P1 | S | WRK-23 (soft — sequence prose); discharges IPC-29 | Done | Write-priority and overflow doc drift from the WRK-07 change |
|
||||
| WRK-27 | Low | — | S | — | Done | Alarm poll bypasses the watchdog's in-flight suppression (15 s vs 75 s) |
|
||||
| WRK-28 | Low | — | S | WRK-21 (land in the same commit cluster) | Done | 10,000 drain cap is a duplicated magic constant with a comment-only sync contract |
|
||||
| WRK-31 | — | — | — | — | Reserved (do not reuse) | Not a finding. `e913dab` (session-owned transport disposal) coined "WRK-31" ad hoc in code comments (`Ipc/WorkerPipeSession.cs:45,177`) and in `Ipc/WorkerPipeSessionTests.cs`, before the register assigned it. WRK-29/WRK-30 appear nowhere in the repo or its history but may have been burned by the same interim numbering, so they are held back defensively too — see the tracker's WRK-29/30/31 rows. Registered here per cycle-3 `archreview/2026-08-19/20-worker.md` (~line 9, "Finding-ID note"); that review's new findings start at **WRK-32** to guarantee no collision. |
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -231,6 +231,8 @@ The published-side collection (`CollectPublishedSymbols`, `:61-79`) grows matchi
|
||||
|
||||
**Disposition: N/A — no action.** `gateway.md:328-330` already declares sequence a per-sender diagnostic aid, never validated; both validators ignore it; the divergence is harmless and now explicitly documented by the IPC-29 doc section (which states the two sides' stamping points). If sequence ever becomes load-bearing, stamp in the gateway write loop (`WorkerClient.cs:390-397`) — recorded here so the next editor finds the decision.
|
||||
|
||||
**Resolution: Done — superseded by GWC-28 (2026-08-20, per cycle-3 verification).** GWC-28 moved the gateway-side stamp from creation to the point of writing: `WriteLoopAsync` now stamps `Sequence` immediately before `_writer.WriteAsync` (`Server/Workers/WorkerClient.cs:465-471`), and `CreateEnvelope` leaves it unset (`:1229-1236`); `gateway.md:425-431` was rewritten to "both sides stamp at the point of writing", closing the divergence this finding recorded. Regression test `WorkerClientTests.ConcurrentInvokesEmitStrictlyIncreasingSequencesOnTheWire` covers it. Per `archreview/2026-08-19/30-contracts-ipc.md` (IPC-31 row), tracker flipped from `N/A` to `Done (superseded by GWC-28)`. See cycle-3 **IPC-39** for the one interaction the change created with the IPC-03 pre-check.
|
||||
|
||||
---
|
||||
|
||||
## IPC-32 — `check-codegen.ps1` check labels are miscounted `Info` · `—` (rides IPC-25)
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
# MxAccessGateway — Architecture Review, Cycle 3
|
||||
|
||||
Date: 2026-08-19. Branch: `main` at `8b3e3b3` (clean; ~288 commits since the cycle-2 baseline `4f5371f`). Working tree: macOS, **static review only** — no builds, no test runs, no source changes, no remote hosts (one sanctioned read-only script, `regen-readme.py --check`, was executed by the testing reviewer).
|
||||
|
||||
## Scope and method
|
||||
|
||||
Third run of the review series ([cycle 1](../00-overall.md) 2026-07-08, [cycle 2](../2026-07-12/00-overall.md) 2026-07-12). Same shape: six domain reviews, each with three passes — (1) verify every cycle-2 finding's tracked status (`../2026-07-12/remediation/00-tracking.md`, plus the NEXT-xx candidate rows in `90-candidate-findings-next-cycle.md`) against code read at HEAD, (2) deep-review the remediation code itself and everything else landed in the period, (3) a fresh sweep. Every claim in the domain reports carries a `file:line` citation read at HEAD.
|
||||
|
||||
| Report | Domain |
|
||||
|---|---|
|
||||
| [10-gateway-core.md](10-gateway-core.md) | Gateway server: sessions, worker lifecycle, IPC client, gRPC streaming, alarm monitor, GalaxyRepository package adoption |
|
||||
| [20-worker.md](20-worker.md) | Worker process: STA pump, COM lifetime, pipe session, event queue, write-completion correlation, subtag failover |
|
||||
| [30-contracts-ipc.md](30-contracts-ipc.md) | Protos, frame protocol (both sides), codegen/descriptor pipeline, launch-environment contract |
|
||||
| [40-security-dashboard.md](40-security-dashboard.md) | API-key auth + failure limiter, LDAP dashboard, tag ACL, secrets store, SignalR hubs, redaction |
|
||||
| [50-clients.md](50-clients.md) | .NET, Go, Java, Python, Rust clients; publish pipeline; cross-client parity |
|
||||
| [60-testing-docs-gaps.md](60-testing-docs-gaps.md) | Test architecture, CI (incl. the windev tier), documentation currency, repo hygiene |
|
||||
|
||||
The period under review is the largest yet: the full cycle-2 remediation train (47 findings), the SEC-25 tag-ACL epic, the secrets-store adoption, the alarm truncation / `snapshot_status` rollout, the OnWriteComplete correlation feature, a worker perf wave, the 0.2.0/0.2.1 client publish, the windev CI tier, and the fca978d marker-sweep restoration.
|
||||
|
||||
## Overall verdict
|
||||
|
||||
**The cycle-2 remediation held completely.** All 47 cycle-2 findings tracked `Done` were verified present, sound, and tested at HEAD — **zero false `Done` claims, zero regressions in previously verified fixes, across all six domains.** Four gateway/IPC items carry a caveat in the precise sense of cycle 2 ("fix is correct for the filed defect; a residual failure mode one layer out survives"), each filed as a new Low (GWC-32/34/35, IPC-33). The worker, security, clients, and testing domains reported no caveats at all.
|
||||
|
||||
The new top line is **0 Critical, 0 High, 4 Medium, 25 Low, 5 Info** across 34 new findings — down from 0/1/14/28/4 (47) in cycle 2 and 1/12/54/39 in cycle 1. For the first time in the series, **none of the new findings is a correctness defect in the core data path that a client can trigger in default configuration.** The four Mediums are: one deployment-posture gap outside this repo's control (SEC-37, GLAuth has no TLS), one operator action outstanding from a tracked runbook (TST-32, runner-token reset), one version-hygiene repeat (CLI-46, constants equal to already-published artifacts — for the second cycle running), and one stale fixture doc with live-test fallout (TST-31, `glauth.md` credential table).
|
||||
|
||||
The residual risk profile is now dominated by **edges of new machinery and process residue**, not product defects: the recency-based write-completion correlation (WRK-32), the fail-open browse scope under a missing ambient identity (GWC-38), the un-tag-scoped alarms hub (SEC-40), the undeclared launch-env-var contract (IPC-36), and a client family that keeps shipping post-publish changes under the published version number (CLI-46).
|
||||
|
||||
## Prior-finding verification
|
||||
|
||||
| Domain | Cycle-2 Done claims | Verified | Caveats (filed as new) | Cycle-1 leftovers |
|
||||
|---|:-:|:-:|---|---|
|
||||
| Gateway core | 7 (GWC-24..30) | 7/7 | GWC-24→**GWC-32**/33, GWC-26→**GWC-35**, GWC-27→**GWC-34** | 13 re-confirmed open, none regressed; GWC-16 recommended Won't-fix (documented deferral) |
|
||||
| Worker | 8 (WRK-21..28) + NEXT-04/05 | 8/8 + 2/2 | none — WRK-21 fix exceeded its design (enqueue-time sizing) | WRK-01/04/06 re-verified intact |
|
||||
| Contracts & IPC | 8 (IPC-23..32) | 8/8 | IPC-23→**IPC-33** | IPC-08 substantially resolved by `7c1ea12` (recommend closing; residual = IPC-34); IPC-10/11/12 open; IPC-31 superseded by GWC-28 (flip to Resolved) |
|
||||
| Security & dashboard | 6 (SEC-31..36) | 6/6 | none; SEC-36 includes the executed operator rotation | SEC-19/21/26 resolved in the period; SEC-13 impact reduced; SEC-24 narrowed; 13/14/15(accepted)/16/17/18/23/27 open |
|
||||
| Clients | 11 (CLI-35..45) + NEXT-02 | 11/11 + 1/1 | none — including the halves prior cycles skipped (CLI-21 tag guard, CLI-02/42 docs, CLI-26 drift test) | CLI-05/08 partly, CLI-09/13/17/32 divergence table updated |
|
||||
| Testing, docs & gaps | 6 (TST-25..30) | 6/6 | TST-30 carries one tracked operator residual (→ **TST-32**) | TST-06 materially narrowed (tracker stale); TST-07/09/14 partially addressed; TST-17 row needs refresh; TST-05/10/18–22 open |
|
||||
|
||||
**Tracker housekeeping recommended:** close GWC-16 (Won't-fix, recorded), IPC-08 (resolved, residual → IPC-34), IPC-31 (superseded by GWC-28); refresh TST-06 and TST-17 rows; note that `e913dab` burned **WRK-31** (and possibly 29/30) as interim IDs and the 2026-08-15 perf pass stamped unregistered **GWC-3x** labels into code comments (GWC-31) — the register must stay the sole ID authority.
|
||||
|
||||
## Severity roll-up (new findings only)
|
||||
|
||||
| Domain | Critical | High | Medium | Low | Info | IDs |
|
||||
|---|:-:|:-:|:-:|:-:|:-:|---|
|
||||
| Gateway core | — | — | — | 6 | 2 | GWC-31..38 |
|
||||
| Worker | — | — | — | 3 | — | WRK-32..34 |
|
||||
| Contracts & IPC | — | — | — | 5 | 2 | IPC-33..39 |
|
||||
| Security & dashboard | — | — | 1 | 3 | 1 | SEC-37..41 |
|
||||
| Clients | — | — | 1 | 4 | — | CLI-46..50 |
|
||||
| Testing, docs & gaps | — | — | 2 | 4 | — | TST-31..36 |
|
||||
| **Total** | **0** | **0** | **4** | **25** | **5** | **34** |
|
||||
|
||||
(Cycle 2: 0 Critical, 1 High, 14 Medium, 28 Low, 4 Info. Cycle 1: 1 Critical, 12 High, 54 Medium, 39 Low.)
|
||||
|
||||
## Headline findings
|
||||
|
||||
### Medium
|
||||
|
||||
- **SEC-37 — the Production plaintext-LDAP hard-stop is unsatisfiable.** The shared GLAuth (`10.100.0.35:3893`) offers no TLS, so every host in the family that authenticates against it cannot run with `DOTNET_ENVIRONMENT=Production` — which is why the deployed service runs as `Staging`. Promotion of NEXT-08; the fix is family-wide (GLAuth TLS in `scadaproj/infra/glauth/`), not in this repo. ([40](40-security-dashboard.md))
|
||||
- **CLI-46 — version constants again equal the already-published 0.2.0/0.2.1** while post-publish changes shipped — including the fix for the published-broken Python `connect()` found by TST-24. Same shape as cycle-2's CLI-39; the publish-pipeline guards (registry-collision hard-stop) will now *block* the next publish rather than ship a duplicate, but the bump discipline still isn't happening at change time. ([50](50-clients.md))
|
||||
- **TST-31 — `glauth.md`'s user table contradicts the directory**, and the wrong credential propagated into the live-LDAP test surface (promotes NEXT-10). ([60](60-testing-docs-gaps.md))
|
||||
- **TST-32 — the Gitea runner registration token (world-readable ~5 months, proven live) still awaits its web-UI reset** — the one operator residual of the otherwise-complete TST-30. ([60](60-testing-docs-gaps.md))
|
||||
|
||||
### Low — items worth a second look
|
||||
|
||||
- **WRK-32 — write-completion correlation is recency-based per handle**: a late `OnWriteComplete` from an earlier write can be reported as the current write's outcome. Parity-clean otherwise (no synthesized events), but the docs should state the attribution rule. ([20](20-worker.md))
|
||||
- **GWC-38 — `GatewayBrowseScopeProvider` fails open** to the full Galaxy hierarchy when the ambient identity is absent, resting a scoping control on an undeclared interceptor-ordering invariant. ([10](10-gateway-core.md))
|
||||
- **SEC-40 / SEC-41 — tag ACL edges**: the session-less alarms hub broadcasts galaxy-wide alarm metadata to every Viewer including empty-grant and anonymous-localhost principals; revoked `GroupToTag` grants survive on cookie principals for up to the 8 h sliding life. ([40](40-security-dashboard.md))
|
||||
- **SEC-38 — limiter aggregate sidestep**: pre-filling the per-peer key-id cap (SEC-32) routes later failures to the fallback partition, which doesn't feed the cross-peer aggregate, restoring the `limit × addresses` guessing bound. ([40](40-security-dashboard.md))
|
||||
- **IPC-33 — DrainEvents byte budget is sized against the pipe max, not gRPC `MaxSendMessageSize`**; with a legal larger `Worker:MaxMessageBytes` the reply crosses the pipe and fails at the gRPC send with events already dequeued — the cycle-2 "no event loss" invariant violated one boundary later. ([30](30-contracts-ipc.md), [20](20-worker.md))
|
||||
- **IPC-34 — `WorkerCancel` fires only on the gateway's own timeout**; a client deadline/disconnect leaves the command on the STA queue (the IPC-08 residual). ([30](30-contracts-ipc.md))
|
||||
- **IPC-36 — six launch env-vars form an undeclared second gateway↔worker contract**, four new since baseline, duplicated as literals on both sides, two documented. ([30](30-contracts-ipc.md))
|
||||
- **GWC-37 — the subtag watch-list is resolved once per monitor lifecycle**; a Galaxy-Repository outage at startup empties the failover's coverage for the session's life. ([10](10-gateway-core.md))
|
||||
- **GWC-32 / GWC-34 — dispose races**: `WorkerClient.DisposeAsync` never enters a terminal state (the GWC-24 overflow fault can misfire); `EnsureDistributorCreated` recreates a pump-less distributor after teardown and a racing attach hangs silently. ([10](10-gateway-core.md))
|
||||
- **CLI-47/48/50 — alarm-feed CLI divergence**: `snapshot_status` rendering differs per language, the new renderer arms are untested in three CLIs (Go has no `stream-alarms` test at all), and the Python CLI's default `--max-messages 1` returns only the provider-status preamble so the README example fetches no alarm data. ([50](50-clients.md))
|
||||
- **TST-35 — the Windows-host run of the *gateway* (net10) suite is still manual-only**, and its failure class recurred three times in August; TST-34 — the code-reviews README check is wired into no CI job. ([60](60-testing-docs-gaps.md))
|
||||
|
||||
## Cross-cutting themes
|
||||
|
||||
1. **"One layer out" is now the only residual pattern in the core.** Every caveated fix this cycle (GWC-24/26/27 → GWC-32/34/35, IPC-23 → IPC-33) is the cycle-2 pattern: the invariant holds at the boundary that was filed and leaks at the next one (pipe max vs gRPC max; shutdown state vs dispose state; unacked→acked vs acked→unacked). The fix shape remains the same advice as last cycle: specify against the invariant, not the symptom — but the blast radius has shrunk from "session-fatal" to "misdiagnosed fault / silent hang in a race".
|
||||
2. **The tag-ACL epic is sound at its two designed seams and leaky at the one it didn't name.** Events hub and session-details subscribe are pinned with decision-table tests and live-LDAP coverage; the alarms hub (SEC-40) was never in scope, the cookie lifetime (SEC-41) wasn't either, and the browse scope (GWC-38) fails open. A one-page "which surfaces are tag-scoped and what principal each trusts" table would close all three or document them as accepted.
|
||||
3. **Version bumps are still not part of the change, for the second cycle** (CLI-39 → CLI-46). The pipeline guards now make this a blocked publish instead of a duplicate publish, which is the right failure — but the process fix is a "did any `clients/**` source change since the last publish tag" check at PR time, not at pack time.
|
||||
4. **Second contracts exist outside the proto.** The launch env-vars (IPC-36) and the GalaxyRepository wire-identity (IPC-37, still unguarded though now testable) are both gateway↔peer agreements with no schema, no test, and partial docs. The codegen guard is now excellent for everything *in* the protos; these are what's left.
|
||||
5. **ID discipline slipped in the period.** Interim WRK-31 and unregistered GWC-3x labels in code comments (GWC-31), plus stale tracker rows (TST-06/17, IPC-08/31, GWC-16) — cheap to fix, and the register is the only thing that makes cycle-over-cycle verification possible.
|
||||
6. **Documentation currency is high and the one big wound healed.** The fca978d sweep's collateral damage was systematically restored (816 lines judged, 17 restorations), every currency spot-check matched code, and the remaining doc findings (TST-31/33/36, GWC-36, IPC-38) are inherited debris rather than new drift.
|
||||
|
||||
## Prioritized roadmap
|
||||
|
||||
**P0 — small, do now**
|
||||
1. CLI-46: bump all five client versions past 0.2.0/0.2.1 before anything else touches `clients/`; add the PR-time "source changed since publish tag" check.
|
||||
2. TST-32: reset the Gitea runner registration token (web UI); TST-31: fix `glauth.md` and the propagated live-test credential.
|
||||
3. Tracker housekeeping: close GWC-16/IPC-08/IPC-31, refresh TST-06/TST-17, register WRK-31 and retire the GWC-3x comment labels (GWC-31).
|
||||
|
||||
**P1 — hardening, each a contained change**
|
||||
4. Tag-ACL completeness: SEC-40 (scope the alarms hub or document it as galaxy-wide-by-design), SEC-41 (re-resolve tag claims on restart / shorten validity), GWC-38 (fail closed when identity is absent and assert interceptor order).
|
||||
5. IPC-33 + IPC-39: budget DrainEvents against `min(pipe max, gRPC send max)`; fold the sequence-stamp window into the pre-check.
|
||||
6. IPC-34: forward `WorkerCancel` on caller cancellation, not only gateway timeout; IPC-35: surface handshake `WorkerFault`s structurally.
|
||||
7. WRK-32: document (or tighten) the recency-based completion attribution; WRK-33: give the discarded command-task observer its fault-observing continuation.
|
||||
8. GWC-32/34: terminal-state on dispose; refuse `EnsureDistributorCreated` after disposal. GWC-37: re-resolve the subtag watch-list on a timer or on Galaxy-Repository recovery.
|
||||
9. SEC-37: family-wide — add TLS to the shared GLAuth (`scadaproj/infra/glauth/`), then move the deployed service to `Production`.
|
||||
|
||||
**P2 — the rest**
|
||||
10. Client CLI parity batch (CLI-47/48/49/50), IPC-36 launch-env contract (single shared constants class + doc table), IPC-37 wire-identity test, TST-34/35 CI wiring, doc debris (TST-33/36, GWC-36, IPC-38, SEC-39 comment), alongside the unchanged cycle-1 backlog.
|
||||
|
||||
## What is demonstrably good
|
||||
|
||||
Everything cycle 2 asked to preserve is intact: STA pump and COM teardown discipline, write-time sequence stamping (now peek-stamp-commit on the worker, GWC-28 on the gateway), negotiated frame maxima with startup cross-validation, additive-only proto evolution with every generated tree regenerated in the same commit as the proto change, fail-closed auth with constant-time compare, owner-scoped `StreamEvents`, and the layered fake-worker test architecture.
|
||||
|
||||
New this period and worth protecting under change:
|
||||
|
||||
- **The windev CI tier** — per-push x86 worker build + Worker.Tests and a nightly live-MXAccess run, hardened by its own acceptance checks (base64-stored key, `MXGW_CI_LOCK_HELD` handoff, SHA shape validation, fail-red on unreachable host), and the full-suite testhost hang root-caused to a zero-buffer pipe and structurally fixed (windev baseline = macOS = 879).
|
||||
- **Byte-budgeted DrainEvents with enqueue-time sizing** — the cycle-2 headline defect fixed beyond its design; the IPC-33 residual is a configuration edge, not the default path.
|
||||
- **The failure limiter's composite partition + aggregate + probe admission** — the cycle-2 lockout-DoS is closed; SEC-38/39 are edge refinements.
|
||||
- **Fixtures-first client conformance** — ten shared command-reply fixtures consumed by all five suites, so HRESULT sign, category-verdict, scrub, and malformed-reply rules are pinned in bytes; plus real-wire tests in all five clients (which immediately found the published-broken Python `connect()`).
|
||||
- **Three-seam value redaction** (events mirror, alarms hub, `/browse`), clone-only, with the config doc telling the truth about all three.
|
||||
- **The codegen guard** — descriptor set, `Contracts/Generated`, Rust vendored protos in both directions, Go/Python bindings with pinned generators, Java aggregates via the java job's diff gate, and Check 4 now Windows-runnable.
|
||||
- **Per-correlation `CommandTooLarge` → `ResourceExhausted`** remains the model for fault isolation the residual IPC-33/34/35 items should copy.
|
||||
@@ -0,0 +1,144 @@
|
||||
# Gateway Server Core — Cycle-3 Review
|
||||
|
||||
- **Date:** 2026-08-19
|
||||
- **Commit:** `8b3e3b3` (`main`, clean)
|
||||
- **Baseline:** `4f5371f` (2026-07-12 cycle-2 review; ~288 commits since, ~99 of them touching `src/ZB.MOM.WW.MxGateway.Server`)
|
||||
- **Method:** static review on macOS; every cited file read at HEAD. No build, no test, no source change.
|
||||
- **Scope:** gateway server process — `Sessions/`, `Workers/`, `Grpc/` (`MxAccessGatewayService`, `EventStreamService`, mapper, validator), `Alarms/` (monitor, watch-list resolver, subtag-fallback gateway half), the session/alarm-feed plumbing the dashboard consumes (`DashboardLiveDataService` session use, `AlarmsHubPublisher` feed consumption), and the `ZB.MOM.WW.GalaxyRepository` package adoption (`AddZbGalaxyRepository`/`MapZbGalaxyRepository` wiring, `GatewayBrowseScopeProvider`, `DashboardGalaxySummaryProjector`). Excludes `Security/` internals and dashboard UI (security-dashboard reviewer), except the browse-scope provider assigned here.
|
||||
- **Prior reports:** `archreview/2026-07-12/10-gateway-core.md`; remediation status source `archreview/2026-07-12/remediation/00-tracking.md` (+ `10-gateway-core.md`); first-cycle backlog `archreview/remediation/00-tracking.md`.
|
||||
|
||||
## Prior-finding verification
|
||||
|
||||
### Cycle-2 findings (GWC-24..30, all tracked Done)
|
||||
|
||||
Every Done claim was verified against code read at HEAD. **All 7 hold; 3 are verified with caveats** — in each case the fix is present and sound for the reported defect, but a residual failure mode one layer out survives and is filed as a new finding.
|
||||
|
||||
| ID | Claimed | Verified? | Evidence (HEAD) | Notes |
|
||||
|----|---------|-----------|-----------------|-------|
|
||||
| GWC-24 | Done | ✅ holds, **with caveat** | `Workers/WorkerClient.cs:107-119` (staging channel bounded at `checked(2 * EventChannelCapacity)`, Wait+TryWrite idiom documented), `:646-676` (`StageWorkerEvent` faults `ProtocolViolation` on a rejected `TryWrite`, message names the bound, the depth, and the actionable fix), `:653-658`/`:350-359` (single `_eventQueueDepth` counter incremented at staging, decremented at consumer read — total undelivered depth), `:120-121` + `Metrics/GatewayMetrics.cs:524` (pull-model gauge registration), `:44-48` (field comment states bound + fault + gauge coverage); startup overflow guard `Configuration/GatewayOptionsValidator.cs:343-357` (`QueueCapacity ≤ int.MaxValue / 2` so the 2× derivation cannot overflow); regression tests `Tests/Gateway/Workers/WorkerClientTests.cs:781` (`StagingChannelOverflowFaultsWorkerWithoutWaitingForFullModeTimeout`), `:865` (`WorkerEventQueueDepthGaugeCountsStagedEvents`) | Implemented exactly per the remediation design, including the shutdown-drop guard (`IsTerminalState()`), and the design's extra validator guard. Caveat: the shutdown-drop guard does not cover `DisposeAsync`, which completes the channels **without** entering a terminal state — a dispose-race `TryWrite` failure is then misdiagnosed as the backpressure fault → **GWC-32**. A faulted-but-retained client also keeps contributing stale depth to the gauge until disposal → **GWC-33** (Info). |
|
||||
| GWC-25 | Done | ✅ holds | `Sessions/SessionEventDistributor.cs:519-535` (empty-ring branch: `oldestAvailableSequence = gap ? _highestSequenceSeen + 1 : 0`, with a full explanatory comment citing the finding), out-param contract updated `:430-440`; emitted verbatim `Grpc/EventStreamService.cs:139-145`, `:227-239`; proto comment amended `Contracts/Protos/mxaccess_gateway.proto:797-798` ("When nothing is retained … `highest observed + 1` … resume formula remains valid"); `docs/Sessions.md:262` (dedicated empty-ring paragraph); tests `Tests/Gateway/Sessions/SessionEventDistributorTests.cs:935` (`ResumeUsingSentinelFormulaAfterEmptyRingGapDeliversLiveEvents`) and `Tests/Gateway/GatewayEndToEndReconnectReplayTests.cs:246` (`ReconnectAfterFullAgeEvictionResumesWithSentinelFormula`) | Sound end to end. The `afterSequence == ulong.MaxValue` wrap guards in both branches (`:521`, `:540`) are preserved, so the old dead-stream input is also inert. Proto + docs + server + tests all landed as one story; the client halves (CLI-35/36, NEXT-02) are closed in the tracking record. |
|
||||
| GWC-26 | Done | ✅ holds, **with caveat** | `Alarms/GatewayAlarmMonitor.cs:246-258` (internal lease attached **before** `SubscribeAlarmsAsync` + first `ReconcileAsync`, with the window rationale in the comment), `:272-281` (drain the same lease after reconcile; order-safety noted), `ApplyReconcile` acked-delta repair `:674-684` (both-present, `existing.CurrentState != incoming.CurrentState && incoming == ActiveAcked` → `Acknowledge` feed transition); `ISessionManager.ReadAlarmEventsAsync` removed (grep: absent from `Sessions/ISessionManager.cs` and `SessionManager.cs`); repair-scope + at-least-once contract documented `:607-633`; regression tests `Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs` | Implemented per design, including the method-removal half and the "never synthesize MxEvents" boundary note (`:610-614`: repairs are `AlarmFeedMessage`s, never `MxEvent`s). Caveat: the state-delta repair is one-directional — only unacked→acked is repaired; an acked alarm silently replaced by a new **unacked** instance is absorbed without a feed transition → **GWC-35**. |
|
||||
| GWC-27 | Done | ✅ holds, **with caveat** | `Sessions/GatewaySession.cs:590-617` (`AttachInternalEventSubscriber` readiness gate under `_syncRoot` before `EnsureDistributorCreated`, same exception code/message shape as `AttachEventSubscriber:955-961`; the poison mechanism is spelled out in the comment); test `Tests/Gateway/Sessions/GatewaySessionTests.cs:683` (`AttachInternalEventSubscriberBeforeReadyThrowsAndDoesNotPoisonDistributor`) | The premature-attach poison path is closed. Caveat: the disposal-side twin remains — `EnsureDistributorCreated` recreates a never-started distributor after `DisposeAsync` nulls `_eventDistributor`, silently hanging a racing attach → **GWC-34**. |
|
||||
| GWC-28 | Done | ✅ holds | `Workers/WorkerClient.cs:463-472` (`WriteLoopAsync` stamps `envelope.Sequence = unchecked(++_nextSequence)` immediately before `_writer.WriteAsync`, single-consumer rationale in comment), `:49-51` (field is a plain `ulong` touched only by the write loop, no interlock), `CreateEnvelope:1225-1241` (sequence deliberately unset, comment cites GWC-28/WRK-04); test `Tests/Gateway/Workers/WorkerClientTests.cs:228` (`ConcurrentInvokesEmitStrictlyIncreasingSequencesOnTheWire`) | Exactly the WRK-04-mirror design. All outbound envelopes flow through `_outboundEnvelopes` (hello `:171`, commands `:251`, cancel `:849/:858`, shutdown `:383`), so coverage is total. Note the `InvokeAsync` size pre-check comment (`:236-241`) correctly explains why the computed size cannot be reused as the writer's `knownSize` post-GWC-28 — the interaction was thought through. |
|
||||
| GWC-29 | Done | ✅ holds | `Grpc/MxAccessGrpcMapper.cs:43-55` (new `MapCommand(MxCommand)` overload; the one required clone kept with the aliasing rationale), `:27-33` (request overload delegates); `Grpc/MxAccessGatewayService.cs:131-136` (`mapper.MapCommand(commandToInvoke)` — the request clone and `invokeRequest` local are gone; `TrackCommandReply` still reads `commandToInvoke` at `:147`, which is why the inner clone must stay and does) | Sound. `WorkerClient.CreateCommandEnvelope`'s no-aliasing invariant comment (`:1194-1198`) remains accurate. |
|
||||
| GWC-30 | Done | ✅ holds | `Workers/WorkerFrameReader.cs:22-24` (per-reader `_lengthPrefix` scratch buffer), `:46` (used per frame), `:11-16` (class remark documents the non-reentrancy contract) | As designed. (Unrelatedly, the *label* "GWC-30" has since been reused by an off-register perf pass for a different change — see **GWC-31**.) |
|
||||
|
||||
### Cycle-1 findings still open at cycle 2 — re-checked at HEAD
|
||||
|
||||
All were re-read at HEAD. **None regressed; none silently fixed; two materially changed shape** (GWC-16 is now an explicitly documented deferral; GWC-20's client-side half improved).
|
||||
|
||||
| ID | Cycle-2 status | HEAD status | Evidence |
|
||||
|----|----------------|-------------|----------|
|
||||
| GWC-05 | Open | **Still open** | `Sessions/SessionWorkerClientFactory.cs:175-185` — pipe created with explicit 128 KiB buffers (the zero-quota fix, `da84635`) but still no ACL and no `PipeOptions.CurrentUserOnly`. The local pipe-squatting exposure and the `gateway.md` pipe-security contract gap are unchanged. |
|
||||
| GWC-09 | Open | **Still open** | `Workers/WorkerProcessStartedProbe.cs:5-19` (no-op `HasExited` probe throwing `WorkerProcessLaunchException`); `Workers/WorkerProcessLauncher.cs:331-339` (`ShouldRetryStartupProbe` still excludes `WorkerProcessLaunchException`); `Configuration/WorkerOptions.cs:19-22` (both dead retry options remain). The retry pipeline still can never retry. |
|
||||
| GWC-10 | Open | **Still open** | `Workers/WorkerEnvelopeValidator.cs:15-39` — version/session/body only; no inbound sequence tracking anywhere in `WorkerClient`. (The outbound half was fixed by GWC-28; enforcement remains unimplemented on both sides, consistent with IPC-31's recorded decision.) |
|
||||
| GWC-11 | Open | **Still open** | `Sessions/GatewaySession.cs:23` (`_workerClient` not volatile); lock-free reads at `:308` (`WorkerProcessId`), `:313` (`WorkerClient`), `:1579` (`CloseAsync`), `:1732`, `:1765`, `:1877` (`DisposeAsync`). |
|
||||
| GWC-12 | Open | **Still open** | `Workers/WorkerClient.cs:419-426` — plain `if (_disposed) return; _disposed = true;`, no interlock. (Now compounds with new GWC-32: dispose also never enters a terminal state.) |
|
||||
| GWC-13 | Open | **Still open** | `Sessions/GatewaySession.cs:1956-2024` — 25 ms poll loop, routed through the injected `TimeProvider` (unchanged since cycle 2). |
|
||||
| GWC-16 | Open | **Still open — now a recorded deferral; recommend closing as Won't-fix** | `Grpc/MxAccessGatewayService.cs:111-115` — a `PERF(followup)` comment now documents the double session lookup as measured-negligible and deliberately not collapsed (the fix would duplicate `InvokeAsync`'s fault mapping at a second entry point). The rationale is sound; the tracker row should be flipped to `Won't fix` citing this comment rather than left `Not started`. |
|
||||
| GWC-17 | Open | **Still open** | `Sessions/SparseArrayExpander.cs:306` (`RpcException` factory below the gRPC layer), `:75` (GWC-03's cap rejection still flows through the same leak); invocation comment at `Sessions/GatewaySession.cs:1103-1111` still documents the layering leak explicitly. |
|
||||
| GWC-18 | Open | **Still open** | `Sessions/GatewaySession.cs:14` — no `IAsyncDisposable`; `DisposeAsync` at `:1787`. |
|
||||
| GWC-19 | Open | **Still open** | `Sessions/GatewaySession.cs:1730-1734` — `KillWorker(string)`; grep across `src/` + `clients/` finds zero callers (one doc-comment mention in `SessionManagerTests.cs:1433`). |
|
||||
| GWC-20 | Open | **Still open (client half improved)** | `Configuration/WorkerOptions.cs:52` — the config key is still `HeartbeatIntervalSeconds` ("interval … for worker heartbeats") while it is bound to the **check** interval at `Sessions/SessionWorkerClientFactory.cs:98`; the `WorkerClientOptions` field was renamed to `HeartbeatCheckInterval` (`Workers/WorkerClientOptions.cs:40`), which fixes the client-side naming but leaves the config-key conflation. The heartbeat loop still uses raw `Task.Delay` without `TimeProvider` (`Workers/WorkerClient.cs:534`). |
|
||||
| GWC-21 | Open | **Still open** | `Sessions/SessionWorkerClientFactory.cs:95-101` populates 4 of 6 `WorkerClientOptions`; `EventChannelFullModeTimeout` / `HeartbeatStuckCeiling` remain fixed defaults (`Workers/WorkerClientOptions.cs:13`, `:23`). Post-GWC-24 the staging bound is *derived* from `EventChannelCapacity` (which is configurable), so the exposure gap is narrower than at cycle 2 but the two knobs are still hard-coded. |
|
||||
| GWC-22 | Open | **Still open** | `Grpc/EventStreamService.cs:217` — `metrics.StreamDisconnected("Detached")` hardcoded in the `finally`; overflow/fault exits are not differentiated. |
|
||||
| GWC-23 | N/A | Unchanged | Decision stands; multi-subscriber mode remains config-gated and the validator comment survives. |
|
||||
|
||||
### NEXT rows touching this domain
|
||||
|
||||
- **NEXT-01** (macOS `sun_path` pipe names) — **verified resolved**: `Sessions/SessionManager.cs:532-536` builds `mxgw-{Environment.ProcessId}-{sessionUid}` with the length-budget rationale in the comment; `CLAUDE.md`/`gateway.md` describe the same format.
|
||||
- **NEXT-03** (reconcile/live duplicate broadcasts) — **verified resolved as designed (best-effort, positive-match-only)**: `Alarms/GatewayAlarmMonitor.cs:483-489` (`IsDuplicateOfCachedState`, timestamp+state marker), `:496-507` (`IsDuplicateOfReconcileClear`, tombstone consumed on match), `:645-664` (tombstones recorded per reconcile generation, cleared at `:649`); at-least-once contract restated at `:616-625`; race-driving regressions in `Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs:159`, `:218`. The one-generation tombstone lifetime and the unset-timestamp passthrough match the recorded design; consumers must (and are documented to) treat transitions idempotently.
|
||||
- NEXT-08 / NEXT-10 (GLAuth TLS posture, glauth.md table) are security/docs-domain rows, not promoted here.
|
||||
|
||||
### Also verified in passing (other-domain fixes landing in gateway-core files)
|
||||
|
||||
- **WRK-21/WRK-28/IPC-23 gateway half** — `Grpc/MxAccessGrpcRequestValidator.cs:74-88` rejects `DrainEvents.max_events` above the now-shared `GatewayContractInfo.MaxDrainEventsPerCommand` constant, with the byte-cap division of labor documented in the comment.
|
||||
- **Zero-quota pipe fix** (`da84635`) — `SessionWorkerClientFactory.cs:15-25`, `:175-185`: explicit 128 KiB in/out buffers with the windev-wedge rationale; this was the root cause of the historical full-suite testhost hang.
|
||||
- **Dead `ISessionManager.ReadEventsAsync` chain removed** (`fa9eb0c`) and `StreamEventsAsync` now consumes the subscriber channel directly (`Grpc/EventStreamService.cs:110-204`, `935f002`) — the WaitToReadAsync/TryRead loop preserves the fault-surface semantics (`:158-204`) including the worker-fault → `MarkFaulted` + metric path.
|
||||
- **Reply ownership transfer** (`3ff073d`) — `Grpc/MxAccessGrpcMapper.cs:62-85`: single-consumer argument for moving `reply.Reply` is documented and holds (one frame → one `PendingCommand` awaiter → one mapper call; `MergeDeniedInto` splices in place on the owned instance).
|
||||
- **GalaxyRepository adoption** — `GatewayApplication.cs:151-176`: scope provider registered *before* `AddZbGalaxyRepository` so the library's `TryAddSingleton` null default cannot win (order-sensitive, correctly commented); snapshot-path default seeded pre-bind with `GalaxyRepositoryOptionsValidator` + `ValidateOnStart` (SEC-33); `MapZbGalaxyRepository` at `:351`. `A2-galaxyrepository-adoption-handoff.md`'s caveats are all marked resolved and match the code. One hardening gap filed as **GWC-38**.
|
||||
- **Exception-total shutdown drain** (`95f8ba9`/`0bc13b5`) — `Sessions/SessionManager.cs:354-427`: bounded-parallel (4) close with an exception-total body, kill fallback on `CancellationToken.None`, and the deliberate token asymmetry against `CloseExpiredLeasesAsync` documented at length; `ISessionManager.ShutdownAsync`'s doc (`ISessionManager.cs:100-110`) states the bounded overrun honestly. Sweep selection stays sequential with the TOCTOU re-check inside `TryBeginCloseIfExpired` (`GatewaySession.cs:1671-1695`) — the cycle-2 close-gate invariants are preserved under the parallelization.
|
||||
|
||||
## New findings
|
||||
|
||||
IDs continue from the register. **Note on ID allocation:** the codebase already contains comment references to "GWC-31" (and a re-used "GWC-30") coined by an off-register remediation pass — those labels are *not* register entries (see GWC-31 below, which claims the ID and records the collision). Register IDs assigned here are authoritative.
|
||||
|
||||
### GWC-31 — The 2026-08-15 perf remediation stamped unregistered "GWC-3x" IDs into code comments, colliding with this register
|
||||
|
||||
- **Severity:** Low
|
||||
- **Category:** Documentation & comments
|
||||
- **Location:** `Workers/WorkerClient.cs:44` ("GWC-30" = pull-model queue-depth gauge), `:236` ("PERF(GWC-31)"), `:253` ("GWC-31: one pooled timer…"); `Metrics/GatewayMetrics.cs:46`, `:213`, `:284`, `:463`, `:524` ("GWC-30" = gauge pull model); `Tests/Gateway/Workers/WorkerClientTests.cs:163`, `:179` ("GWC-31" = WorkerCancel forwarding); analogous worker-side citations (`Worker/Ipc/WorkerFrameReader.cs`/`WorkerFrameWriter.cs`, per `docs/plans/2026-08-15-perf-review-remediation.md:544-548`)
|
||||
- **Description:** The 2026-08-15 performance review (whose findings live only in `docs/plans/2026-08-15-*.md`, not in `archreview/`) labeled its changes with GWC-series IDs that were never added to any register. "GWC-30" now means two different things — the archreview register says *frame-reader 4-byte prefix allocation*; the code comments say *pull-model worker queue-depth gauge* — and "GWC-31" exists only in comments. Finding IDs are the repo's cross-reference currency ("cite it in commits, branches, and PRs" — remediation tracker); a reader following a code comment to the register lands on the wrong finding, and future review cycles (this one included) must dodge already-burned numbers. REVIEW-PROCESS.md's "never reused" rule is violated in spirit.
|
||||
- **Recommendation:** Sweep the perf-pass citations to a distinct prefix (e.g. `PERF-nn` keyed to `docs/plans/2026-08-15-perf-review-remediation.md` task numbers) or append the perf findings to the register under fresh, unclaimed IDs and fix the comments to match. Add a line to REVIEW-PROCESS.md stating that GWC/WRK/IPC/SEC/CLI/TST prefixes are reserved for the archreview registers.
|
||||
|
||||
### GWC-32 — `WorkerClient.DisposeAsync` never enters a terminal state, so the GWC-24 staging-overflow fault can misfire during a dispose race and the "already terminal" shutdown-drop claim is not guaranteed
|
||||
|
||||
- **Severity:** Low
|
||||
- **Category:** Concurrency & thread safety (residual of GWC-24)
|
||||
- **Location:** `Workers/WorkerClient.cs:419-434` (`DisposeAsync` sets `_disposed`, cancels, completes all three channels — but `_state` is untouched), `:661-676` (`StageWorkerEvent` fallback: `TryWrite` false → `if (IsTerminalState()) return;` → otherwise `ProtocolViolation` fault), `:632-644` (XML doc: a completed staging channel is "safely dropped because the client is already terminal")
|
||||
- **Description:** `MarkClosed` and `SetFaulted` both flip `_state` to a terminal value *before* completing the channels, so the `IsTerminalState()` guard in `StageWorkerEvent` correctly classifies a completed-channel `TryWrite` miss as shutdown. `DisposeAsync` is the third channel-completing path and does not: a client disposed while still `Ready` keeps reporting `IsTerminalState() == false`. If the read loop is inside `DispatchEnvelope` when `DisposeAsync` completes `_eventStaging` (the `_stopCts.Cancel()` only interrupts the *next* `ReadAsync`), the in-flight `StageWorkerEvent` takes the overflow arm: a spurious `ProtocolViolation` session fault with the misleading "consumer is draining slower than the worker produces" message, a `worker-event-staging` overflow metric, a redundant `KillOwnedProcess`, and a `WorkerStopped("ProtocolViolation")` metric — all during an intentional teardown. Reachability is narrow: in the normal teardown order (`CloseAsync`/`Kill` → `DisposeAsync`) the state is already terminal; the exposed path is disposing a still-Ready client, which `SessionManager.OpenSessionAsync`'s failure arm does (`Sessions/SessionManager.cs:133-140` — `MarkFaulted` faults the *session*, not the worker client) and `SessionWorkerClientFactory`'s catch avoids only because it calls `Kill` first (`SessionWorkerClientFactory.cs:120-132`). Impact is corrupted diagnostics/metrics, not data loss — but the fault fires on exactly the code path an operator investigates after a failed open, pointing them at backpressure that never happened. It also makes the `StageWorkerEvent` XML doc's "already terminal" claim false on one of the three completion paths.
|
||||
- **Recommendation:** Treat disposal as terminal in the guard — `if (_disposed || IsTerminalState()) return;` — or have `DisposeAsync` transition `_state` to `Closed` under `_syncRoot` before completing the channels (which would also give GWC-12's double-dispose interlock a natural home). Update the `StageWorkerEvent` doc to name all three completion paths.
|
||||
|
||||
### GWC-33 — A faulted-but-retained worker client keeps contributing its stale undelivered-event depth to the pull-model gauge until session disposal
|
||||
|
||||
- **Severity:** Info
|
||||
- **Category:** Performance & resource management (observability)
|
||||
- **Location:** `Workers/WorkerClient.cs:120-121` (gauge source registered at construction), `:426-429` (unregistered only in `DisposeAsync`), `:1011-1039` (`SetFaulted` completes channels but leaves staged+queued events uncounted-down); retention window: `Configuration/SessionOptions.cs` `FaultedGraceSeconds` + `Sessions/SessionManager.cs:283-289` (faulted sessions swept only when reapable)
|
||||
- **Description:** The GWC-24 gauge counts staged + queued events and is decremented only on consumer read. When a client faults, its channels are completed with the fault and never drained, so `_eventQueueDepth` freezes at whatever was undelivered. The registration is removed only in `DisposeAsync`; with a positive `FaultedGraceSeconds` (deliberately configurable so operators can observe faults before the reap) the dead client's frozen depth inflates the aggregate `worker event queue depth` gauge for the whole grace window. With the default `FaultedGraceSeconds = 0` the window is one sweep interval, so the default posture is fine — recorded so the next metrics pass knows it was seen.
|
||||
- **Recommendation:** Optional: dispose the gauge registration inside `SetFaulted`/`MarkClosed` (the comment at `:427-428` already articulates the principle: "the sum must not keep counting a client that is going away").
|
||||
|
||||
### GWC-34 — `EnsureDistributorCreated` recreates a never-started distributor after `DisposeAsync`; an attach that races a session teardown registers on a pump-less distributor and hangs silently
|
||||
|
||||
- **Severity:** Low
|
||||
- **Category:** Concurrency & thread safety (residual of GWC-27 — the disposal-side twin)
|
||||
- **Location:** `Sessions/GatewaySession.cs:1865-1875` (`DisposeAsync` nulls `_eventDistributor`; `_eventDistributorStarted` is never reset — `:41`, `:554-559`), `:535-563` (`EnsureDistributorCreated`: `_eventDistributor is null` → constructs a fresh distributor; `startNow` is false because the started flag is already latched), `:946-998` (`AttachEventSubscriber`: readiness gate under `_syncRoot`, then the distributor calls run *outside* the lock)
|
||||
- **Description:** GWC-27 closed the premature-attach poison; the mirror-image hole remains open on the teardown side. An `AttachEventSubscriber` (or `AttachEventSubscriberWithReplay` / `AttachInternalEventSubscriber`) that passes its readiness gate while the session is still `Ready`, then loses the CPU until `CloseAsync` + `DisposeAsync` complete, reaches `EnsureDistributorCreated` after `_eventDistributor` was nulled: a **new** distributor is constructed, `startNow` comes back `false` (the latch is one-way), the subscriber registers on a distributor whose pump will never start and whose channel will never be completed — `WaitToReadAsync` in `EventStreamService.cs:167` then blocks until the client's own deadline/cancel. No error, no log, no fault; the replacement distributor is also never disposed. Contrast the adjacent race that *is* handled: if disposal instead wins between `EnsureDistributorCreated` returning the **old** distributor and `Register`, the old distributor's disposed check throws `ObjectDisposedException` loudly and the attach rolls back (`RegisterSubscriber`, `SessionEventDistributor.cs:361-386`). Reachability is small (the attach must stall across an entire multi-second close + dispose between two adjacent statements), but the failure mode is the worst kind — silent, per-stream, and invisible to metrics (the backlog registration reports 0 forever). This is the same "one distributor per session, never reset" lifecycle assumption GWC-27 documented, violated from the other end.
|
||||
- **Recommendation:** Make the latch symmetric with the null-out: in `DisposeAsync`, set a `_eventDistributorDisposed` flag (or reuse a session-terminal check) under `_syncRoot` when nulling `_eventDistributor`, and have `EnsureDistributorCreated` throw `SessionManagerException(SessionNotReady)` instead of re-creating when it is set. One flag, one throw — the attach paths' existing catch/rollback already handles the exception.
|
||||
|
||||
### GWC-35 — `ApplyReconcile`'s state-delta repair is one-directional: an acked alarm replaced by a new unacked instance (or any acked→active regression) is absorbed silently by the snapshot replace
|
||||
|
||||
- **Severity:** Low
|
||||
- **Category:** Correctness & logic (alarm feed completeness; residual of GWC-26)
|
||||
- **Location:** `Alarms/GatewayAlarmMonitor.cs:666-691` (`ApplyReconcile` both-present loop: broadcasts only when `incoming.Value.CurrentState == AlarmConditionState.ActiveAcked`), snapshot replace at `:687-691`
|
||||
- **Description:** The GWC-26 repair covers three legs — missing→present (`Raise`), present→missing (`Clear`), and unacked→acked (`Acknowledge`). The fourth leg is absent: an alarm the cache holds as `ActiveAcked` that the worker snapshot now reports as `Active` (the acked instance cleared and re-raised as a new unacked instance while the live `Clear`+`Raise` transitions were lost — e.g. the internal alarm lease overflowed and disconnected, or worker-side event loss) is absorbed by the `_alarms` replace with no feed transition. `CurrentAlarms`/`QueryActiveAlarms` are correct (they read the replaced cache), but live `StreamAlarms`/dashboard subscribers keep rendering the alarm as acknowledged until its next genuine transition — for a re-raised alarm, precisely the state an operator must not misread. Reachability is much lower than the pre-GWC-26 window (a lease overflow restarts the whole monitor lifecycle, which re-snapshots every subscriber), so this is a completeness gap in the defense-in-depth rather than a live bug.
|
||||
- **Recommendation:** Generalize the both-present delta to any `CurrentState` change: broadcast `Acknowledge` for `→ ActiveAcked` and `Raise` (with the incoming snapshot's new instance identity) for `→ Active`. The existing `TransitionFromSnapshot` already carries the timestamps the NEXT-03 dedup needs to suppress the live duplicates.
|
||||
|
||||
### GWC-36 — `EventSubscriberReplayAttachment.OldestAvailableSequence` doc still describes the pre-GWC-25 empty-ring semantics
|
||||
|
||||
- **Severity:** Info
|
||||
- **Category:** Documentation & comments
|
||||
- **Location:** `Sessions/EventSubscriberReplayAttachment.cs:26-30` ("The oldest worker sequence still retained and replayable; `0` when nothing is retained.")
|
||||
- **Description:** GWC-25 changed the empty-ring-with-gap value to `highest observed + 1`; the distributor's out-param doc (`SessionEventDistributor.cs:430-440`), the proto, and `docs/Sessions.md` were all updated, but this record's XML doc was missed. "`0` when nothing is retained" is now exactly the value the fix exists to avoid — a reader implementing against this doc re-introduces the dead-stream formula.
|
||||
- **Recommendation:** Mirror the distributor's out-param wording (retained-oldest, or `highest observed + 1` when the ring is empty; `0` only when `Gap` is false).
|
||||
|
||||
### GWC-37 — The alarm subtag watch-list is resolved once per monitor lifecycle: a Galaxy-Repository outage at startup silently empties the subtag failover's coverage for the session's entire lifetime
|
||||
|
||||
- **Severity:** Low
|
||||
- **Category:** Error handling & resilience (alarm subtag fallback)
|
||||
- **Location:** `Alarms/GatewayAlarmMonitor.cs:330-357` (`SubscribeAlarmsAsync` resolves the watch list once and sends it in the `SubscribeAlarms` command); `Alarms/AlarmWatchListResolver.cs:54-83` (GR discovery failure → warn and continue with a config-only, typically empty set), `:141-145` (zero-target warning); no re-resolve anywhere — `ApplyProviderModeChangeAsync` (`GatewayAlarmMonitor.cs:513-566`) triggers only a reconcile on failover
|
||||
- **Description:** The subtag fallback's coverage is fixed at `SubscribeAlarms` time: the resolved `AlarmSubtagTarget` list travels once to the worker and is never refreshed while the monitor session lives. Discovery failure is deliberately non-fatal (`ResolveAsync` catches, warns, and returns whatever config contributes) — reasonable for the resolver, but the monitor then subscribes with an empty watch list and runs indefinitely. In `Auto` mode the alarmmgr primary works fine, so nothing looks wrong until the moment a failover to subtag fires — at which point the degraded provider covers zero alarms, the feed goes quiet, and the only evidence is a single `LogWarning` emitted possibly weeks earlier at startup. The same staleness applies to Galaxy objects added after monitor start (a documented design property), but the outage-at-startup case differs in kind: the operator configured discovery, it silently produced nothing, and the safety net the feature exists to provide is empty. The monitor's supervisor restarts (and thus re-resolves) only when the session faults — a healthy alarmmgr session never does.
|
||||
- **Recommendation:** Re-resolve and re-subscribe on the failover edge (`ApplyProviderModeChangeAsync` when `toMode == Subtag`), or treat "GR discovery failed AND the effective mode can need the watch list (Auto/ForceSubtag) AND zero targets resolved" as a lifecycle failure so the supervisor's backoff loop retries resolution instead of latching the empty set. At minimum, surface the empty-watch-list condition on the provider status (`Degraded`/`Reason`) rather than only in a startup log line.
|
||||
|
||||
### GWC-38 — `GatewayBrowseScopeProvider` fails open to the full Galaxy hierarchy when the ambient identity is absent, resting the browse-scoping control on an undeclared interceptor-ordering invariant
|
||||
|
||||
- **Severity:** Low
|
||||
- **Category:** Security (defense-in-depth; GalaxyRepository adoption)
|
||||
- **Location:** `Security/Authorization/GatewayBrowseScopeProvider.cs:14-25` (`identityAccessor.Current?.EffectiveConstraints ?? ApiKeyConstraints.Empty` → empty `BrowseSubtrees` → library treats as "no scoping"); registration-order dependency documented at `GatewayApplication.cs:151-154`
|
||||
- **Description:** The per-key browse-subtree scoping — the one piece of authorization mxaccessgw adds on top of the shared GalaxyRepository package — resolves to *unscoped* (full hierarchy) whenever `IGatewayRequestIdentityAccessor.Current` is null. The code's own comment says this is "safe only because the global interceptor has already authenticated and authorized the request first. Do not reorder the interceptor without revisiting this." That is a correct description of today's wiring and an honest warning, but it makes the control's fail state *open*: any future path that reaches the library's galaxy service without the interceptor populating the ambient identity (a new endpoint mapping, an interceptor scoped per-service instead of globally, a library upgrade changing pipeline order) silently grants every scoped key the full hierarchy, with nothing failing and nothing logged. Scoping is a constraint-narrowing control, so its absence is invisible in normal operation. (Whether identity-absent is legitimately reachable under `Authentication:Mode=Disabled` is the security reviewer's domain; this finding is about the fail-open shape itself.)
|
||||
- **Recommendation:** Distinguish "authenticated with no BrowseSubtrees constraint" (legitimately unscoped) from "no ambient identity" (invariant broken): when `Current` is null *and* API-key authentication is enabled, throw `RpcException(PermissionDenied)` (or log-and-deny) instead of returning unscoped. One branch converts a silent authorization bypass into a loud misconfiguration signal.
|
||||
|
||||
## Severity roll-up (new findings)
|
||||
|
||||
| Severity | Count | IDs |
|
||||
|----------|:-:|-----|
|
||||
| Critical | 0 | — |
|
||||
| High | 0 | — |
|
||||
| Medium | 0 | — |
|
||||
| Low | 6 | GWC-31, GWC-32, GWC-34, GWC-35, GWC-37, GWC-38 |
|
||||
| Info | 2 | GWC-33, GWC-36 |
|
||||
|
||||
Prior-finding verification: 7/7 cycle-2 Done claims hold (3 with caveats filed as GWC-32/34/35); 13 still-open cycle-1 findings re-confirmed open with none regressed; GWC-16 recommended for closure as a recorded Won't-fix.
|
||||
|
||||
## Still good / preserve under change
|
||||
|
||||
The cycle-2 remediation wave and the subsequent perf pass landed without breaking the core invariants, and several of them got structurally *stronger*. The backpressure topology is now bounded end to end again — worker queue → pipe → bounded staging (2×) → bounded consumer channel → bounded per-subscriber channels — with a loud fail-fast fault at every seam and a single observable depth gauge across the first two gateway stages; the GWC-24 implementation even added the startup overflow guard the design didn't ask for. The replay→live handoff argument is intact after the GWC-25 change, and the empty-ring sentinel arithmetic is documented in four places that all agree (distributor, proto, docs/Sessions.md, e2e test). The GWC-28 write-loop stamping is the textbook fix, and the `InvokeAsync` size-pre-check comment proves the interaction with it was reasoned about rather than stumbled past. The alarm monitor's attach-before-subscribe plus the NEXT-03 positive-match dedup give the alarm feed a defensible, *stated* delivery contract (at-least-once, idempotent-state), and the reconcile repairs are correctly quarantined to the feed layer — the "never synthesize events" rule for `MxEvent`s is respected everywhere, with the ReplayGap sentinel still the sole, documented control signal. Session state remains single-lock disciplined through the new detach-grace/faulted-grace/close-gate arms; the parallelized sweep and shutdown preserved the TOCTOU re-check and made the shutdown body exception-total with an honestly documented token asymmetry. The GalaxyRepository adoption is clean consumer wiring: scope-provider-before-package registration order is commented, the snapshot path is validated at startup, and the host-side projector is a pure function memoizable on the cache sequence. Comment quality across the new code is exceptional — most of this cycle's findings were *found via* comments that honestly documented their own residual assumptions ("safe only because…", "already terminal") — preserve that habit; it is what keeps one-layer-out defects findable. `GatewaySession` has grown again (2,173 lines) and the long-standing split-out recommendation stands.
|
||||
@@ -0,0 +1,108 @@
|
||||
# Worker Process — Architecture Review (cycle 3)
|
||||
|
||||
## Scope & method
|
||||
|
||||
Date 2026-08-19 · commit `8b3e3b3` (`main`, clean) · macOS static review — the worker targets .NET Framework 4.8 x86 and was **not built or tested** here; every claim cites file:line evidence read directly from the working tree at HEAD. Scope: `src/ZB.MOM.WW.MxGateway.Worker` (STA runtime/pump, `MxAccessSession`/`MxAccessStaSession` COM lifetime, `Ipc/` pipe session + frame reader/writer, event queue and DrainEvents, value/write-completion caches, alarm consumers incl. the subtag failover, watchdog/heartbeat, program lifecycle/exit codes) and `src/ZB.MOM.WW.MxGateway.Worker.Tests`. This is a verification pass over the 2026-07-12 review (`archreview/2026-07-12/20-worker.md`, remediation IDs WRK-21..28 in `archreview/2026-07-12/remediation/20-worker.md` / `00-tracking.md`, plus NEXT-04/NEXT-05 from `remediation/90-candidate-findings-next-cycle.md`), a re-check of the still-open cycle-1 findings, and a deep review of everything landed since `4f5371f` (~50 worker commits, 53 files, +9,014/−427).
|
||||
|
||||
Changed-in-scope since `4f5371f`: the WRK-21/28/23/IPC-30 drain cluster (`33ba612`, `6bc3f9b`), the WRK-22/24/25/27/IPC-26 write-seam batch (`8df35cd`), the OnWriteComplete correlation feature (`fc23a65`..`b0e65d4`), the perf wave (pooled frame buffers `f56798a`, reverse tag index `afec56d`, message-driven completion waits `dc9424d`, signal-driven event drain `f4b065b`, exact-format timestamp parse `94fdc18`/`25cbe5c`, enqueue-time frame sizing `58d97ad`, value-cache borrowing `9871d47`, class-transition flush `aac7957`, drain-pass unparking `9130994`), the NEXT-04/NEXT-05 abandonment-observation batch (`84dbf20`, `3ef56be`, `b8b7b69`), session-owned transport disposal (`e913dab`, labeled WRK-31 in code), the alarm-side truncation/failover hardening (`f3e1de5`, `b5ea6bb`, `693a78d`, `b9fb0dd`), and the WorkerPipeSessionTests harness rework (`7da52b6`, `462850a`, `aaeb86b`, `753d070`).
|
||||
|
||||
**Finding-ID note:** the tracker instruction says new IDs continue at WRK-29, but `e913dab` already consumed **WRK-31** as an interim ID (cited in code at `Ipc/WorkerPipeSession.cs:45,177` and in `Ipc/WorkerPipeSessionTests.cs`); WRK-29/WRK-30 appear nowhere in the repo or its history but may have been burned in the same interim numbering. To guarantee no reuse, this review's new findings start at **WRK-32**.
|
||||
|
||||
## Executive summary
|
||||
|
||||
- All eight cycle-2 Done claims (WRK-21..28) are present, sound, and tested at HEAD; none is a false Done and none carries a functional caveat. The WRK-21 fix went beyond the design: the drain sizes events at enqueue time (`58d97ad`) so the byte-budget walk no longer holds the queue lock the STA needs, and `6bc3f9b` made the budget monotonic at the reserve boundary so drain-until-empty terminates even at the 1024-byte negotiated floor.
|
||||
- NEXT-04 (fault-observing continuations on abandoned/tombstoned frames) and NEXT-05 (lazy tombstone purge as a documented decision) are both verified — the observer discipline is applied uniformly in the writer (`ObserveAbandonedFault`/`ObserveFault`), and `e913dab`/`3ef56be` extended it to the abandoned net48 pipe read and overrun background tasks, with a GC-forcing `UnobservedTaskException` regression test.
|
||||
- The two riskiest perf changes checked out. The value-cache borrow (`9871d47`) is safe because the post-publish hook runs only after `Enqueue` and every consumer of the borrowed instances is serialize-only (verified through `MxAccessSession.SucceededRead` and the write-completion cache, which deliberately still clones); the class-transition flush (`aac7957`) and drain-pass unparking (`9130994`) preserve the "written and flushed before completion" contract, are lost-wakeup-free (a walked-away lock acquisition is detached, drains, and releases — never dropped), and cannot double-release the semaphore.
|
||||
- One cycle-1 finding was fixed incidentally: **WRK-13** (unpooled writer buffers) is resolved by `f56798a` — the writer now rents one combined prefix+payload buffer from `ArrayPool` (`Ipc/WorkerFrameWriter.cs:675-687`). The old tracker should be updated. WRK-18's stale-doc half is also discharged (the design doc now describes the implemented fail-fast and honestly names the generic exit code); its dedicated-exit-code half stays open.
|
||||
- The remaining cycle-1 opens (WRK-02/03/05/08/09/10/14/16/17/18/19 and the WRK-02/03 slice of WRK-20) are all still present. WRK-05 is materially narrowed: in auto/failover mode `FailoverAlarmConsumer.RunPrimary` absorbs primary COM failures (`MxAccess/FailoverAlarmConsumer.cs:284-291`), but the default alarmmgr-only path is still one-poll-exception-fatal (`MxAccess/MxAccessStaSession.cs:392-405`).
|
||||
- Fresh sweep found no Medium or higher. Three new Lows: recency-based OnWriteComplete correlation can attribute a stale completion from an earlier write to the current write's reply (WRK-32); the discarded command-task observer is the one discarded task in the worker without a fault-observing continuation (WRK-33); the StaHung fault writes the in-flight correlation id into `WorkerFault.CommandMethod` (WRK-34, pre-existing since 2026-04-26).
|
||||
|
||||
## Prior-finding verification — cycle 2 (all tracked Done) and NEXT items
|
||||
|
||||
| ID | Claimed status | Verified? | Evidence (file:line at HEAD) | Notes |
|
||||
|----|----------------|-----------|------------------------------|-------|
|
||||
| WRK-21 (= IPC-23) | Done | **Yes** | Byte-budgeted `Drain(uint, int)` dequeues only events known to fit, leaves a non-fitting head queued, reports `OversizedHeadSequence` (`MxAccess/MxAccessEventQueue.cs:389-429`); per-event size memoized at enqueue outside the drain's lock hold (`:248-281`, `:502-509`, slack constants `:73-81`); reply builder consumes it with truncation `DiagnosticMessage` (`Ipc/WorkerPipeSession.cs:978-1020`); budget = `max(MaxMessageBytes − 64 KiB, MaxMessageBytes/2)` so it stays monotonic at the reserve boundary (`:1034-1039`, `6bc3f9b`); shared result type is net48-safe (`MxAccess/WorkerEventDrainResult.cs`). Backstops: control-reply write catches `MessageTooLarge` and answers with a small `InvalidRequest` reply (`Ipc/WorkerPipeSession.cs:847-864`), STA command replies get the same catch (`:1136-1147`), and the fallback write itself is size-guarded so a degenerate negotiated max cannot make the backstop fatal (`:876-900`). Tests: `WorkerPipeSessionTests.DrainEvents_ByteHeavyQueue_ReplyIsBoundedAndSessionSurvives` (:571), `DrainEvents_RepeatedCalls_RecoverAllEventsWithoutLoss` (:630), `ControlReplyTooLarge_WritesErrorReplyInsteadOfDying` (:770), `CommandReplyTooLarge_WritesErrorReplyInsteadOfFaulting` (:815), plus `MxAccessEventQueueTests` byte-budget cases. | Sound, and stronger than designed (no sizing walk under the STA-shared lock). No diagnostics command is session-fatal on size anymore. |
|
||||
| WRK-22 (= IPC-26) | Done | **Yes** | `PendingFrame.Claimed` interlock documented and mutated only under `_gate` (`Ipc/WorkerFrameWriter.cs:54-62`); cancelled waiter tombstones only unclaimed frames (`:169-183`, `:308-338`); `DequeueNext` skips cancelled frames and claims atomically in the same critical section (`:584-618`); contract documented in the method doc (`:105-112`) and `docs/WorkerFrameProtocol.md:203-211`. Tests `WorkerFrameProtocolTests.WriteAsync_CancelledWhileWaitingForLock_FrameIsNeverWritten` (:695), `WriteAsync_CancelledEventFrame_DoesNotTrailShutdownAck` (:800). | Sound. The claimed-mid-write residual window is documented as by-design. |
|
||||
| WRK-23 | Done | **Yes** | Peek-stamp-commit: candidate stamped before the size checks (sequence participates in `CalculateSize`), counter committed only once the frame is certain to be written, so a per-frame rejection burns no number (`Ipc/WorkerFrameWriter.cs:648-666`; comment `:72-76`). | Wire sequence stays contiguous across rejections; the gap-free concurrent-sequence test's guarantee is now genuinely true. |
|
||||
| WRK-24 | Done | **Yes** | `MinNegotiableFrameBytes = 1024` matching the gateway validation floor, rejected at the handshake with `InvalidConfiguration` (`Ipc/WorkerFrameProtocolOptions.cs:30`, `:144-150`); range documented in the property doc and `docs/WorkerFrameProtocol.md`. | Also closes the degenerate config that could have made WRK-21's fallback swallow reachable (`Ipc/WorkerPipeSession.cs:866-875` documents that chain). |
|
||||
| WRK-25 | Done | **Yes** | `WriteBatchAsync` enqueues the whole batch under one `_gate` acquisition, drains under one lock, awaits every completion so no per-frame rejection faults unobserved, rethrows the first failure in batch order (`Ipc/WorkerFrameWriter.cs:236-306`); the event drain loop submits drained batches through it (`Ipc/WorkerPipeSession.cs:615-630`). Tests `WriteBatchAsync_FlushesOnceAndPreservesOrder` (:839), `WriteBatchAsync_ControlFrameQueuedDuringBatch_JumpsRemainingEvents` (:876), `WorkerPipeSessionTests.EventBurst_DrainLoopCoalescesFlushes` (deflaked `50322ba`). | The WRK-12 coalescing now engages on the exact path it was built for: an uninterrupted N-event burst pays one flush. `aac7957` later refined the mixed-class case (see deep review). |
|
||||
| WRK-26 (discharges IPC-29) | Done | **Yes** | `docs/MxAccessWorkerInstanceDesign.md:717-751` — Outbound Queues rewritten to the two-class scheduler, explicitly records the accepted collapse of the five-level order and why, and rewrites the overflow paragraph to the implemented fail-fast (naming the still-generic `UnexpectedFailure` exit code honestly — WRK-18's exit-code half remains tracked); `docs/WorkerFrameProtocol.md:79+` "Write Scheduling And Sequencing" covers priority classes, write-time stamping, per-frame vs stream-failure semantics, flush coalescing, and cancellation. | No stale five-level prose remains. |
|
||||
| WRK-27 | Done | **Yes** | `volatile bool staAlarmPollInProgress` flipped on the STA exactly around `PollOnce` (`MxAccess/MxAccessStaSession.cs:64-70`, `:352-365`); surfaced via `WorkerRuntimeHeartbeatSnapshot.StaCallInProgress` (`MxAccess/WorkerRuntimeHeartbeatSnapshot.cs`, ctor default keeps net48-safe plain ctor); watchdog suppression honors it alongside the in-flight correlation id up to the ceiling (`Ipc/WorkerPipeSession.cs:1359-1370`). Test `WorkerPipeSessionTests.Watchdog_StaCallInProgress_SuppressedUntilCeiling` (:1260). | A slow-but-healthy poll now gets the 75 s ceiling, not the 15 s grace; a poll blocking past the ceiling still faults, per the ceiling's contract. |
|
||||
| WRK-28 | Done | **Yes** | `GatewayContractInfo.MaxDrainEventsPerCommand = 10_000` in Contracts (`src/ZB.MOM.WW.MxGateway.Contracts/GatewayContractInfo.cs:33`), consumed by the gateway validator (`src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGrpcRequestValidator.cs:82-86`) and the worker clamp (`Ipc/WorkerPipeSession.cs:992-994`). | Single shared constant; the desync failure mode is removed, not merely detected. |
|
||||
| NEXT-04 | Resolved 2026-08-10 | **Yes** | Unconditional fault-observing continuation on every frame a cancelled caller abandons — `TombstoneIfUnclaimed`/`TombstoneUnclaimed` → `ObserveAbandonedFault` → `ObserveFault` (`Ipc/WorkerFrameWriter.cs:308-372`); a cancelled task never fires `OnlyOnFaulted`, so the unconditional attach is safe, and it runs outside `_gate` because an already-faulted task continues inline. Also applied to the detached lock wait (`:389-426`) and, session-side, to overrun background tasks and the abandoned pipe read (`Ipc/WorkerPipeSession.cs:521-576`). Regression test drives the exact abandonment and asserts a marker exception never reaches `TaskScheduler.UnobservedTaskException` (`WorkerFrameProtocolTests:733-785`). | Verified; the discipline is now near-uniform (the one residual gap is WRK-33). |
|
||||
| NEXT-05 | Resolved as documented decision | **Yes** | `DequeueNext` still lazily skips tombstones (`Ipc/WorkerFrameWriter.cs:588-618`); the rationale — any subsequent write drains both queues, the heartbeat bounds tombstone residency to one interval, and eager `Queue<T>` rebuilds would add ordering-invariant surface next to the WRK-22 interlock — is recorded in `docs/WorkerFrameProtocol.md:218-221`. | Decision recorded where the review asked for it. |
|
||||
|
||||
Done claims that failed verification: **none**. Regressions in previously-verified fixes: **none** — WRK-01 (pump refresh, `Sta/StaRuntime.cs:95-100`), WRK-04/GWC-28 (write-time sequence stamping, now peek-stamp-commit), WRK-06 (converter cache, extended by `94fdc18`'s compiled accessors), WRK-07 (two-class scheduler), WRK-11 (ownership transfer, now load-bearing for both the value-cache borrow and enqueue-time sizing), WRK-12 (flush coalescing, finally effective via WRK-25), and WRK-15 (doc names) all remain intact at HEAD.
|
||||
|
||||
## Prior-finding verification — still-open cycle-1 items
|
||||
|
||||
| ID | Status at HEAD | Evidence | Notes |
|
||||
|----|----------------|----------|-------|
|
||||
| WRK-02 | **Still open** | `Sta/StaRuntime.cs:265-269` (loop exception stored in write-only `startupException` after startup; only `startedEvent.Set()`); `InvokeAsync` (`:161-189`) checks disposal and shutdown but not thread death — work enqueued after `ThreadMain` unwinds is consumed by nobody and its task hangs forever (`CancelQueuedCommands` in the `finally` at `:272` only drains what was queued at death). | Highest-value remaining worker item, unchanged three cycles running. |
|
||||
| WRK-03 | **Still open** | `Ipc/WorkerPipeSession.cs:1226-1229` — `TryStartCommandTask` silently `return`s when `_acceptingCommands` is false: no reply, no log. | The gateway's correlation wait times out blind. |
|
||||
| WRK-05 | **Still open, narrowed** | `MxAccess/MxAccessStaSession.cs:392-405` — any exception from one poll records a fault and stops the loop; the drain loop turns it into session termination (`Ipc/WorkerPipeSession.cs:589-598`). Narrowed: in auto mode the `FailoverAlarmConsumer` absorbs primary COM failures inside `PollOnce` (`MxAccess/FailoverAlarmConsumer.cs:284-291`, catches all but OOM and counts toward failover), so a transient alarmmgr E_FAIL no longer escapes. The default alarmmgr-only mode (`Ipc/WorkerPipeSession.cs:78`, and subtag-source failures in standby mode) is still one-failure-fatal. | Worth re-scoping to "non-failover alarm modes" if kept. |
|
||||
| WRK-08 | **Still open** | `Ipc/WorkerPipeSession.cs:487-491` — the message-loop `finally` cancels the drain loop after the shutdown ack; events already drained from the queue (`:600`) whose batch write is cancelled are tombstoned and lost, and the residual queue is never shipped. `docs/MxAccessWorkerInstanceDesign.md` still documents no discard (`grep residual/unshipped` finds nothing). | Same scale as before the batch change (the old loop also drained 128 ahead of writing). Documentation half remains the cheap part. |
|
||||
| WRK-09 | **Still open** | No `AppDomain.UnhandledException`/`TaskScheduler.UnobservedTaskException` handler anywhere under `src/ZB.MOM.WW.MxGateway.Worker/` (grep at HEAD; the only mentions are comments noting the absence). | The NEXT-04 observer network reduces the practical exposure but is not a process-level backstop. |
|
||||
| WRK-10 | **Still open** | `WorkerApplication.cs:112-139` — all three catch blocks log `exception_type` only; the redactor (`Bootstrap/WorkerLogRedactor.cs`) is available and unused here. | Still a cheap diagnostics win. |
|
||||
| WRK-13 | **Resolved (side effect of `f56798a`)** | `Ipc/WorkerFrameWriter.cs:675-687` — single `ArrayPool<byte>.Shared.Rent` for prefix+payload, one stream write, returned in `finally`; matches the reader's pooling. | Flip to Resolved in `archreview/remediation/00-tracking.md` with commit `f56798a`. |
|
||||
| WRK-14 | **Still open** | `Ipc/WorkerPipeSession.cs:33-62` `_camelCase` vs `MxAccess/MxAccessStaSession.cs:50-71` and `Sta/StaRuntime.cs:10-24` bare `camelCase`. | Cosmetic; new files followed their host file's convention, so the split persists. |
|
||||
| WRK-16 | **Still open** | `Ipc/WorkerPipeSession.cs:1449-1544` — seven `CreateEnvelope`/`CreateBaseEnvelope` pairs; `Ipc/WorkerPipeClient.cs` still has 7 public constructors. | |
|
||||
| WRK-17 | **Still open** | `WorkerApplication.cs:110-119` maps every `WorkerFrameProtocolException` (including `EndOfStream`) to exit 6 `ProtocolViolation`. | |
|
||||
| WRK-18 | **Half discharged** | Doc half done: `docs/MxAccessWorkerInstanceDesign.md:743-751` now describes the implemented fail-fast and names the generic `UnexpectedFailure` exit honestly. Code half open: a drain-loop fault still exits 1 via the generic catch (`WorkerApplication.cs:131-139`); no dedicated overflow/event-fault exit code. | |
|
||||
| WRK-19 | **Still open** | `Sta/StaCommandDispatcher.cs` — no logger field, no start/end/failure log anywhere in the file. | |
|
||||
| WRK-20 | **Residual only** | The WRK-21..28 batch added its full test matrix (byte-budget drain, size backstops, cancellation/tombstoning, batch flush, floor handshake, watchdog suppression), plus teardown `UnobservedTaskException` tests (`WorkerPipeSessionTests:2328-2434`). Still missing: STA-thread-death (WRK-02) and command-refused-during-shutdown (WRK-03) tests, because their fixes are open. | |
|
||||
|
||||
## Deep review of the new code
|
||||
|
||||
**Frame-writer unparking (`9130994`) and class-transition flush (`aac7957`).** `WriteAsync` now races its frame's completion against the lock acquisition (`Ipc/WorkerFrameWriter.cs:148-198`): a caller whose frame is delivered by another drainer returns at the boundary flush instead of at the winner's release, and the acquisition it walks away from is handed to `DetachLockWait` (`:389-399`), whose continuation either observes a cancelled wait (no count taken, nothing to release) or drains-and-releases (`:409-452`). I probed this seam hard: acquisition and cancellation are mutually exclusive in `SemaphoreSlim`, so no count can leak and no double-release path exists (`DrainDetachedAsync` releases in a `finally` that runs exactly once per successful acquisition); a frame enqueued between a drainer's last `DequeueNext` and its release is picked up by the detached pass, so no lost wakeup; every discarded task in the chain carries the NEXT-04 observer. The drain flushes and completes at each control→event boundary (`:484-511`), so a heartbeat/reply/fault's delivery point is never charged for the event backlog behind it; frames completed at a boundary leave `written`, so a later flush failure in the same pass cannot reach back and fail an already-delivered frame (`:543-557`) — the failure-attribution edges are all correct. Cost claims in the comments are accurate: a pure-event pass still pays one flush; only a genuinely mixed pass pays one extra per class transition. Residual (accepted, documented at `Ipc/WorkerPipeSession.cs:186-191`): a detached acquisition can schedule an empty drain pass after session teardown — harmless because the queues are empty and an empty pass writes nothing.
|
||||
|
||||
**Value-cache borrowing (`9871d47`).** The hunt for aliasing bugs came up clean. The borrow is sound on three legs, each verified: (1) ordering — `MxAccessBaseEventSink.EnqueueEvent` runs the `postPublish` cache write only after `eventQueue.Enqueue` returns (`MxAccess/MxAccessBaseEventSink.cs:248-295`), so the event is already write-once (WRK-11 ownership plus the sequence/timestamp stamps under the queue lock); (2) consumers — `MxAccessSession.SucceededRead` copies the borrowed `Statuses` element references into a fresh `RepeatedField` on the reply and reference-shares `Value`/`SourceTimestamp` (`MxAccess/MxAccessSession.cs:1066-1097`), and everything downstream only serializes (the worker↔gateway process boundary prevents any other consumer from aliasing); (3) the size-memoization coupling — a post-enqueue mutation would now also invalidate `QueuedEvent.Size`, and the ownership contract text in both classes names that hazard explicitly (`MxAccess/MxAccessValueCache.cs:76-108`, `MxAccess/MxAccessEventQueue.cs:15-32`). The write-completion cache deliberately still clones and documents why the asymmetry is not an oversight (`MxAccess/MxAccessWriteCompletionCache.cs:77-89`). Protobuf reads (`CalculateSize`/`WriteTo`) from multiple threads on a frozen message are safe, and every cross-thread hand-off passes through a lock.
|
||||
|
||||
**Enqueue-time frame sizing (`58d97ad`).** `CalculateWorkerEventSize` reconstructs the generated `CalculateSize` arithmetic for exactly the two stamped fields (`MxAccess/MxAccessEventQueue.cs:493-509`); I checked the tag math — `worker_sequence` (field 9, varint) and `worker_timestamp` (field 10, length-delimited) are both single-byte tags, the `WorkerEvent.event` wrapper (field 1) matches the `DrainEventsReply.events` packing shape exactly, and the +8 `RepeatedFieldOverheadBytes` slack absorbs the documented renumbering hazard (`:66-81`). Clearing the two fields before measurement makes the pre-lock measurement exact even against a caller that pre-filled them. The wake signal (`SignalWake`, `:471-491`) is a level, not a count; I walked the check-then-release race against a concurrent consumer and every interleaving either sees the event in its re-drain or leaves a pending permit — no lost wake, and the 25 ms fallback tick bounds any hypothetical future unsignalled mutation.
|
||||
|
||||
**Session-owned transport disposal and abandonment observation (`e913dab`, `3ef56be`, WRK-31).** The session now owns the pipe stream and disposes it as the last teardown step — the only thing that unblocks a net48 pipe read parked in the kernel — then awaits the abandoned read's resulting fault (`Ipc/WorkerPipeSession.cs:143-290`). The `_pendingReadTask` bookkeeping preserves the reader's single-consumer invariant through teardown (set at issue, cleared before the loop's own await, read only after the loop's task completes — the await supplies the happens-before), and two `Debug.Assert`s pin it (`:447-449`, `:470-472`). `ObserveBackgroundTaskStopAsync` correctly separates the logging budget from observation: an overrun task gets `ObserveFaultWhenever` before the method returns (`:521-558`), so the 1 s budget is a diagnostics decision, not a correctness one. `DisposeTransportStream`'s total catch is well argued (nothing thrown while closing a handle may displace the session's terminal exception, `:212-250`), and `WorkerPipeClient` keeps its `using` as an explicitly-documented idempotent backstop (`Ipc/WorkerPipeClient.cs:153-165`).
|
||||
|
||||
**Oversized-event death (IPC-30) through the batch path.** A `MessageTooLarge` surfacing from the batch's awaited completions is mapped back to the first envelope in batch order that overshoots the negotiated max, logged with identity-but-never-value fields, written as a structured `ProtocolViolation` fault, and then thrown (`Ipc/WorkerPipeSession.cs:626-725`). `FindOversizedEvent`'s recomputation is consistent with what the writer rejected (the rejected envelope retains its tentative sequence stamp, so `CalculateSize` agrees), and the unreachable-fallback comment is honest. Other per-frame rejection codes still unwind the session as worker bugs — the deliberate IPC-30 scope.
|
||||
|
||||
**Write-completion correlation (`fc23a65`..`b0e65d4`).** Parity-clean in the load-bearing respects: no synthesized events (the completion rides the unary reply's `statuses` only), acceptance-only `Hresult`/`ProtocolStatus` either way, expiry yields empty statuses ("unconfirmed, not failed"), zero timeout restores pure fire-and-forget, and the pump-wait keeps the STA dispatching for the whole wait via the shared `MsgWaitForMultipleObjectsEx` declaration (`Sta/StaNativeMethods.cs`, single-copy rationale documented). `StaWaitHelper` handles the failed-wait degrade (one-shot trace + blind sleep, `Sta/StaWaitHelper.cs:104-112`) and documents the `MWMO_INPUTAVAILABLE` drain-to-empty contract that would otherwise busy-spin. The one soft spot is attribution across successive writes to the same handle — WRK-32 below.
|
||||
|
||||
**Alarm subtag failover and truncation.** The production wiring works despite the `standbyFactory: null` default: the `comFactory` threaded through `MxAccessStaSession.StartAsync` lets `AlarmCommandHandler.BuildStandby` construct the `SubtagAlarmConsumer` over `LmxSubtagAlarmSource` (`MxAccess/AlarmCommandHandler.cs:206-248`); mode selection (alarmmgr / subtag / auto-failover) matches the documented matrix, provider-mode changes are surfaced as queue events with handler detach on unsubscribe/dispose (`:159-183`, `:258-267`). WnWrap truncation handling is careful where it matters: a truncated fetch merges into the retained snapshot instead of clearing it (`ApplySnapshotUpdate`), so Clear transitions are never synthesized from records that merely fell off a truncated fetch, and the truncation verdict rides out of the same locked fold that produced the transitions (`FoldFetch`), with a rate-limited warning. STA affinity for the wnwrap COM object is asserted at every consumer entry (`MxAccess/MxAccessStaSession.cs:410-440`), and the affinity-violation exception is deliberately routed to the fault path rather than the graceful-stop path (`:378-405`).
|
||||
|
||||
**Handle-registry reverse index (`afec56d`).** Every mutation maintains all four secondary indexes and the three memoized views; I walked the consistency edges — re-registration with a changed definition drops the stale index entry first, per-server teardown removes exactly the owned sets, advice de-indexing trims empty containers, and the sorted-insertion keeps `GetItemHandlesForDefinition` in the same order the old linear scan visited (`MxAccess/MxAccessHandleRegistry.cs`). The registry is STA-confined by documented contract, so the lock-free indexes are sound. The returned live-view caveat is documented at the API.
|
||||
|
||||
**Timestamp fast path (`94fdc18`, `25cbe5c`).** The per-culture derived formats are validated by round-tripping a probe instant before they ever reach the event path, so a pathological culture pattern (quoted `ss`, existing fraction) degrades to no-fast-path instead of a throwing `TryParseExact`; the fast path is provably a shortcut, never a different answer, because the formats are derived from the same culture the general parse would use (`MxAccess/MxAccessEventMapper.cs:373-509`).
|
||||
|
||||
**Test harness rework.** `FakeRuntimeSession` now mirrors production's coalesced wake signal (never-disposed `SemaphoreSlim`, same rationale comment), models STA-pump-at-heartbeat (`refreshStaActivityOnCapture`), and its 30 s dispatch safety net is decoupled from scenario timing with the reasoning written down (`TestSupport/FakeRuntimeSession.cs:23-51`). The teardown tests arm `TaskScheduler.UnobservedTaskException` and force GC — the strongest possible assertion shape for the NEXT-04/WRK-31 class of fix. Test counts in scope: 40 facts/theories in `WorkerPipeSessionTests`, 31 in `WorkerFrameProtocolTests`, 23 in `MxAccessEventQueueTests`, plus new suites for the completion cache, wait helper, registry, and WnWrap XML parsing.
|
||||
|
||||
## New findings
|
||||
|
||||
**WRK-32 — Low (error handling / documentation).** Write-completion correlation is recency-based per handle; a late `OnWriteComplete` from an earlier write can be reported as the current write's outcome.
|
||||
Evidence: the baseline is `CurrentVersion` captured before the COM call (`MxAccess/MxAccessCommandExecutor.cs:447-450`, `:522-528`), and `AwaitWriteCompletion` copies onto the reply *any* completion whose version exceeds it (`:1005-1027`; version test only, `MxAccess/MxAccessWriteCompletionCache.cs:162-169`). MXAccess's `OnWriteComplete` carries no correlation token, only the handle pair — so if write A to a tag times out unconfirmed (its callback still undelivered) and the client immediately writes B to the same tag, A's stale callback dispatching during B's pump-wait bumps the version past B's baseline and A's status rows ride B's reply. The executor doc comment claims "the OnWriteComplete row *matching* a WriteSecured/WriteSecured2 call" (`:996-1003`), and neither `docs/MxAccessWorkerInstanceDesign.md` nor the proto comments mention the ambiguity (grep for misattribution/stale/earlier-write finds nothing). Impact is bounded — the statuses are an enrichment field, `Hresult`/`ProtocolStatus` stay acceptance-only, and back-to-back same-tag writes with a timed-out predecessor is the only exposed sequence — and the ambiguity is inherent to the COM surface (a native client correlating by handle has the same problem, so mechanical parity is arguably preserved).
|
||||
Recommendation: document the recency-based contract at the executor, in the design doc's write-completion section, and in the reply-field proto comment; additionally, run one `pumpStep()` immediately before capturing the baseline so an already-queued stale callback is folded into the baseline instead of the wait window. Full closure is impossible without an MXAccess-side token — say so where consumers will read it.
|
||||
|
||||
**WRK-33 — Low (concurrency / NEXT-04 discipline gap).** The discarded command-task observer is the one discarded task in the worker without a fault-observing continuation.
|
||||
Evidence: `TryStartCommandTask` discards its wrapper — `_ = ObserveCommandTaskAsync(commandTask);` (`Ipc/WorkerPipeSession.cs:1235`) — and `ObserveCommandTaskAsync` catches only `OperationCanceledException` (`:1238-1254`). `ProcessCommandAsync` handles its own failures, but its catch path ends in `TryWriteFaultAsync`, whose filter is exactly `IOException`/`ObjectDisposedException`/`WorkerFrameProtocolException` (`:1440-1447`); any other exception escaping that write — or a future edit anywhere in the catch path — propagates out of `ProcessCommandAsync`, faults the discarded wrapper, and surfaces only as a finalizer-time unobserved-task exception in a process that installs no handler (WRK-09). Every other discarded task in this codebase now carries an observer (`ObserveFaultWhenever` lives 660 lines up in this same file); the gap is unlikely to fire but contradicts the discipline the NEXT-04/WRK-31 batch established and tested.
|
||||
Recommendation: `ObserveFaultWhenever(ObserveCommandTaskAsync(commandTask));`, or broaden `ObserveCommandTaskAsync` to catch-all-and-log (the redaction-safe fields are already at hand in `LogCommandResultDropped`'s shape).
|
||||
|
||||
**WRK-34 — Low (documentation / diagnostics).** The StaHung fault writes the in-flight correlation id into `WorkerFault.CommandMethod`.
|
||||
Evidence: `ReportWatchdogFaultIfNeededAsync` builds its fault with `CreateFault(WorkerFaultCategory.StaHung, snapshot.CurrentCommandCorrelationId, $"STA activity is stale by {staleFor}.")` (`Ipc/WorkerPipeSession.cs:1384-1389`), whose second parameter is `commandMethod` (`:1712-1728`) — so the wire fault's `command_method` field carries a correlation id (or empty), a correlation id never lands in a correlation-shaped field, and a gateway/dashboard consumer reading `CommandMethod` as a method name gets an opaque id string. Pre-existing since `4a3560c` (2026-04-26), never previously flagged; noted now because the WRK-27 work touched this exact branch and the field's semantics matter more once polls also suppress here.
|
||||
Recommendation: set `CommandMethod` to a stable label (empty, or `"Watchdog"`), and carry the stuck command's correlation id in the fault's diagnostic message or the envelope's `CorrelationId` where the gateway already correlates.
|
||||
|
||||
## Severity roll-up (new findings)
|
||||
|
||||
| Severity | Count | IDs |
|
||||
|----------|-------|-----|
|
||||
| High | 0 | — |
|
||||
| Medium | 0 | — |
|
||||
| Low | 3 | WRK-32, WRK-33, WRK-34 |
|
||||
|
||||
## Still good / preserved
|
||||
|
||||
Every structural strength recorded in the prior two reviews is intact at HEAD, and several were deepened. The STA core still runs the canonical `MsgWaitForMultipleObjectsEx` + `PeekMessage`/`DispatchMessage` loop with the wake event and 50 ms idle pump (`Sta/StaRuntime.cs:255-261`), and the subtle Win32 wait contract now has a single shared declaration with the drift hazard written down (`Sta/StaNativeMethods.cs`). All MXAccess COM calls remain on the dedicated STA (commands via the dispatcher, alarm polls via `InvokeAsync` with an affinity assertion, the consumer factory on-thread at `MxAccess/MxAccessStaSession.cs:282-296`); COM lifetime discipline (`FinalReleaseComObject`, sink-detach-before-release, UnAdvise → RemoveItem → Unregister, alarm handler disposed on the STA before data-side teardown) is unchanged. Nothing synthesizes events — the completion-correlation feature explicitly kept `OperationComplete` untouchable and the truncation work explicitly refuses to fabricate Clear transitions. Fail-fast backpressure stands (no drop policy; overflow self-records the first fault; the dispatcher's 128-entry bound rejects loudly). Credentials and values stay out of logs (the oversized-event fault logs sizes and identity only, `AuthenticateUser` passes the password straight through, the DrainEvents truncation message names sequences not values). net48 constraints are respected in every new type (plain ctors, get-only properties, `readonly struct`, no init/records — several carry the CS0518 rationale in their remarks). The IPC layer's failure-attribution story is now the best part of the worker: written-and-flushed-before-completion, per-frame vs stream-failure separation, boundary-flush completion, tombstone/claim interlock, and a fault observer on effectively every abandoned task, each with a regression test that asserts the invariant rather than the implementation. Comment quality across the new code is exceptional — ownership contracts, memory-model reasoning, and rejected alternatives are recorded at the exact seams where the next editor would otherwise re-litigate them.
|
||||
|
||||
## Verification needed on Windows (windev)
|
||||
|
||||
- This review is static: the x86 worker was not built and `Worker.Tests` were not run here. The tracking log and repo memory record the worker suite fully green on windev after the harness fixes (`7da52b6`/`462850a`/`aaeb86b`), and the WRK-21 cluster carries windev evidence (`c992568`); any fix for WRK-32..34 needs an x86 build plus filtered `MxAccessCommandExecutorTests`/`WorkerPipeSessionTests` runs there.
|
||||
- WRK-32's pre-baseline pump mitigation should get a windev test that records a stale completion into the fake's message path before the second write's baseline and asserts it does not ride the second reply.
|
||||
- WRK-33 is assertable with the existing `UnobservedTaskException`-arming test shape (`WorkerPipeSessionTests:2328-2434`): drive an exception through `TryWriteFaultAsync`'s filter gap with a throwing fake writer and assert nothing reaches the finalizer.
|
||||
- Live MXAccess only: the WRK-05 narrowing (failover absorbing a transient alarmmgr E_FAIL without a session fault) deserves a live-rig confirmation next time the opt-in alarm smoke runs; the rig's object-driven alarms make this drivable only from the simulator side.
|
||||
@@ -0,0 +1,143 @@
|
||||
# Contracts & IPC Protocol — Cycle-3 Review
|
||||
|
||||
- **Date:** 2026-08-19
|
||||
- **Commit:** `8b3e3b3` (`main`, clean)
|
||||
- **Baseline:** `4f5371f` (2026-07-12 cycle-2 review; ~288 commits since, ~55 of them touching the contracts, frame-protocol, codegen, or IPC-doc paths in scope)
|
||||
- **Method:** static review on macOS; every cited file read at HEAD. No build, no test, no source change. `git diff 4f5371f..HEAD -- '*.proto'` walked line by line; generated trees grep-checked for every new symbol; the GalaxyRepository package proto compared against the sibling `scadaproj` checkout.
|
||||
- **Scope:** `src/ZB.MOM.WW.MxGateway.Contracts` (three protos, `Generated/` hygiene, `net10.0;net48` multi-targeting, `galaxy_repository.proto` retention vs the `ZB.MOM.WW.GalaxyRepository` package), the frame protocol on both sides (`Worker/Ipc/*`, `Server/Workers/WorkerFrame*`, `WorkerClient` envelope/sequence/size handling, hello/handshake, `max_frame_bytes` negotiation, DrainEvents byte bound), the codegen/descriptor pipeline (`scripts/check-codegen.ps1` Checks 1–4, `scripts/publish-client-proto-inputs.ps1`, `clients/proto/*`, vendored Rust protos, Go/Python/Java binding freshness guards, Gradle `checkGeneratedClean`), proto-evolution hygiene, and currency of `docs/WorkerFrameProtocol.md`, `gateway.md` (IPC envelope section), `docs/Grpc.md`, `docs/ClientProtoGeneration.md`, `docs/Contracts.md`. Worker-internal mechanics of the drain/write seam are owned by `20-worker.md` and are cross-referenced, not re-reviewed.
|
||||
- **Prior reports:** `archreview/2026-07-12/30-contracts-ipc.md` (cycle 2); `archreview/30-contracts-ipc.md` (cycle 1); remediation status sources `archreview/2026-07-12/remediation/00-tracking.md` + `30-contracts-ipc.md`, `90-candidate-findings-next-cycle.md`; first-cycle backlog `archreview/remediation/00-tracking.md`.
|
||||
|
||||
## Proto evolution since baseline — verified additive-only
|
||||
|
||||
`git diff 4f5371f..HEAD -- src/ZB.MOM.WW.MxGateway.Contracts/Protos/` contains exactly four wire-affecting lines, all additive with next-free tags:
|
||||
|
||||
| Change | Location | Tag check |
|
||||
|---|---|---|
|
||||
| `bool snapshot_truncated = 2;` on `QueryActiveAlarmsReplyPayload` | `mxaccess_gateway.proto:735` | message previously ended at field 1 |
|
||||
| `bool from_truncated_snapshot = 16;` on `ActiveAlarmSnapshot` | `:951` | previous highest was `source_provider = 15` |
|
||||
| `AlarmSnapshotStatus snapshot_status = 5;` oneof arm on `AlarmFeedMessage` | `:1028` | previous highest arm was `provider_status = 4` |
|
||||
| new `message AlarmSnapshotStatus { bool truncated = 1; }` | `:1048-1054` | new type |
|
||||
|
||||
Everything else in the diff is comment-only (write-completion `statuses` contract at `:538-551`, DrainEvents truncation at `:702-705`, `ReplayGap` empty-ring sentinel at `:797-801`, `AlarmFeedMessage` open sequence at `:1006-1012`, and the `GatewayHello.max_frame_bytes` byte-cap sentence at `mxaccess_worker.proto:50-52`). No renumbering, no repurposing; `reserved 1` / `reserved "session_id"` intact on the two retired fields (`mxaccess_gateway.proto:964-965`, `:978-979`); wire-compat policy headers present in all three files; `gatewayProtocolVersion: 3` / `workerProtocolVersion: 1` in `clients/proto/proto-inputs.json:4-5` match `GatewayContractInfo.cs:12,15` (additive changes correctly did not bump either). `galaxy_repository.proto` is unchanged and still wire-identical to the package source (see IPC-37).
|
||||
|
||||
Generated-tree freshness for the new symbols, grep-verified at HEAD: `Contracts/Generated/MxaccessGateway.cs` (`AlarmSnapshotStatus` ×16, `from_truncated_snapshot` ×2), Go `mxaccess_gateway.pb.go` (×15 / ×3), Python `mxaccess_gateway_pb2.py` (descriptor bytes carry `\x66rom_truncated_snapshot`, `AlarmSnapshotStatus`), Java `MxaccessGateway.java` (×106 / ×7), the committed protoset (×2 / ×2), and the Rust vendored copies `cmp`-identical for all three files. Protos, protoset, and every binding set were last touched in the same commit (`c3c603f`, 2026-08-18) — the regen-everything-together discipline held, including for a comment-only change.
|
||||
|
||||
## Prior-finding verification
|
||||
|
||||
### Cycle-2 findings (IPC-23..32)
|
||||
|
||||
Every tracked status was verified against code read at HEAD. **All eight Done claims hold; one carries a residual filed as a new finding (IPC-23 → IPC-33).**
|
||||
|
||||
| ID | Claimed | Verified? | Evidence (HEAD) | Notes |
|
||||
|----|---------|-----------|-----------------|-------|
|
||||
| IPC-23 | Done (mechanics in WRK-21) | ✅ holds, **with caveat** | Contract half: `GatewayHello.max_frame_bytes` comment extended (`mxaccess_worker.proto:50-52`); `DrainEventsReply` truncation comment (`mxaccess_gateway.proto:702-705`); `docs/WorkerFrameProtocol.md:43-51` states the byte-cap rule and drain-until-empty contract; `gateway.md:565-572` documents both axes. Mechanics: byte-budgeted `Drain(uint, int)` leaves a non-fitting head queued and reports `OversizedHeadSequence` (`Worker/MxAccess/MxAccessEventQueue.cs:389-429`, per-event size memoized at enqueue `:493-509`); reply builder `CreateDrainEventsReply` consumes it with a truncation `DiagnosticMessage` (`Worker/Ipc/WorkerPipeSession.cs:978-1020`); budget = `max(Max − 64 KiB, Max/2)` (`:1034-1039`); shared count ceiling `GatewayContractInfo.MaxDrainEventsPerCommand` (`Contracts/GatewayContractInfo.cs:33`) consumed by both the gateway validator (`Grpc/MxAccessGrpcRequestValidator.cs:81-86`) and the worker clamp (`:992-994`); reply-too-large backstop answers `InvalidRequest` instead of dying (`:847-864`). R1–R3 satisfied. | Caveat: R1 bounds the reply against the **pipe** frame max; the same `MxCommandReply` then crosses the public gRPC boundary whose send cap is `MaxGrpcMessageBytes` = pipe max − 64 KiB, and the budget's own 64 KiB headroom consumes exactly that margin → **IPC-33**. The tracker's rationale for leaving `docs/Grpc.md` without a DrainEvents row ("a worker IPC diagnostic, not a public RPC") is inaccurate — `DrainEvents` is a public `MxCommandKind` (`mxaccess_gateway.proto:136`) reachable through `Invoke` → **IPC-38**. |
|
||||
| IPC-24 | Done | ✅ holds | Churn-revert step deleted; `java` job runs `gradle test` then `git diff --exit-code -- clients/java/src/main/generated` (`.gitea/workflows/ci.yml:135-142` (test step `:135-137`, diff gate `:138-142`), comment at `:113-119` records the rationale); toolchain pinned `grpcVersion 1.76.0` / `protobufVersion 4.33.1` (`clients/java/build.gradle:8,11`); `checkGeneratedClean` revived for Gradle 9 via `providers.exec` evaluated lazily in `doLast` (`clients/java/zb-mom-ww-mxgateway-client/build.gradle:71-92`, `df45cb4`); docs updated (`docs/ClientProtoGeneration.md:97,100-106`). Java aggregate at HEAD carries every new symbol (see above). | The lazy `ProviderFactory.exec` shape is correct: the provider is first realized in `doLast`, after the `generateProto` dependency ran, so the git probe sees the regenerated tree. |
|
||||
| IPC-25 | Done | ✅ holds | Go/Python bindings regenerated and fresh (grep above; `max_frame_bytes` now in `mxaccess_worker.pb.go` ×2 and `mxaccess_worker_pb2.py` ×1); Check 4 regenerates both via the per-client scripts and fails on any `git status --porcelain` diff, with tool-missing → fail not skip (`scripts/check-codegen.ps1:109-129`, header `:19-24`); Go script asserts `protoc-gen-go v1.36.11` / `protoc-gen-go-grpc 1.6.2` exactly and warns-only on protoc (`clients/go/generate-proto.ps1:13-15,63-78`), with the Windows `.exe` banner normalized (`:22-28`, `c94c4d4`); Python script asserts grpcio-tools 1.80.0 with PS5.1-safe quoting (`clients/python/generate-proto.ps1:36-49`, `8ae0c2f`); CI installs the pinned generators before the check (`ci.yml:63-72`); pins table in `docs/ClientProtoGeneration.md:90-98`. | Windows-side fixes landed after the cycle-2 design (Check 4 now runs on windev too). Backslash-joined paths in both scripts are normalized by pwsh on Unix (verified locally with `Test-Path`). |
|
||||
| IPC-26 | Done (mechanics in WRK-22) | ✅ holds | Contract requirement ("cancelled means never written") implemented by claim/tombstone interlock: `TombstoneIfUnclaimed` (`Worker/Ipc/WorkerFrameWriter.cs:181,308-338`), `DequeueNext` skips tombstones (`:588`), NEXT-04 fault observer on every abandoned frame (`:350`); documented with the claimed-mid-write residual at `docs/WorkerFrameProtocol.md:202-224`. Full mechanics verification in `20-worker.md` (WRK-22, NEXT-04/05). | Doc landed in the same batch as the code, as the plan required. |
|
||||
| IPC-27 | Done | ✅ holds | `ClientProtoInputTests.Descriptor_ContainsEveryContractSymbol` collects messages, fields, nested/top-level enums and values, services and methods from the protoset (`Tests/Contracts/ClientProtoInputTests.cs:37-62,108-146`) and walks all three reflection descriptors including `GalaxyRepositoryReflection.Descriptor` (`:65-99`); doc prose updated (`docs/ClientProtoGeneration.md:76-82`). | Presence-only (no field numbers / types), per the recorded design; the protoc-pinned script check remains the byte-level gate. |
|
||||
| IPC-28 | Done | ✅ holds | `docs/Grpc.md:285` lists `CommandTooLarge → ResourceExhausted`; Invoke-section paragraph at `:85` explains the per-correlation failure and the headroom rule; matches `Grpc/MxAccessGatewayService.cs:1056`. | |
|
||||
| IPC-29 | Done (discharged by WRK-26) | ✅ holds | `docs/WorkerFrameProtocol.md:79-224` "Write Scheduling And Sequencing": two-class scheduler, peek-stamp-commit write-time stamping (`:60-64`, `:98-107`), per-frame vs stream failure (`:109-120`), flush coalescing and boundary flush (`:122-172`), unparking (`:174-200`), cancellation tombstones (`:202-224`); plus a new "Pipe Buffers" section (`:226-235`, `da84635`). Each claim cross-checked against `Worker/Ipc/WorkerFrameWriter.cs` anchors cited in `20-worker.md`. | The doc now slightly *exceeds* the code in one place: `mxaccess_worker.proto:50-52` says "reply builders truncate to fit" immediately after listing events — events do not truncate (IPC-30 policy). Readable as intended; no action. |
|
||||
| IPC-30 | Done | ✅ holds | Batch-path `MessageTooLarge` is mapped to the first overshooting envelope (`Worker/Ipc/WorkerPipeSession.cs:633-648`, `FindOversizedEvent :648-660`), logged identity-only, written as a `ProtocolViolation` fault with `CommandMethod = "EventDrain"` (`:713`), then thrown; policy documented at `docs/WorkerFrameProtocol.md:53-58`. | Gateway side needed no change: `DispatchEnvelope` already routes `WorkerFault` to `SetFaulted(WorkerFaulted, …)` (`Server/Workers/WorkerClient.cs:614-618`). |
|
||||
| IPC-31 | N/A (accepted divergence) | **Superseded — resolved by GWC-28** | The gateway now stamps `Sequence` in `WriteLoopAsync` immediately before the write (`Server/Workers/WorkerClient.cs:465-471`), `CreateEnvelope` leaves it unset (`:1229-1236`); `gateway.md:425-431` rewritten to "both sides stamp at the point of writing". Regression test `WorkerClientTests.ConcurrentInvokesEmitStrictlyIncreasingSequencesOnTheWire`. | The two sides now implement the same semantics; flip the IPC-31 row from N/A to Resolved-by-GWC-28 in the tracker. See IPC-39 for the one interaction the change created with the IPC-03 pre-check. |
|
||||
| IPC-32 | Done | ✅ holds | Banners read `Check 1/4` … `Check 4/4` (`scripts/check-codegen.ps1:40,52,78,110`); header comment lists four checks (`:4-24`). | |
|
||||
|
||||
**Done claims that do not fully hold:** none outright. IPC-23 holds for the defect as filed but leaves the gRPC-boundary residual (IPC-33).
|
||||
|
||||
### Cycle-1 findings still open at cycle 2 — re-checked at HEAD
|
||||
|
||||
| ID | Cycle-2 status | HEAD status | Evidence |
|
||||
|----|----------------|-------------|----------|
|
||||
| IPC-08 | Open (`WorkerCancel` defined, handled, never sent) | **Substantially resolved by `7c1ea12`/`a1a38b5` — recommend closing with a residual** | `InvokeAsync` forwards a `WorkerCancel` when the gateway's own command timeout fires and it owned the removal (`Server/Workers/WorkerClient.cs:269-291`); `TrySendCancelForTimedOutCommand` is structurally no-throw (`:835-870`, doc `:806-833`); worker routes it to `CancelCommand` (`Worker/Ipc/WorkerPipeSession.cs:755-756` → `MxAccess/MxAccessStaSession.cs:529-532` → `Sta/StaCommandDispatcher.CancelQueuedCommand :151`); test `WorkerClientTests.InvokeAsync_WhenCommandTimesOut_SendsWorkerCancelForThatCorrelation` (`:185`); `docs/GatewayProcessDesign.md:596-602` documents it. Residual: the caller-cancellation path sends nothing → **IPC-34**. |
|
||||
| IPC-10 | Open (softened) | **Still open, unchanged** | `Server/Workers/WorkerEnvelopeValidator.cs:15-39` and `Worker/Ipc/WorkerEnvelopeValidator.cs` check version/session/body only; `mxaccess_worker.proto:23` `uint64 sequence = 3;` still has no comment. `gateway.md:425-431` continues to state honestly that sequence is a diagnostic aid, and now that both sides stamp at write (GWC-28/WRK-04) the property an inbound check would rely on actually holds. Acceptable state. |
|
||||
| IPC-11 | Open | **Still open, unchanged** | Strict equality on both sides: `Worker/Ipc/WorkerPipeSession.cs:370-375` (`ProtocolVersionMismatch`), `Server/Workers/WorkerClient.cs:913-918`; no min/max-range note on `supported_protocol_version` (`mxaccess_worker.proto:43`). No action was ever required. |
|
||||
| IPC-12 | Open | **Still open, surface unchanged** | `Server/Workers/WorkerFrameWriter.cs:8-11` class doc still has no single-writer remark and the class has no lock; the invariant is now stated at the *call site* instead (`WorkerClient.cs:465-470`: "This loop is the channel's single consumer (SingleReader = true)"), and every outbound envelope — hello `:171`, commands `:251`, cancels `:848-858`, shutdown — funnels through `_outboundEnvelopes`. The gateway reader, by contrast, got its contract written into the class remark when GWC-30 made it stateful (`WorkerFrameReader.cs:11-16`); the writer deserves the same one-paragraph note. |
|
||||
| IPC-16 / IPC-22 | N/A | Unchanged | Correlation id still carried on envelope + inner reply (`gateway.md:432-434` documents envelope as authoritative); status-code-plus-prose error model unchanged. |
|
||||
| IPC-18 | Preserve | **Preserved** | See "Proto evolution" above — `Generated/`, all five client trees, the protoset, and the Rust vendored copies are in sync at HEAD; `galaxy_repository.proto` retention comment intact (`ZB.MOM.WW.MxGateway.Contracts.csproj:36-41`). |
|
||||
|
||||
### NEXT rows touching this domain
|
||||
|
||||
- **NEXT-04 / NEXT-05** (frame-writer abandonment observation; lazy tombstone purge as a documented decision) — both verified in `20-worker.md`; the contract-level statement lives at `docs/WorkerFrameProtocol.md:213-224`. Nothing further from this domain.
|
||||
- **NEXT-01** (pipe name length) — verified in `10-gateway-core.md`; the frame protocol is unaffected (the name is a transport address, not a wire field).
|
||||
|
||||
### Also verified in passing
|
||||
|
||||
- **TST-28** — the gateway half of the `max_frame_bytes` handshake is pinned on the CI-run suite: `WorkerClientTests.StartAsync_SendsGatewayHelloWithConfiguredMaxFrameBytes` (`:46-57`), hello built at `WorkerClient.cs:1172-1184` from `FrameOptions.MaxMessageBytes`.
|
||||
- **Negotiation floor/ceiling agreement** — worker accepts `[1024, 256 MiB]` (`Worker/Ipc/WorkerFrameProtocolOptions.cs:19,30,137-161`); the gateway validator bounds `Worker:MaxMessageBytes` to the identical range (`Configuration/GatewayOptionsValidator.cs:10-11,290-293`) and enforces pipe ≥ gRPC + 64 KiB with `long` arithmetic (`:614-628`). The worker can never reject a value the gateway's validator admits.
|
||||
- **Public gRPC caps are actually applied** — `MaxReceiveMessageSize` **and** `MaxSendMessageSize` are both set from `Protocol:MaxGrpcMessageBytes` (`Security/Authorization/GrpcAuthorizationServiceCollectionExtensions.cs:36-37`); default 16 MiB (`Configuration/ProtocolOptions.cs:16`). Load-bearing for IPC-33.
|
||||
- **Check 3 both directions** (`711793f`) — a canonical proto without a vendored Rust copy now fails too (`scripts/check-codegen.ps1:95-103`), closing the `build.rs`-prefers-canonical blind spot its comment describes.
|
||||
- **Contracts package version** bumped to 0.2.0 with the CLI-39 release train (`ZB.MOM.WW.MxGateway.Contracts.csproj:10`); `Grpc.Tools` 2.80.0 / `Google.Protobuf` 3.34.1 pins unchanged.
|
||||
|
||||
## New findings
|
||||
|
||||
IDs continue from the register (cycle 2 ended at IPC-32). No `IPC-33+` label exists anywhere in the repo or its history (grep), so there is no collision of the GWC-31 kind here.
|
||||
|
||||
### IPC-33 — The DrainEvents byte budget is sized against the pipe frame maximum, not the public gRPC send cap; the reply can cross the pipe and then fail at the gRPC boundary with the drained events already consumed
|
||||
|
||||
- **Severity:** Low
|
||||
- **Files:** `src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs:31` (`DrainReplyFrameHeadroomBytes = 64 KiB`), `:1034-1039` (`ResolveDrainReplyByteBudget` = `max(MaxMessageBytes − 64 KiB, MaxMessageBytes / 2)`); `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs:389-429` (events dequeued as they are packed); `src/ZB.MOM.WW.MxGateway.Server/Configuration/WorkerOptions.cs:57-66` (pipe default = gRPC 16 MiB + 64 KiB reserve); `src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs:614-628` (pipe ≥ gRPC + reserve, no upper coupling); `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GrpcAuthorizationServiceCollectionExtensions.cs:37` (`MaxSendMessageSize = MaxGrpcMessageBytes`); `src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_gateway.proto:20` (`Invoke` returns the bare `MxCommandReply`).
|
||||
- **Description:** WRK-21/IPC-23 made the reply fit the *pipe* (R1), but the same `MxCommandReply` is then returned verbatim through `Invoke`, where the gateway's gRPC `MaxSendMessageSize` applies. The two limits are coupled only from below: the validator requires pipe ≥ gRPC + 64 KiB, and the drain budget subtracts the same 64 KiB — so at default configuration the event bytes are budgeted to exactly `MaxGrpcMessageBytes` (16 MiB), leaving the `MxCommandReply` wrapper (34-byte correlation id, protocol status, kind, the `drain_events` length prefix, and — precisely in the truncated case — the `DiagnosticMessage`) with zero margin. The per-event `RepeatedFieldOverheadBytes` slack (+8 per event) absorbs this whenever the packed batch has more than ~30 events, so the default-config overshoot needs a backlog of ≥ 16 MiB composed of a few very large (≥ ~500 KiB) array events — narrow but real. The coupling is entirely absent for the configurations the validator explicitly permits: `Worker:MaxMessageBytes` may be anything up to 256 MiB while `Protocol:MaxGrpcMessageBytes` stays 16 MiB, and then every `DrainEvents` against a backlog above 16 MiB builds a reply of up to `pipeMax − 64 KiB`, which the worker writes, the pipe carries, the gateway parses — and the gRPC layer refuses to send (`ResourceExhausted: Sending message exceeds the maximum configured message size`). At that point the events are already dequeued from the worker queue (`Drain` removes what it packs), so up to 10 000 events are lost to both the diagnostic caller and the event stream, and the caller sees only a failed `Invoke`. This is the IPC-23 invariant ("no diagnostics command may kill a session") satisfied, and its sibling ("no event loss on truncation", R2) violated one boundary later. The same asymmetry exists for single events in the 64 KiB window between the gRPC cap and the pipe cap (an event the pipe accepts but `StreamEvents` cannot send), but a single ≥ 16 MiB event is pathological and already fails loudly on the stream; it is noted, not filed.
|
||||
- **Recommendation:** Bound the drain against the *smaller* of the two boundaries. Cheapest correct fix without a wire change: the gateway knows both caps, so either (a) add an upper coupling in `GatewayOptionsValidator` (`Worker:MaxMessageBytes ≤ MaxGrpcMessageBytes + 2 × reserve`, or simply document and validate the equality the default already has) and change the worker budget to `MaxMessageBytes − 2 × reserve` so the reply wrapper has a full reserve below the gRPC cap; or (b) convey `MaxGrpcMessageBytes` in `GatewayHello` (additive `uint32`, same zero-means-unknown convention as `max_frame_bytes`) and budget against `min(pipe − reserve, grpc − reserve)`. Add a gateway fake-worker test that drains a reply packed to the budget through the real gRPC pipeline (the TST-24 wire-test harness can host it) so the boundary is exercised end to end.
|
||||
|
||||
### IPC-34 — `WorkerCancel` is forwarded only when the gateway's own command timeout fires; a caller-cancelled `Invoke` (client deadline, disconnect) leaves the command queued on the worker's STA
|
||||
|
||||
- **Severity:** Low
|
||||
- **Files:** `src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs:269-291` (timeout arm → `TrySendCancelForTimedOutCommand`), `:294-305` (`OperationCanceledException` arm: `RemovePendingCommandAsFailed(GatewayShutdown, "Command wait was canceled.")`, no cancel sent), `:307-318` (timer-won-but-token-cancelled arm, same); `src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs:138` (`context.CancellationToken` flows into `InvokeAsync`); `src/ZB.MOM.WW.MxGateway.Server/Sessions/GatewaySession.cs:1100`; worker side `src/ZB.MOM.WW.MxGateway.Worker/Sta/StaCommandDispatcher.cs:151-185` (`CancelQueuedCommand` drops only not-yet-started entries); `docs/GatewayProcessDesign.md:596-602`; `gateway.md:913-919`.
|
||||
- **Description:** The IPC-08 closure's own rationale — "the gateway has stopped waiting, but the worker has not stopped working … a queued-but-not-started command is dropped instead of occupying the STA behind a caller that is already gone" (`WorkerClient.cs:279-284`) — applies identically when the caller's token fires, and that is the *common* abandonment path: gRPC clients routinely set per-call deadlines shorter than the gateway's `CommandTimeout`, and a client disconnect cancels `context.CancellationToken`. Both land in the `OperationCanceledException` arm, which removes the pending entry and returns, so the correlation stays on the single STA queue and executes when its turn comes. Under a burst of deadline-cancelled calls (a client retry storm against a slow provider) the STA works through dead commands while live callers queue behind them — exactly the backlog the cancel was added to shorten. The gateway-side design text still frames cancellation as "stop waiting" only (`gateway.md:915`), and `docs/GatewayProcessDesign.md:596` says "on timeout", so the asymmetry is documented-by-omission rather than decided. Secondary: the arm labels caller cancellation `WorkerClientErrorCode.GatewayShutdown` (pre-existing; maps to `Cancelled` at the gRPC layer per `docs/Grpc.md:285`, so the client sees the right status, but metrics/logs record a shutdown that did not happen).
|
||||
- **Recommendation:** Call the same no-throw `TrySendCancelForTimedOutCommand` (renamed to `TrySendCancelForAbandonedCommand`, reason "caller cancelled") from the caller-cancellation arms, gated on the same `removed` ownership check so a reply that won the race is never chased. Skip it only when `_stopCts` is the source of the cancellation (the channel is being completed anyway). Add a `WorkerClientTests` twin of the timeout test driven by the caller token. Separately, introduce a `CallerCancelled` error code (or reuse `CommandTimeout` semantics) so the metric label stops saying `GatewayShutdown`; update the `docs/Grpc.md:285` mapping line in the same change.
|
||||
|
||||
### IPC-35 — A `WorkerFault` the worker emits during the startup handshake is flattened by the gateway into a generic `ProtocolViolation`, discarding the structured fault category and diagnostic message
|
||||
|
||||
- **Severity:** Low
|
||||
- **Files:** `src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs:894-907` (`ReadHandshakeEnvelopeAsync`: any body other than the expected one → `WorkerClientException(ProtocolViolation, "Worker handshake expected {expected} but received {actual}")`), `:164-185` (`StartAsync` expects `WorkerHello` then `WorkerReady`); worker side `src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs:335-357` (`CompleteStartupHandshakeAsync` writes a `WorkerFault` for every `WorkerFrameProtocolException` and for `MxAccessCreationException` — COM creation failure, the most likely real-world startup fault — before rethrowing), `:1392-1447` (`TryWriteFaultAsync` overloads); `src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs:132-160` (the exception message is what reaches `MarkFaulted`, the warning log, and the `SessionManagerException` inner).
|
||||
- **Description:** The worker deliberately reports startup failures as structured frames — `MxAccessCreationException.From(...)` carries the COM HRESULT/ProgID context and `WorkerFaultCategory` — so the gateway can tell "MXAccess is not installed / not registered / access denied" apart from a pipe-level protocol problem. The gateway's handshake reader does not look inside: it sees `BodyCase == WorkerFault` where it expected `WorkerReady`, throws a `ProtocolViolation` whose message names only the body type, and `SessionManager.OpenSessionAsync` surfaces that text to the operator log and to the client's `OpenFailed`. The post-handshake path does this correctly (`DispatchEnvelope` → `SetFaulted(WorkerFaulted, CreateWorkerFaultMessage(fault))`, `WorkerClient.cs:614-618`), so the information loss is confined to the window in which it matters most — first session open on a freshly deployed host, where the answer is almost always an MXAccess installation or permission problem, and the operator is pointed at "protocol violation" instead. The `FakeWorkerHarness` only ever scripts well-formed handshakes (`Tests/Gateway/Workers/Fakes/FakeWorkerHarness.cs:140-200`), so nothing pins the behaviour either way.
|
||||
- **Recommendation:** In `ReadHandshakeEnvelopeAsync`, special-case `BodyCase == WorkerFault`: throw `WorkerClientException(WorkerFaulted, CreateWorkerFaultMessage(envelope.WorkerFault))` (the same formatter the steady-state path uses) so category and diagnostic reach the log and the `OpenFailed` detail. Keep `ProtocolViolation` for every other unexpected body. Add a `WorkerClientTests` case where the fake writes a `WorkerFault{Category = MxaccessCreationFailed}` in place of `WorkerReady` and asserts the category text surfaces.
|
||||
|
||||
### IPC-36 — The worker launch environment has become a second, undeclared gateway↔worker contract: six variable names duplicated as independent literals on both sides, four of them new since baseline, two documented
|
||||
|
||||
- **Severity:** Low
|
||||
- **Files:** gateway literals `src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerProcessLauncher.cs:18-52` (`MXGATEWAY_WORKER_NONCE`, `…_PIPE_CONNECT_ATTEMPT_TIMEOUT_MS`, `…_WRITE_COMPLETION_WAIT_MS`, `MXGATEWAY_ALARM_POLL_INTERVAL_MS`, `MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH`, `MXGATEWAY_EVENT_QUEUE_CAPACITY`), set at `:207-217`; worker literals `src/ZB.MOM.WW.MxGateway.Worker/Bootstrap/WorkerOptions.cs`, `Ipc/WorkerPipeClient.cs`, `MxAccess/MxAccessStaSession.cs:14-24`, `MxAccess/MxAccessEventQueue.cs`, `MxAccess/WnWrapAlarmConsumer.cs` (one copy each); the one precedent that *is* shared: `src/ZB.MOM.WW.MxGateway.Contracts/GatewayContractInfo.cs:35-43` (`LiveMxAccessOptInVariableName`, shared "so any future opt-in tweak does not silently leave one project behind"); docs `docs/WorkerProcessLauncher.md:25-27,47-51` (nonce + connect timeout only), `docs/MxAccessWorkerInstanceDesign.md:93-98` ("Expected protected environment values": nonce + log context only); public proto comment `src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_gateway.proto:540` names `MXGATEWAY_WORKER_WRITE_COMPLETION_WAIT_MS`.
|
||||
- **Description:** Since `4f5371f`, four runtime knobs were plumbed to the worker through `ProcessStartInfo.Environment` rather than through `GatewayHello` — write-completion wait, alarm poll cadence, alarm fetch cap, event-queue capacity. Environment conveyance is a reasonable choice (the values are needed before or independent of the handshake, and the worker's fallback-to-default on a missing variable keeps old/new pairs working), but it makes the launch environment a wire contract in everything but name, and it is treated with none of the discipline the proto contract gets: each name exists as two unrelated string literals in two projects (a typo on either side silently reverts that knob to its default — the worker's documented "missing or unparseable → default" fallback turns a contract break into a no-op), the Contracts project already demonstrates the right home for exactly this kind of name and was not used, and the documentation that should enumerate the contract lists two of the six. The public `mxaccess_gateway.proto` comment additionally exposes the internal env-var name to every language-client consumer, where the operator-facing key is `MxGateway:Worker:WriteCompletionWaitMilliseconds` (the `WorkerOptions.cs:28-34` and launcher `:25-27` doc comments are also stale — they still say "WriteSecured/WriteSecured2" although `b0e65d4` extended the wait to plain `Write`/`Write2`).
|
||||
- **Recommendation:** Move the six names into a `WorkerLaunchEnvironment` static class in `ZB.MOM.WW.MxGateway.Contracts` (net48-safe `const string`s, like `GatewayContractInfo`) and reference it from both sides; add a one-table "Launch environment" section to `docs/WorkerProcessLauncher.md` (variable, source config key, worker default on absence) and point `docs/MxAccessWorkerInstanceDesign.md:93-98` at it; replace the env-var name in the proto comment with the config key; refresh the two "WriteSecured-only" comments. A small test asserting the launcher sets every constant in the class keeps the table and the code from drifting.
|
||||
|
||||
### IPC-37 — The `galaxy_repository.proto` wire-identity with the `ZB.MOM.WW.GalaxyRepository` package is still unguarded, although both descriptors are now loadable in one test process
|
||||
|
||||
- **Severity:** Low
|
||||
- **Files:** `src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj:36-41` (retention rationale: the proto is the codegen source for all five clients and must stay wire-identical to the package); `src/ZB.MOM.WW.MxGateway.Server/ZB.MOM.WW.MxGateway.Server.csproj:23` (`ZB.MOM.WW.GalaxyRepository` 0.2.0 — the server serves the package's implementation); `src/ZB.MOM.WW.MxGateway.Tests/Contracts/ClientProtoInputTests.cs:65-70` (walks `GalaxyRepositoryReflection.Descriptor` from *Contracts* only); package proto `~/Desktop/scadaproj/ZB.MOM.WW.GalaxyRepository/src/ZB.MOM.WW.GalaxyRepository/Protos/galaxy_repository.proto` (last changed `5572edd`, 2026-06-23).
|
||||
- **Description:** Verified at HEAD: the two files differ only in `option csharp_namespace` (comment-stripped `diff`), so the invariant holds today. Nothing keeps it true. The package version moves with the whole `ZB.MOM.WW.*` family (repo memory: a version jump is not evidence the package changed), so a routine `PackageReference` bump can bring a wire change in `galaxy_repository.v1` — a new field, a renamed RPC, a different streaming shape — that the server starts serving while the five clients keep generating from the retained, now-stale copy. Check 1–4 and the descriptor test would all stay green, because they compare the clients against the *retained* proto, never the retained proto against the *served* one. Cycle 1 recorded this as an optional test (`archreview/remediation/30-contracts-ipc.md:340-344`) when it would have required a cross-repo path; that is no longer the case — the test project transitively loads the package through the server reference (`Tests/Configuration/GalaxyRepositoryOptionsValidatorTests.cs` and four other test files already `using ZB.MOM.WW.GalaxyRepository`), so `ZB.MOM.WW.GalaxyRepository.Grpc.GalaxyRepositoryReflection.Descriptor` and `ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy.GalaxyRepositoryReflection.Descriptor` are both in-process.
|
||||
- **Recommendation:** Add `GalaxyContractParityTests`: take `FileDescriptor.ToProto()` from both, clear `SourceCodeInfo` and `Options.CsharpNamespace` (and `Name` if the paths differ), and assert byte-equality of the serialized `FileDescriptorProto`. One test, no protoc, runs in the portable CI job, and turns a silent package-bump drift into a red build with the exact differing symbol named. Record the new guard in the csproj retention comment and `docs/Contracts.md`.
|
||||
|
||||
### IPC-38 — `docs/Grpc.md` has no coverage of `DrainEvents` although it is a public `MxCommandKind` reachable through `Invoke`; the tracker's reason for omitting it is inaccurate
|
||||
|
||||
- **Severity:** Info
|
||||
- **Files:** `docs/Grpc.md` (grep for `DrainEvents` → 0 hits); `src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_gateway.proto:136` (`DrainEventsCommand drain_events = 103` in the public `MxCommand` oneof), `:520-523`, `:701-706`; `src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGrpcRequestValidator.cs:74-88` (public-boundary validation of `max_events`); `archreview/2026-07-12/remediation/00-tracking.md:87` ("`docs/Grpc.md` intentionally carries no DrainEvents row since it scopes to the seven public gRPC RPCs and DrainEvents is a worker IPC diagnostic"); `gateway.md:565-572` (does document it).
|
||||
- **Description:** `DrainEvents` is not an IPC-only diagnostic: any client with `invoke:read` can issue it through `Invoke`, the public validator has a rule for it, and the cycle-2 plan (IPC-23 step 5) asked for a rules-table note because the reply is now truncated on two axes with a drain-until-empty contract that callers must know. The tracker closed that step on a premise the proto contradicts. `gateway.md` carries the prose, so the information exists; `docs/Grpc.md` is the document the client READMEs and the rules table point readers at for per-command behaviour, and it is silent.
|
||||
- **Recommendation:** Add a `DrainEvents` row to the rules table in `docs/Grpc.md` (count cap, byte cap, `DiagnosticMessage` truncation marker, drain-until-empty, and that drained events are removed from the session's event stream), cross-linking `gateway.md:565`. Correct the tracker note.
|
||||
|
||||
### IPC-39 — The IPC-03 `CommandTooLarge` pre-check now measures the envelope before GWC-28 stamps the sequence, leaving an ≤ 11-byte window in which the writer's `MessageTooLarge` is still session-fatal; the explanatory comment has the direction of conservatism backwards for the pre-check itself
|
||||
|
||||
- **Severity:** Info
|
||||
- **Files:** `src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs:236-249` (pre-check: `CalculateSize()` on an envelope whose `Sequence` is still 0 → the `sequence` field is not encoded), `:465-471` (`WriteLoopAsync` stamps `Sequence` → up to 1 tag + 10 varint bytes added), `src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerFrameWriter.cs:42-55` (post-stamp `CalculateSize()` > max → `MessageTooLarge`), `:478-483` (write-loop exception → `SetFaulted(WriteFailed)`, session-fatal).
|
||||
- **Description:** The comment at `:236-241` correctly explains why the pre-check size cannot be reused as the writer's `knownSize` ("a lower bound … conservative in the right direction") — true for that purpose, but the same fact inverts for the pre-check's own purpose: a lower bound admits envelopes the writer will reject. An envelope in `(MaxMessageBytes − 11, MaxMessageBytes]` passes `:243` and faults the session at the write. Reachability is nil under the shipped validator: every caller-controlled command arrives through gRPC capped at `MaxGrpcMessageBytes`, which is ≥ 64 KiB below the pipe max, and gateway-authored commands are tiny — so this is a correctness-of-argument note, not a live defect. It is worth recording because the headroom rule is the only thing making it unreachable, and the comment currently reads as if the pre-check were exact.
|
||||
- **Recommendation:** Either compare against `MaxMessageBytes − MaxSequenceEncodingBytes` (11) in the pre-check, or reword the comment to say the pre-check is a lower bound that relies on the 64 KiB reserve for its safety margin. No test needed beyond the existing `CommandTooLarge` case.
|
||||
|
||||
## Severity roll-up (new findings)
|
||||
|
||||
| Severity | Count | IDs |
|
||||
|----------|:-:|-----|
|
||||
| Critical | 0 | — |
|
||||
| High | 0 | — |
|
||||
| Medium | 0 | — |
|
||||
| Low | 5 | IPC-33, IPC-34, IPC-35, IPC-36, IPC-37 |
|
||||
| Info | 2 | IPC-38, IPC-39 |
|
||||
|
||||
Prior-finding verification: 8/8 cycle-2 Done claims hold (IPC-23 with the IPC-33 residual); IPC-31 superseded by GWC-28 (recommend flipping to Resolved); cycle-1 IPC-08 substantially resolved by `7c1ea12` (recommend closing, residual tracked as IPC-34); IPC-10/11/12 confirmed still open, none regressed.
|
||||
|
||||
## Still good / preserve under change
|
||||
|
||||
The contract boundary is in the best shape it has been across three cycles. Proto evolution since baseline was purely additive with correct next-free tags and reserved ranges intact, and — for the first time — every generated artifact (C# `Generated/`, Go, Python, Java, Rust vendored copies, the protoset) moved in the *same commit* as the proto, including for a comment-only edit; the freshness net that makes this checkable is now genuinely closed (Checks 1–4 + the reflection test + the Java `git diff` gate, all in CI, all tool-missing-fails-not-skips, all with their generator pins asserted rather than assumed). The frame protocol core is unchanged where it should be — length validated before allocation on both readers, exact-read loops with typed `EndOfStream`, pooled buffers with sound lifetimes on all four codec halves, single-serialization single-write framing — and materially stronger where it changed: both sides now stamp `sequence` at the point of writing so the documented "monotonic per sender" contract is finally true on both ends, the negotiated `max_frame_bytes` handshake has matching `[1024, 256 MiB]` bounds on both sides so a legal gateway configuration can never be refused by the worker, the DrainEvents reply is bounded on count *and* bytes with no event loss at the pipe, oversized events die structured and named, and cancelled writes are guaranteed never to reach the wire. `WorkerCancel` is no longer a dead arm. `docs/WorkerFrameProtocol.md` is now a faithful, claim-by-claim description of the shipped writer, and `gateway.md`'s envelope section points at the proto as the single source of truth instead of carrying a drifting copy. Preserve in particular: the per-client generator pins and the "tool missing = check fails" rule (a skipped guard is the silent-drift hole the whole P0 codegen batch existed to close); the pipe-over-gRPC 64 KiB reserve invariant in the validator (IPC-33 and IPC-39 both rest on it); the habit of landing proto-comment changes with the full regen wave; and the csproj retention comment on `galaxy_repository.proto` — until IPC-37's test exists, that comment is the only thing standing between a package bump and a five-client wire drift.
|
||||
|
||||
## Verification needed on Windows (windev)
|
||||
|
||||
- This review is static. The worker-side halves cited here (byte-budgeted drain, IPC-30 structured death, tombstones, the floor/ceiling handshake rejection) are exercised only by `Worker.Tests -p:Platform=x86`; the tracker and repo memory record the suite fully green on windev after the harness fixes, which this review did not re-run.
|
||||
- IPC-33's end-to-end shape (reply built at budget → `Invoke` send failure) cannot be reproduced with the fake worker alone on macOS if the fix lands worker-side; the validator/gateway-side variant (option a) is portable and should get a `GatewayOptionsTests` case plus a TST-24-style wire test.
|
||||
- IPC-35's regression test is portable (fake worker writes a handshake-phase `WorkerFault`); no windev run needed.
|
||||
@@ -0,0 +1,110 @@
|
||||
# Security & Dashboard — Re-review (Cycle 3)
|
||||
|
||||
**Date:** 2026-08-19 · **Commit:** `8b3e3b3` (`main`, clean) · **Baseline:** `4f5371f` (2026-07-12 cycle) · **Method:** static review on the macOS working tree (no builds, no runtime probes, no source modifications). Every claim cites a `path:line` read at HEAD. Prior findings are the `SEC-NN` IDs from `archreview/remediation/40-security-dashboard.md` (cycle 1) and `archreview/2026-07-12/40-security-dashboard.md` / `archreview/2026-07-12/remediation/40-security-dashboard.md` (cycle 2); new findings continue from SEC-37. This domain absorbed the most new machinery since the baseline (~55 of ~247 commits touched Security/Dashboard/Configuration): the SEC-31/32 failure-limiter re-partition, SEC-33/34/36 remediations, the SEC-25 tag-ACL epic (`DashboardTags` constraints, session tag inheritance, `GroupToTag`/`UntaggedSessionVisibility`, events-hub + in-process ACL), the `ShowTagValues` three-seam closure, the secrets-store adoption (`${secret:}` expansion, `/admin/secrets`), the bounded async audit writer with retention, and the snapshot-feed/alarm-page reliability rework.
|
||||
|
||||
## Prior-finding verification
|
||||
|
||||
### Cycle-2 findings (SEC-31…SEC-36)
|
||||
|
||||
| ID | Claimed | Verified? | Evidence (file:line at HEAD) | Notes |
|
||||
|----|---------|-----------|------------------------------|-------|
|
||||
| SEC-31 | Done | **Yes** | Composite `(transport peer, key id)` partition + cross-peer aggregate + probe admission, exactly per the remediation design: `Security/Authorization/ApiKeyFailureLimiter.cs:14-27` (two-layer remarks), `:143-204` (`Check` with probe claim/compensation), `:206-235` (`RecordFailure` into both layers), `:237-276` (`Reset` reachable-while-throttled); interceptor threads the partition and reuses the check-time resolution (`GatewayGrpcAuthorizationInterceptor.cs:83-115`, `:133-137`); new knobs validated (`SecurityOptions.ApiKeyFailureAggregateLimit`/`ApiKeyFailureProbeIntervalSeconds`, wired at `ApiKeyFailureLimiter.cs:74-83`); throttle metric tagged by stage only (`Metrics/GatewayMetrics.cs:350-356`, with an explicit SEC-14/SEC-20 cardinality comment). `ApiKeyFailureLimiterTests.cs` exists. The lockout inversion is structurally closed: a correct secret always reaches the verifier via a probe slot and resets both layers. | Probe-claim compensation is version-stamped (`:341-383`), closing the same-tick stomp the follow-up commits (`5b681ee`, `acebe18`) addressed. Two new residual corners found in the rework — see **SEC-38** (aggregate-layer bypass via the per-peer key-id cap) and **SEC-39** (null==null fallback-partition reset). Neither reopens the SEC-31 lockout: they weaken bounds, they do not deny the legitimate holder. |
|
||||
| SEC-32 | Done | **Yes** | Token shape checked before a key-id partition is minted: `GatewayGrpcAuthorizationInterceptor.cs:149-193` (`TryResolveKeyId` — `mxgw` must be the entire first segment, key id ≤ 64 chars, non-empty third segment; internal for direct test pinning). Per-peer key-id cap `MaxKeyIdPartitionsPerPeer = 32` with overflow collapsing to the address's fallback partition (`ApiKeyFailureLimiter.cs:48-53`, `:393-444`). Eviction never removes load-bearing state below a 2× ceiling and prefers expired windows (`:479-548`). | The junk-token flush is closed as designed. The cap's overflow-collapse rule has a side effect the design did not anticipate — see **SEC-38**. |
|
||||
| SEC-33 | Done | **Yes** | `IsRootedForAnyPlatform` is gone; rooting is host-meaningful `Path.IsPathRooted` in the shared `Configuration/GatewayConfigPathRules.cs:14-39` (used by both validators, with the SEC-01/SEC-33 mechanism named in the doc-comment). `appsettings.json` no longer carries either Windows literal (Authentication section `:20-24` has no `SqlitePath`; Galaxy section `:79-84` has no `SnapshotCachePath`). Gateway-side Galaxy default seeded pre-bind (`GatewayApplication.cs:156-168` — the init-only-property deviation is documented in place) and validated by the new `Configuration/GalaxyRepositoryOptionsValidator.cs:44-72` registered with `ValidateOnStart` (`GatewayApplication.cs:172-176`). No stray `C:*` artifact under `src/` (`find`-verified). | Went beyond the finding: `882c7ca` added `AddIfUnderContentRoot` (`GatewayConfigPathRules.cs:93-124`) after a real 2026-08-09 production key-loss incident (store inside the renamed app directory), and the secrets store got the same absolute default treatment (`GatewayApplication.cs:201-239`). |
|
||||
| SEC-34 | Done | **Yes** | Invalidate race closed with the per-key generation counter: bump-before-evict (`Security/Authentication/CachingApiKeyVerifier.cs:181-199`), snapshot-before-inner + set-then-recheck with self-evict (`:128-141`, `:147-178`); `TryParseKeyId` correctness argument (full key id, `_` forbidden by both key-creation paths) documented at `:205-218`. Expiry window took the documented fallback per the recorded outcome: the library verification identity carries no `ExpiresUtc`, so the ≤ TTL expiry window is documented in the remarks (`:49-60`) with the donor-library ask. | Matches the remediation outcome exactly, including the deviation. Cache-key design unchanged (SHA-256 of the token, success-only). |
|
||||
| SEC-35 | N/A (doc-only) | **Yes** | `docs/GatewayConfiguration.md:282` — the "hard-stops key on the exact environment name" contract is documented (both guards fire only on `IHostEnvironment.IsProduction()`). Validator unchanged: `GatewayOptionsValidator.cs:38`, `:211-214` (LDAP), `:387-391` (DisableLogin). | Note the consequence is now live: windev runs as `Staging` (NEXT-07 relabel), so neither guard applies there — that is the posture gap promoted as **SEC-37**. |
|
||||
| SEC-36 | Done | **Yes (repo side + rotation executed)** | `appsettings.json:33` ships `"ServiceAccountPassword": "${secret:ldap/mxgateway/bind}"` — a fail-closed encrypted-store reference expanded pre-host (`GatewayApplication.cs:75-106`), not a literal. `git grep` for the retired value returns nothing in tracked files. `glauth.md` samples use placeholders; `docs/runbooks/SEC-36-ldap-credential-rotation.md` records the 2026-08-07 rotation (GLAuth `passsha256` replaced in `scadaproj` commit `aada53b`, instance recreated, dashboard `/login` verified through the real `DashboardAuthenticator` path post-NEXT-07). `<UserSecretsId>mxaccessgw-server</UserSecretsId>` present in the csproj; the blank-password validator message names both channels. | The load-bearing half (rotation) is done: the historically recoverable value no longer binds `dc=zb,dc=local`. Fully closed. |
|
||||
|
||||
### Cycle-1 residuals (still-open SEC-13…SEC-27 era)
|
||||
|
||||
| ID | Prior status | At HEAD | Evidence | Notes |
|
||||
|----|--------------|---------|----------|-------|
|
||||
| SEC-13 | Open | **Open, impact reduced** | `Diagnostics/GatewayLogRedactor.cs:11-17` still lists only `AuthenticateUser`/`WriteSecured`/`WriteSecured2` — the `WriteSecuredBulk`/`WriteSecured2Bulk` command kinds (`Grpc/MxAccessGatewayService.cs:364-381`) are still missing. However the redactor is no longer latent: `GatewayLogRedactorSeam` is registered as the telemetry `ILogRedactor` (`GatewayApplication.cs:139`) and masks **every** non-null `CommandValue` because the seam exposes no value-logging opt-in (`Diagnostics/GatewayLogRedactorSeam.cs:38-56`; honestly documented at `docs/Diagnostics.md:120`). The Bulk omission only bites if a future opt-in knob passes `valueLoggingEnabled: true`. | Fix the two missing names when (if) the opt-in knob lands; until then fail-closed. |
|
||||
| SEC-14 | Open | **Confirmed open** | `GatewayApplication.cs:347-348` — `MapZbHealth()`/`MapZbMetrics()` with no `RequireAuthorization`. | Mitigating discipline held: the new throttle counter deliberately carries no key/peer tags because "/metrics is not authenticated (open SEC-14)" (`GatewayMetrics.cs:352-354`). Note `/health/active` has off-repo consumers (family Overview dashboard), so any auth change needs cross-repo coordination. |
|
||||
| SEC-15 | Open (accepted) | **Confirmed open** | GET `/logout` still skips antiforgery, intentional (`Dashboard/DashboardEndpointRouteBuilderExtensions.cs:92-99`). | Accepted risk; unchanged. |
|
||||
| SEC-16 | Open | **Confirmed open** | `Program.cs:37-39` still maps `command.Pepper` → `MxGateway:ApiKeyPepper` (pepper on the CLI command line). | |
|
||||
| SEC-17 | Open | **Confirmed open** | `GatewayGrpcAuthorizationInterceptor.cs:27-54` overrides only unary + server-streaming. All current RPCs remain those shapes. | |
|
||||
| SEC-18 | Open | **Confirmed open** | `Dashboard/DashboardApiKeyManagementService.cs:22,289` — `"pepper unavailable"` message-substring match. | |
|
||||
| SEC-19 | Open | **Resolved** (this period) | `Security/Audit/SqliteCanonicalAuditStore.cs:23-31` — the `CREATE TABLE IF NOT EXISTS` is latched: `AuditDrainService` runs `EnsureInitializedAsync` once at startup and later operations skip the DDL round-trip (self-bootstrap kept for the CLI/DI-free paths; the latch is deliberately racy-idempotent). Landed with the `e2ac5d1`/`7b2d046` audit rework (bounded async writer, batched inserts). | |
|
||||
| SEC-21 | Open | **Resolved** (this period) | `Dashboard/DashboardSnapshotService.cs:25,38,42,287-310` — API-key summaries refresh at most every 15 s behind a zero-timeout gate with a 2 s store timeout, and the snapshot tick itself is idle-gated (`77c5731`, `DashboardSnapshotFeed` idle gate). The per-tick unconditional store read is gone. | |
|
||||
| SEC-23 | Open | **Confirmed open** | `GatewayOptionsValidator.ValidateDashboard` (`:380-455`) gained GroupToRole/GroupToTag/UntaggedSessionVisibility shape checks and the Production `DisableLogin` stop, but still ignores `AutoLoginUser`, `RequireHttpsCookie`, and `CookieName` — an explicit `CookieName` starting `__Host-` with `RequireHttpsCookie=false` is still applied unconditionally (`DashboardServiceCollectionExtensions.cs:154-158`) with no validator diagnosis. | The runtime guard against *deriving* `__Host-` without Secure is correct (`:159-162`); only the explicit-override consistency check is missing. |
|
||||
| SEC-24 | Open | **Confirmed open (narrowed)** | `Configuration/GatewayConfigurationProvider.cs:58-74` / `EffectiveDashboardConfiguration.cs:3-12` — the projection gained `ShowTagValues`, `GroupToRole`, `GroupToTag` (with comparer propagation, `ff16c69`), and `UntaggedSessionVisibility`, and the settings page renders them (`SettingsPage.razor:53-104`); it still omits `DisableLogin`/`AutoLoginUser`/`RequireHttpsCookie`/`CookieName` and the `MxGateway:Security` limiter keys the SEC-31 design asked to absorb here. Secrets stay redacted (`:20,30`). | |
|
||||
| SEC-26 | Open | **Resolved** (this period) | Retention sweep exists: `SqliteCanonicalAuditStore.cs:172` (`DELETE FROM audit_event` with a documented NULL-timestamp keep rule at `:251-253`), driven by `SecurityOptions.AuditRetentionDays` (default 90, `SecurityOptions.cs:96-99`) from the audit drain service. | |
|
||||
| SEC-27 | Open | **Confirmed open** | `DashboardSnapshotService.cs:17,114` — `GatewayStatus` is still the hardcoded `"Healthy"` constant. | |
|
||||
|
||||
### SEC-25 epic verification (the tag-ACL machinery is real and fail-closed)
|
||||
|
||||
Not a table row above because cycle 2 closed SEC-25 as "mitigation, ACL deferred"; the deferred ACL has now shipped and was reviewed as new code:
|
||||
|
||||
- **Tags come from the owning key, never the request:** `Grpc/MxAccessGatewayService.cs:36-46` (owner identity from `identityAccessor`, `owner?.EffectiveConstraints.DashboardTags` into `OpenSessionAsync`); `Sessions/GatewaySession.cs:207-212,270` (immutable ordinal-ignore-case `FrozenSet`); `Security/Authentication/ApiKeyConstraints.cs:35-44` (defensive copy on init; rides the constraints blob to avoid a schema migration; explicitly visibility-only, never data-access).
|
||||
- **ACL decision table:** `Dashboard/DashboardSessionAcl.cs:50-84` — Admin (authenticated) bypass; phantom session denied; untagged session follows `UntaggedSessionVisibility` (default `AdminOnly`, fail-closed); otherwise ordinal-ignore-case tag intersection over `DashboardTagClaimType` claims; `null` principal denied outright, even under `AllViewers` (`IDashboardSessionAcl.cs:25-32`). Anonymous localhost is an empty-grant Viewer.
|
||||
- **Both subscribe seams gated identically:** `Dashboard/Hubs/EventsHub.cs:59-79` (ACL before group join, registry-before-group ordering, opaque `HubException`), and the session-details in-process seam `SessionDetailsPage.razor:321-361` (ACL before any field write or `Subscribe`). The `7ec0b35` re-entrancy fix is sound: the attach generation is claimed synchronously before the first await (`:217-233`), a stale continuation returns without attaching or detaching (`:340-343`), and the pump's `ReferenceEquals` batch guard drops cross-session batches on the dispatcher (`:383-407`).
|
||||
- **Claims plumbing:** tags stamped at login (`DashboardAuthenticator.cs:114-158`) and re-resolved from group claims at hub-token mint so a token bounds grant staleness to 5 minutes (`HubTokenService.cs:80-98`); token `Validate` still rejects identity-less payloads (`:118-126`) and rehydrates tag claims for the hub-path ACL (`:139-145`).
|
||||
- **Owner-scoped gRPC attach (the threat-model anchor) holds:** `Grpc/EventStreamService.cs:52-63` — reattach is `PermissionDenied` unless the caller's key id ordinal-equals the session owner; null owner matches only a null caller.
|
||||
- **Redaction (second, independent layer) covers all three value seams:** events-hub mirror deep-clone (`Dashboard/Hubs/DashboardEventBroadcaster.cs:91,247-259` — top-level `value` plus alarm `CurrentValue`/`LimitValue`, source never mutated), alarms hub deep-clone on both value-bearing arms (`AlarmsHubPublisher.cs:59,103-126` — the cycle-2 closeout's named "alarms-hub redaction gap" residual, closed by `eff17d1`/TST-16 on 2026-08-17), and `/browse` live values at the service boundary with the `'-'`-placeholder-on-failed-reads fix (`DashboardLiveDataService.cs:132-140`; `1ea6f60`). `docs/GatewayConfiguration.md:195` documents all three seams and the "redaction applies to successful reads only" rule accurately.
|
||||
- **Tests exist for the corners:** `Tests/Gateway/Dashboard/DashboardSessionAclTests.cs`, `EventsHubTests.cs`, `SessionDetailsPageEventAclTests.cs`, `EventsHubViewerRegistryTests.cs`, `HubTokenServiceTests.cs`, `AlarmsHubPublisherTests.cs`, `DashboardGroupTagMappingTests.cs`; live-LDAP ACL proof via `IntegrationTests/DashboardLdapLiveTests.cs:186-243` (real bind → real group claims → GroupToTag → ACL admit/deny, plus Admin bypass).
|
||||
- **Docs honesty:** the short-name `GroupToRole`/`GroupToTag` key constraint (full-DN keys can never match the pre-stripped groups) is stated at the mapping site (`DashboardGroupTagMapping.cs:8-13,41-47`) and in the class summaries (`db7b1db`, `4b6f907`).
|
||||
|
||||
## New findings
|
||||
|
||||
### SEC-37 · Medium (Security / deployment posture) — The shared GLAuth offers no TLS, so the Production plaintext-LDAP hard-stop is unsatisfiable for every host that authenticates against it (promotion of NEXT-08)
|
||||
|
||||
**Files:** `src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs:208-214`; `docs/GatewayConfiguration.md:264-265`; `glauth.md:30,357-360`; `archreview/2026-07-12/remediation/90-candidate-findings-next-cycle.md` (NEXT-08).
|
||||
|
||||
The SEC-06 guard refuses to start a `Production`-environment host with `Ldap:Transport=None`, and the config doc instructs "Deployed hosts must set `Ldaps` or `StartTls`" — but the only shared directory (`10.100.0.35:3893`) has LDAPS disabled, port 3894 closed, and answers StartTLS with `protocolError` (per the NEXT-07/08 record), and `glauth.md:30` documents "LDAPS | disabled in dev". Neither permitted value can work, so the instruction is unsatisfiable: every deployed gateway that binds this directory must either run plaintext (which `glauth.md:358-360` itself warns sends passwords cleartext) or avoid the `Production` label. That is exactly what happened — windev was relabelled `Staging` on 2026-08-07 to boot past the guard. The net posture is that the hard-stop protects no real deployment: hosts that would trip it are renamed around it, and the SEC-35 documentation note now formalizes the escape hatch. The dashboard admin surface (API-key CRUD, session Kill, secrets UI) sits behind credentials that transit this plaintext bind.
|
||||
|
||||
**Recommendation:** resolve the posture, in one of two written directions: (a) enable LDAPS (or StartTLS) on the shared GLAuth — certificate issuance plus a trust story on each gateway host, source of truth `scadaproj/infra/glauth/` — and flip the deployed hosts' `Transport`; or (b) record an explicit decision that production gateways bind a different, TLS-capable directory and that the shared GLAuth is dev/staging-only, and say so in `docs/GatewayConfiguration.md`'s `Transport` row and `glauth.md`. Until one lands, every "must use Ldaps/StartTls" sentence is aspirational and should be labeled as such. (Cross-repo: coordinate with the family per the umbrella-index rule.)
|
||||
|
||||
### SEC-38 · Low (Security) — The failure limiter's cross-peer aggregate layer can be sidestepped by pre-filling the per-peer key-id cap, restoring the `limit × addresses` guessing bound the layer was built to remove
|
||||
|
||||
**Files:** `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs:225-231` (aggregate fed only when `effectiveKeyId` is non-null), `:393-444` (`ResolvePartitionKey` returns the `(peer, null)` fallback once the address has minted `MaxKeyIdPartitionsPerPeer = 32` key ids), `:153-166` (`Check` consults the aggregate only via `effectiveKeyId`).
|
||||
|
||||
The SEC-32 cap collapses the 33rd-and-later key id from one address onto that address's fallback partition, and — per the comment's cardinality rationale — such failures deliberately do not feed the per-key-id aggregate. The side effect: an attacker who first mints 32 junk `mxgw`-shaped key-id partitions from an address (32 requests) makes every subsequent guess against the *victim's real key id* from that address resolve to `(peer, null)` — so those failures neither feed the victim's aggregate (`RecordFailure`, `:227`) nor consult it (`Check`, `:158-163`). A source-rotating attacker (the IPv6-/64 scenario the aggregate was designed for, per the SEC-31 remediation design's G1 analysis) thereby recovers `ApiKeyFailureLimit` guesses per window per address with the aggregate fully bypassed, at ~32 extra requests per address — instead of the designed bound of `ApiKeyFailureAggregateLimit` total followed by ~1 probe per interval globally. Severity is Low, not Medium: gateway secrets are library-generated high-entropy values, so online guessing remains computationally hopeless and the limiter is defense-in-depth plus store-read cost shielding (which still holds per-address); the finding is that the *documented* distributed-attack bound (`ApiKeyFailureLimiter.cs:17-19`, `docs/Authentication.md` limiter section) is not the actual bound.
|
||||
|
||||
**Recommendation:** feed (and consult) the aggregate for any *validly shaped* key id regardless of whether it earned a composite partition — the cardinality concern the comment cites is already handled by `_aggregates`' own capped eviction (`:234`), whose policy protects over-limit entries and preferentially evicts single-failure junk. Alternatively, count cap-overflow failures into the aggregate of the presented key id only when that aggregate already exists (zero new cardinality, closes the consult-bypass for hot keys). Update the class remarks and `docs/Authentication.md` either way.
|
||||
|
||||
### SEC-39 · Info (Security) — `Reset` treats "no key id" as owning the shared fallback partition: a verified token that fails the interceptor's key-id parse clears the address's fallback state the guard comment promises to preserve
|
||||
|
||||
**Files:** `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs:252-276`; `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcAuthorizationInterceptor.cs:149-193`.
|
||||
|
||||
`Reset`'s ownership check is `string.Equals(effectiveKeyId, partition.KeyId)` (`:264`), which is true when both are null — and a partition with `KeyId = null` resolves to the transport-peer fallback key (`:398-403`), so the reset then `RemovePartition(peer)`s the *shared* fallback bucket, the exact wipe the comment above it (`:259-263`) explains must not happen because that bucket holds junk-spray failures from the same address. A null `KeyId` on a *successful* verification requires a token the library verifier accepts but `TryResolveKeyId` rejects — the identifiable shapes are a key id longer than 64 chars (`:178`) and a secret whose first character is `_` (`:187`, matching the old `Split`-based rejection); whether the shared verifier accepts either depends on library parsing not readable in this repo. Practical impact is minimal (the attacker must hold a *valid* key of an unusual shape merely to clear their own address's junk-spray throttle), hence Info — but the null==null branch contradicts the stated invariant and costs one line to close.
|
||||
|
||||
**Recommendation:** make the fallback non-resettable explicitly: `if (effectiveKeyId is not null && string.Equals(effectiveKeyId, partition.KeyId, StringComparison.Ordinal))`, and pin with a test (successful reset with a null-key-id partition leaves the fallback partition tracked).
|
||||
|
||||
### SEC-40 · Low (Security) — Alarm visibility is not tag-scoped: the session-less alarms hub (and alarms page) broadcasts galaxy-wide alarm metadata to every Viewer, including empty-grant Viewers and anonymous localhost
|
||||
|
||||
**Files:** `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/AlarmsHub.cs:12-25` (`[Authorize(HubClientsPolicy)]`, auto-join `__alarms__` on connect); `AlarmsHubPublisher.cs:48-63` (unfiltered `StreamAsync(alarmFilterPrefix: null)` re-broadcast to all clients); `Dashboard/DashboardAuthorizationHandler.cs:39-53` (loopback/disabled bypasses satisfy the Viewer requirement the hub policy uses); `docs/GatewayDashboardDesign.md:277` ("all clients receive every message").
|
||||
|
||||
The SEC-25 ACL scopes *session event* visibility by tag intersection, and CLAUDE.md/docs describe dashboard event visibility as tag-scoped — but the alarm feed is a parallel, session-less surface with no per-caller scoping at all: any principal satisfying `HubClientsPolicy` (any Viewer role, the `Authentication:Mode=Disabled` bypass, or anonymous localhost under the default `AllowAnonymousLocalhost=true`) receives every `AlarmFeedMessage` the central monitor emits. Values are stripped by default (`ShowTagValues`, verified above), but the surviving metadata — alarm reference (a full tag path), state, severity, category, area, and acknowledging-operator usernames — discloses process structure and operator activity for the whole subscription expression to callers the tag model would deny every single session. A Viewer with an empty tag grant, whom the ACL was explicitly designed to show nothing, still sees the full plant alarm picture. The alarm monitor being gateway-central (not per-session) means tag scoping does not map mechanically, so this may well be a deliberate posture — but no design doc states it against the ACL: `GatewayDashboardDesign.md:277` describes the fan-out without a scoping rationale, and the ACL plan (`docs/plans/2026-07-10-dashboard-session-acl-tst15.md`) does not carve alarms out.
|
||||
|
||||
**Recommendation:** decide and document. If deliberate, add one paragraph to the ACL section of `docs/GatewayDashboardDesign.md` (and the `GroupToTag` row of `docs/GatewayConfiguration.md`) stating that alarm metadata is intentionally visible to every dashboard Viewer and why (central monitor, no per-session provenance). If not, the cheap tightening is a dedicated policy on `AlarmsHub` (e.g. require a real role claim, excluding the anonymous-loopback bypass) or an area/tag-prefix filter derived from the caller's grant.
|
||||
|
||||
### SEC-41 · Low (Security) — Tag grants revoked by a `GroupToTag` config change survive on cookie principals across the restart that applies the change: the ACL trusts login-stamped tag claims for up to the 8-hour sliding cookie life, while the hub-token path re-resolves at mint
|
||||
|
||||
**Files:** `src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardAuthenticator.cs:145-149` (tags stamped as claims at login); `Dashboard/DashboardSessionAcl.cs:75-81` (ACL reads the *stamped* tag claims); `Dashboard/HubTokenService.cs:84-90` (token mint *re-resolves* tags from group claims against current options — the asymmetry); `Dashboard/DashboardServiceCollectionExtensions.cs:114-130` (`ZbCookieDefaults.Apply(..., idleTimeout: 8h)` — sliding, so an active session renews indefinitely).
|
||||
|
||||
`GatewayOptions` binds once at startup (no reload — noted at `DashboardSnapshotService.cs:45-49`), so changing `GroupToTag` requires a gateway restart. A restart tears down circuits and hub connections, but **not** the auth cookie: data-protection keys persist, and a browser returning after the restart re-authenticates from the cookie, whose `DashboardTagClaimType` claims were resolved under the *old* map at login. `DashboardSessionAcl` consults exactly those claims, so a user whose group's tag grant was removed from config keeps observing the corresponding sessions' event streams — hub joins and the session-details in-process seam alike — until they log out or the sliding 8-hour window finally lapses, renewable indefinitely by activity. The hub-token seam does not have this gap for the *grant* (mint re-resolves from group claims, bounding staleness to 5 minutes, exactly as its comment at `:41-47` claims) — but that comment's promise that "revoking a GroupToTag entry takes effect ... within one lifetime" holds only for token-authenticated connections, and the cookie-authenticated in-process seam it shares an ACL with is the weaker path. LDAP group-membership revocations have the same 8-hour cookie bound on both paths (standard cookie-auth behavior, but worth stating for a *security grant*).
|
||||
|
||||
**Recommendation:** have the ACL (or a claims transformation) re-resolve tags from the principal's `LdapGroupClaimType` claims against current options — the same one-call `DashboardGroupTagMapping.MapGroupsToTags` the token mint already uses — instead of trusting login-stamped tag claims; the group claims are already on the cookie. That makes a config-revoked grant take effect at the next subscribe after restart, symmetrically with the token path. At minimum, document the revocation bounds (restart + re-login for cookies, 5 minutes for tokens, never for an already-open subscription) in `docs/GatewayDashboardDesign.md`'s ACL section.
|
||||
|
||||
## Severity roll-up (new findings)
|
||||
|
||||
| Severity | Count | IDs |
|
||||
|----------|-------|-----|
|
||||
| Medium | 1 | SEC-37 |
|
||||
| Low | 3 | SEC-38, SEC-40, SEC-41 |
|
||||
| Info | 1 | SEC-39 |
|
||||
|
||||
Prior-backlog movement this period: SEC-19, SEC-21, SEC-26 resolved; SEC-31…SEC-36 all verified Done (SEC-36 including the operator rotation); still open from cycle 1: SEC-13 (impact reduced — redactor now wired fail-closed), SEC-14, SEC-15 (accepted), SEC-16, SEC-17, SEC-18, SEC-23, SEC-24 (narrowed), SEC-27.
|
||||
|
||||
## Still good / preserved
|
||||
|
||||
The load-bearing controls all survive at HEAD, several strengthened:
|
||||
|
||||
- **Fail-closed authorization core:** unrecognized request → `admin` scope (`GatewayGrpcScopeResolver`); opaque auth failures that never disclose the failing stage (`GatewayGrpcAuthorizationInterceptor.cs:96-109`); constant-time compare stays inside the shared verifier on every cache miss with only SHA-256 digests in memory (`CachingApiKeyVerifier.cs:36-39,284-286`); the throttle message reveals nothing about secret validity.
|
||||
- **The SEC-25 ACL is genuinely fail-closed on every branch** (unknown session, empty grant, untagged-session default `AdminOnly`, null principal), enforced identically on both subscribe seams, with the decision-table corners and the live-LDAP group-shape proof pinned by tests; session tags are immutable and inherited only from the owning key's constraints, never the client request; the gRPC event-stream attach stays owner-scoped (`EventStreamService.cs:52-63`).
|
||||
- **Read-only environment bypasses stay read-only:** the `Disabled`-mode and anonymous-loopback bypasses satisfy only requirements containing Viewer, never `AdminOnly` (`DashboardAuthorizationHandler.cs:37-53`); API-key CRUD and session Close/Kill keep their service-layer Admin re-checks (`DashboardApiKeyAuthorization.cs:10-18`, `DashboardSessionAdminService`) beneath the policy layer; the secrets UI mounts behind its own `secrets:manage` policy (`GatewayApplication.cs:147-150`), which the dashboard bypass handler cannot satisfy (different requirement type).
|
||||
- **Value redaction is now a complete, tested, three-seam system** (events mirror, alarms hub, `/browse`), clone-only so gRPC clients and the replay ring are untouched, with the config doc telling the truth about all three seams and the failed-read placeholder rule.
|
||||
- **Secret handling improved materially:** the committed LDAP literal is gone and rotated; config secrets ride fail-fast `${secret:}` references expanded pre-host; the effective-config projection redacts pepper name and LDAP password; hub tokens remain 5-minute, data-protected, identity-required; no new call site logging credentials, tag values, or grant contents was found in the changed files (the ACL logs decisions, never tag values — `DashboardAuthenticator.cs:145-146`).
|
||||
- **Credential-store paths are now defended in depth:** host-meaningful rooting, the under-content-root rule born of a real outage, and absolute defaults for all three stores (auth DB, secrets store, Galaxy snapshot) via one shared rule set (`GatewayConfigPathRules`).
|
||||
- **Dashboard hygiene held:** local Bootstrap only (no UI component libraries in any new page); antiforgery on POST login/logout; `SanitizeReturnUrl` still blocks open redirects (`DashboardEndpointRouteBuilderExtensions.cs:238-249`); the login limiter stays remote-IP-partitioned; the viewer registry / snapshot feed / broadcaster concurrency reworks (`EventsHubViewerRegistry.cs`, `DashboardSnapshotFeed.cs`, `DashboardEventBroadcaster.cs`) are careful, generation-guarded code with the subtle orderings documented in place and covered by dedicated tests.
|
||||
@@ -0,0 +1,104 @@
|
||||
# Language Clients — Architecture Review (Cycle 3)
|
||||
|
||||
**Date:** 2026-08-19 · **Commit:** `8b3e3b3` (main) · **Baseline:** `4f5371f` (2026-07-12 cycle) · **Method:** static review on macOS (no builds, no tests run)
|
||||
|
||||
Scope: `clients/dotnet` (lib + CLI + tests), `clients/go` (mxgw-go), `clients/java`, `clients/python`, `clients/rust` (lib + CLI + build.rs + vendored protos), cross-client parity, `docs/CrossLanguageSmokeMatrix.md`, `docs/ClientLibrariesDesign.md`, `docs/ClientPackaging.md`, `docs/ClientBehaviorFixtures.md`, client READMEs, `scripts/pack-clients.ps1` / `scripts/tag-go-module.ps1`. Prior findings tracked as CLI-01..34 (`archreview/remediation/00-tracking.md`) and CLI-35..45 (`archreview/2026-07-12/remediation/50-clients.md` + `00-tracking.md`), plus NEXT-02 (`archreview/2026-07-12/remediation/90-candidate-findings-next-cycle.md`).
|
||||
|
||||
Since 4f5371f the client family absorbed: the full CLI-35..45 remediation train, NEXT-02, the 0.2.0/0.2.1 family publish (2026-08-07, `5b153da`), TST-24 real-wire tests for .NET and Python (`a8f86b5`), alarm truncation-flag regen (`d9ea8a8`), the feed-level `snapshot_status` rollout (`c748361` + `4d4f443`), StreamAlarms open-sequence doc truth (`c3c603f`), and Gradle-9 survival for `checkGeneratedClean` (`df45cb4`).
|
||||
|
||||
## Prior-finding verification
|
||||
|
||||
### 2026-07-12 cycle (CLI-35..45) and NEXT-02
|
||||
|
||||
| ID | Claimed status | Verified? | Evidence (file:line at HEAD) | Notes |
|
||||
|----|----------------|-----------|------------------------------|-------|
|
||||
| CLI-35 | Done | **Yes** | `clients/python/src/zb_mom_ww_mxgateway_cli/commands.py:24` (imports `ReplayGap`), `:1660-1666` (`_event_row` branches on `isinstance(item, ReplayGap)` and emits `{"replayGap": {"requestedAfterSequence": …, "oldestAvailableSequence": …}}`); regression `clients/python/tests/test_cli.py:935` (`test_stream_events_renders_replay_gap`, asserts exit 0, the gap row at index 0, the normal event row after, `:971-974`) | Exactly the designed fix, camelCase row matching the cross-CLI shape. |
|
||||
| CLI-36 | Done | **Yes** | `clients/go/cmd/mxgw-go/main.go:1012-1029` (`if result.IsReplayGap()` → JSON `replayGapCursors` struct `:70`, text `REPLAY_GAP requested_after=%d oldest_available=%d`); test `clients/go/cmd/mxgw-go/main_test.go:734` (`TestRunStreamEventsPrintsReplayGap` over a real fake gateway `:698-713`, both output modes) | Gap counted toward `-limit` as designed. |
|
||||
| NEXT-02 | Resolved 2026-08-10 | **Yes** | .NET `clients/dotnet/ZB.MOM.WW.MxGateway.Client.Cli/MxGatewayClientCli.cs:1425-1429` (stream loop branch), `:1845-1846` (`EventToJsonElement` inside `--json` aggregate), `:1858-1873` (`FormatReplayGapRow` — hand-built so cursors are JSON numbers, not proto-JSON uint64 strings); Java `clients/java/zb-mom-ww-mxgateway-cli/.../MxGatewayCli.java:1665-1674` (`replayGap` JSON row + `REPLAY_GAP` text line); tests `MxGatewayCliTests.java:909-937`, `clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/MxGatewayClientCliTests.cs` (grep `FormatReplayGapRow`/`replayGap` hits); `docs/CrossLanguageSmokeMatrix.md:43-67` now records one converged contract with a per-CLI table | All five CLIs emit the same two camelCase keys with numeric cursors; the matrix doc explains *why* the row is hand-typed (proto3 JSON stringifies uint64). Genuinely converged. |
|
||||
| CLI-37 | Done | **Yes — all five conformant on `category`** | .NET `MxStatusProxyExtensions.cs:15-20` (`Category is MxStatusCategory.Ok` only); Go `clients/go/mxgateway/status.go:15-17` (`nil ∥ category == OK`, doc `:7-14` states the diagnostics-only rule); Java `MxStatuses.java:33-35` (+ `success()` Javadoc `:55-57` marks it diagnostic); Python `errors.py:144-152`; Rust `error.rs:407-413` (`status.category != Ok as i32`, doc `:383-388` covers the unspecified-category-fails edge); shared fixtures `clients/proto/fixtures/behavior/command-replies/write.status-category-error-success-set.reply.json` + `write.status-category-ok-success-zero.reply.json`, registered in the manifest, documented in `docs/ClientBehaviorFixtures.md` (Reply Validation Conformance table), and consumed by all five suites (`MxCommandReplyExtensionsTests.cs`, `protofixtures_test.go`, `MxGatewayFixtureTests.java`, `test_errors.py`, `client_behavior.rs`) | The 4-vs-1 divergence is gone; the edge decisions (absent entry = success, present-unspecified = failure) are uniform and documented. |
|
||||
| CLI-38 | Done | **Yes — all five on `hresult < 0`** | .NET `MxCommandReplyExtensions.cs:38` (`HasHresult && Hresult < 0`); Go `errors.go:299` (`GetHresult() < 0`); Java `MxGatewayErrors.java:52` (`getHresult() < 0`); Python `errors.py:137`; Rust `error.rs:407`; fixtures `write.hresult-s-false.reply.json` (S_FALSE=1 → success) + `write.hresult-e-fail.reply.json` (E_FAIL → failure) wired into all five suites | Lands prior-cycle CLI-08 (cycle-1 tracker row updated to Done via CLI-38 — verified). `docs/ClientLibrariesDesign.md:112-125` is now *true*, and states both rules explicitly. |
|
||||
| CLI-39 | Done | **Yes (bump + both guards); pattern has already recurred → CLI-46** | Versions: `clients/rust/Cargo.toml:3`/`:28` = 0.2.0, `clients/python/pyproject.toml:9` + `version.py:3` = 0.2.0 (+ new drift test `test_cli.py:17` `test_version_matches_pyproject_toml`, closing the CLI-26 residual), `clients/go/mxgateway/version.go` = 0.2.0, `.../ZB.MOM.WW.MxGateway.Client.csproj:22` = 0.2.0, Contracts `.csproj:10` = 0.2.0 (`440e7cf`), Java `build.gradle` = 0.2.1. Guards: `scripts/tag-go-module.ps1:40-53` (tag fails on `ClientVersion` mismatch — the CLI-21 guard finally implemented), `scripts/pack-clients.ps1:91-156` (`Test-GiteaPackageExists` + hard abort "never force-overwrites"). Publish record: `docs/ClientPackaging.md:102-113` (2026-08-07: nuget/pypi/cargo 0.2.0, maven 0.2.1 with the rationale for the Java exception, Go tag `clients/go/v0.2.0` — tag exists in this repo) | Executed to the letter, including the doc's "bump before every publish" rule and the collision-guard rationale. However the *same hazard is live again at HEAD*: post-publish library changes (incl. a shipped Python `connect()` fix) sit at the already-published 0.2.0/0.2.1 — see **CLI-46**. |
|
||||
| CLI-40 | Done | **Yes** | Java `MxGatewaySecrets.java:62-78` (`redactExact`); .NET `MxGatewaySecretRedaction.cs` (`Redact`/`RedactReply:55-68`/`RedactStatus:81-89`/`RedactStatuses` — scrubs message *and* the structured reply accessors, the `0d874f9` follow-up); Rust `error.rs:208-231` (`MxAccessError.secrets` + `with_secrets`, hand-written `Debug` prints only the count `:202-204`); fixtures `authenticate-user.echoed-credential.reply.json` + `.echoed-credential-mxaccess-failure.reply.json` in the manifest, consumed by all five suites (`MxGatewaySessionReplyContractTests.cs`, `command_reply_fixtures_test.go`, `MxGatewayCredentialReplyTests.java`, `test_malformed_reply.py`/`test_errors.py`, `client_behavior.rs`) | The echoed-credential fixture is exactly the test the prior review said was missing ("the fakes don't echo"). |
|
||||
| CLI-41 | Done | **Yes** | Go `errors.go:164-176` (`MalformedReplyError`); Python `errors.py:56`; Java `MxGatewayMalformedReplyException.java` + throw sites `MxGatewaySession.java:889/:917/:948` (redaction rethrow preserves the type `:1143-1144`); .NET `MxGatewayMalformedReplyException.cs`; Rust `session.rs:1024-1042` (payload → present `return_value` → `Error::MalformedReply`, now uniform across its extractors); fixtures `authenticate-user.missing-payload.reply.json` + `authenticate-user.return-value-only.reply.json`; contract documented in `docs/ClientLibrariesDesign.md` ("Malformed-reply extraction" paragraph) | Silent-0 user ids and the .NET NRE are gone; the payload → return_value → typed-error ladder is uniform. |
|
||||
| CLI-42 | Done | **Yes** | `clients/rust/README.md:24-36` (repo-path-first / vendored-fallback resolution, same-commit refresh rule, both-direction drift guard); `docs/ClientPackaging.md:199-213` (same, plus why `cargo package` runs *with* verification); `711793f` extended check-codegen Check 3 to sweep both directions (canonical proto missing a vendored copy now fails) | Vendored copies verified byte-identical to Contracts at HEAD (all three protos, diff clean). |
|
||||
| CLI-43 | Done | **Yes** | `docs/style-guides/JavaStyleGuide.md:8` — "Target Java 17 (the Ignition 8.3 baseline; the client build enforces …)" | `grep "Java 21"` in style-guides: clean. |
|
||||
| CLI-44 | Done | **Yes** | `clients/go/mxgateway/session.go:1077` (Recv-error path uses `sendTerminalEventResult`), `:1096-1111` (bypasses the overflow substitution via the reserved slot) | Root cause of a genuine stream error is no longer relabeled `ErrSlowConsumer`. |
|
||||
| CLI-45 | Done | **Yes** | Canonical env `MXGATEWAY_VERIFY_PASSWORD` in all five: Go `main.go:452` (default) + `:483-491` (fail-fast before dialing, message names flag+var only); Java `MxGatewayCli.java:194` + `:1166-1179` (`ParameterException` on null/blank); Rust `main.rs:151` (clap default) + test `:2644-2660`; Python `commands.py:38` (`DEFAULT_VERIFY_PASSWORD_ENV`); .NET `MxGatewayClientCli.cs:353-359` (canonical default + `MXGATEWAY_VERIFY_USER_PASSWORD`/`--verify-user-password*` kept as deprecated aliases, resolution order documented `:385-386` and in help `:2455-2456`). Tests: Go `main_test.go:605` (`RejectsEmptyPassword`) + `:632` (`ReadsPasswordFromCanonicalEnv`). Contract documented at `docs/CrossLanguageSmokeMatrix.md:90-119`, incl. the parity rationale (libraries still transmit whatever they are given) and the .NET deprecation window | One exported variable now drives all five CLIs; empty credential is a usage error everywhere. |
|
||||
|
||||
**Done claims that don't fully hold:** none. Every 2026-07-12 remediation claim checked out at HEAD with the designed shape (fixtures-first conformance, docs in the same commit, guards actually implemented — including the CLI-21 tag guard skipped last cycle). This is the cleanest verification pass of the three cycles.
|
||||
|
||||
### Cycle-1 residue (CLI-05..33, previously Open)
|
||||
|
||||
Re-verified briefly at HEAD; all unchanged unless noted:
|
||||
|
||||
- **CLI-08 → Done** via CLI-38 (see above); cycle-1 tracker row correctly updated.
|
||||
- Still open, unchanged: **CLI-05** (`MxGatewaySession.cs:19` internal ctor, no .NET `AttachSession`), **CLI-06** (`:1501-1505` `DisposeAsync` → unbounded `CloseAsync`), **CLI-07** (`MxGatewayClient.cs:311`), **CLI-09** (no typed auth errors in Go `errors.go`), **CLI-10** (`client.go:64/:68` `WithBlock`/`DialContext`), **CLI-11** (library has `RequireCertificateValidation` (`client.go:239`) but `mxgw-go` still exposes no flag), **CLI-13** (`MxGatewayClient.java:248` hardcoded 16), **CLI-14** (no TOFU runtime warning in `options.py`), **CLI-17/20** (TLS posture divergence: .NET accept-all `MxGatewayClient.cs:369`, Go skip-verify, Java `InsecureTrustManagerFactory` `MxGatewayClient.java:391`, Python TOFU, Rust strict), **CLI-19** (duplicate `InternalsVisibleTo`), **CLI-22** (`session.go:1176-1181` empty correlation id on rand failure), **CLI-23** (nil-vs-empty asymmetry — now at least documented in the `WriteBulk` doc comment `session.go:431-432`), **CLI-25** (`close()` doesn't await termination `MxGatewayClient.java:347-351`), **CLI-27** (`session.py` close still lock-free), **CLI-28** (bottom-of-file import `session.py`), **CLI-31** (`mxgw-cli/src/main.rs` now **2,991** lines — grew again; the other CLIs sit at 1,668–2,478), **CLI-32** (no .NET/Java client-side bulk cap), **CLI-33** (no backpressure section in `docs/ClientBehaviorFixtures.md`).
|
||||
|
||||
## New findings
|
||||
|
||||
### CLI-46 — Version constants again equal the already-published 0.2.0/0.2.1 while post-publish changes shipped — including the fix for a published-broken Python `connect()`
|
||||
**Severity:** Medium · **Category:** packaging / release hygiene
|
||||
**Files:** `clients/python/pyproject.toml:9` + `clients/python/src/zb_mom_ww_mxgateway/version.py:3` (0.2.0 = the pypi artifact published 2026-08-07, `docs/ClientPackaging.md:102-113`); `clients/python/src/zb_mom_ww_mxgateway/client.py:61-72` (the fixed `connect()`); post-publish library commits `a8f86b5` (2026-08-10), `d9ea8a8`, `c748361` (2026-08-17/18); guard `scripts/pack-clients.ps1:145-156`
|
||||
The published pypi 0.2.0 Python client cannot open a real connection at all: its `connect()` built the `grpc.aio` channel inside `asyncio.to_thread`, and a `grpc.aio` channel binds to the event loop current on the constructing thread, so every non-stub connection raised "There is no current event loop in thread" (TST-24's first real-wire run caught it; fixed in-tree by `a8f86b5` — `resolve_channel_security` runs off-loop, `create_channel` on-loop). That fix has sat unpublished for nine days while the version constant still reads 0.2.0. Since the publish, all five libraries also gained wire-surface additions at the published numbers (alarm truncation fields `d9ea8a8`, feed-level `snapshot_status` bindings `c748361`; Java at its published 0.2.1). This is precisely the CLI-39 pattern one cycle later — now *mitigated* (the pack-script collision guard refuses a same-version publish, and the bump-together deferral is deliberate) but the live registry keeps serving a broken Python client, and nothing in the tree records that 0.2.0-as-published ≠ 0.2.0-as-source.
|
||||
**Recommendation:** cut the next release train soon (0.2.1 four clients / 0.2.2 Java, or 0.3.0 across the board), prioritizing the Python publish; until then, note the known-broken `connect()` against pypi 0.2.0 in `clients/python/README.md`/`docs/ClientPackaging.md` (or yank the pypi artifact). Consider bumping version constants in the same commit as the first post-publish library change rather than at publish time — the collision guard then enforces nothing because nothing needs catching.
|
||||
|
||||
### CLI-47 — `snapshot-status`/`provider-status` rendering diverges: .NET text mode hides the flag on the frames that matter most; Rust's JSON is the lone always-explicit shape
|
||||
**Severity:** Low · **Category:** cross-client parity / CLI
|
||||
**Files:** .NET `MxGatewayClientCli.cs:15` (`JsonFormatter.Default` — omits default-valued proto3 fields) + `:1563-1566` (text arms format the payload through it); Go `main.go:1112-1114` (`snapshot-status truncated=%t`); Java `MxGatewayCli.java:2309-2312` (`truncated=%b`) + `protoJson` `:2362-2368` (default printer); Rust `main.rs:2262-2264` (summary) + `:2314-2318` (hand-built JSON, always includes `"truncated"`); Python CLI renders generic `MessageToDict`
|
||||
The c748361 rollout achieved surface parity (all five render the new arm) but not rendering parity. The `snapshot_status` clearing frame — `truncated = false`, emitted on monitor restart, explicitly called out by the commit and `docs/Grpc.md` as a frame consumers must see — renders in the .NET CLI's text mode as `snapshot-status {}` (proto3 JSON omits the false bool), while Go/Java/Rust print `truncated=false` explicitly. The same applies to a healthy `provider-status` (`degraded=false` invisible in .NET text). In JSON mode the split flips: Rust's hand-built row always carries `"truncated"`, while .NET/Go/Java/Python protobuf formatters emit `{"snapshotStatus": {}}` for the clearing frame. Semantically recoverable (absent = false), but the same class of divergence NEXT-02 just cleaned up for `ReplayGap`, where the cross-CLI row was deliberately hand-specified.
|
||||
**Recommendation:** hand-format the .NET text arms like the other three (`snapshot-status truncated={bool}`, `provider-status mode=… degraded=… reason=…`); decide one JSON shape (either Rust adopts formatter-omission or the matrix documents Rust's explicitness the way `CrossLanguageSmokeMatrix.md:54-67` documents the ReplayGap row's deliberate choices).
|
||||
|
||||
### CLI-48 — The new alarm-feed renderer arms are untested in three of the four CLIs that changed; the Go CLI has no `stream-alarms` test at all
|
||||
**Severity:** Low · **Category:** testing coverage
|
||||
**Files:** `clients/go/cmd/mxgw-go/main_test.go` (test inventory `:16-734` — no stream-alarms/`formatAlarmFeedMessage` test of any kind); .NET `MxGatewayClientCliTests.cs:671-706` (`stream-alarms` test feeds only `ActiveAlarm` + `SnapshotComplete`); Java `MxGatewayCliTests.java:653-687` (asserts `active-alarm`/`snapshot-complete`/`transition` arms only); Rust `main.rs:2986` (the one snapshot-status render test, added by `4d4f443`)
|
||||
The review-nits commit covered only the Rust render; the .NET/Go/Java `provider-status` and `snapshot-status` arms shipped with zero assertions. The .NET `{}`-rendering divergence (CLI-47) is exactly what a text-mode assertion on a `truncated=false` frame would have caught — cycle 2's lesson ("each new epic achieves surface parity but re-introduces behavioral divergence") repeating at the test layer.
|
||||
**Recommendation:** one fake-fed `stream-alarms` test per CLI whose feed includes a `provider_status` and a `snapshot_status` frame (including the false/clearing shape), asserting the rendered row in both output modes; for Go this also creates the missing `stream-alarms` scaffold.
|
||||
|
||||
### CLI-49 — Go library exposes no aliases for `AlarmProviderStatus`/`AlarmSnapshotStatus`, whose generated types live in an `internal` package; the `AlarmFeedMessage` alias doc names three of five payload arms
|
||||
**Severity:** Low · **Category:** code organization & conventions / documentation
|
||||
**Files:** `clients/go/mxgateway/types.go:129-137` (alias block: `AlarmFeedMessage`, `ActiveAlarmSnapshot`, `OnAlarmTransitionEvent` — but not the two status payloads; `ReplayGap` got its alias at `:38` when it shipped); generated types under `clients/go/internal/generated` (Go `internal` visibility)
|
||||
Because the generated package is `internal`, a Go consumer cannot name `*generated.AlarmSnapshotStatus`: they can call `message.GetSnapshotStatus().GetTruncated()` on an inferred value but cannot declare a variable, struct field, or function parameter of the type — unlike every other StreamAlarms payload, which `types.go` deliberately aliases for exactly this reason. (`AlarmProviderStatus` has had the same gap since provider-mode shipped.) The `AlarmFeedMessage` alias comment (`types.go:130-132`) still describes the oneof as "an active-alarm snapshot row, a snapshot-complete sentinel, or a transition" — stale by two arms.
|
||||
**Recommendation:** add `AlarmProviderStatus = pb.AlarmProviderStatus` and `AlarmSnapshotStatus = pb.AlarmSnapshotStatus` aliases and refresh the `AlarmFeedMessage` comment to name all five arms.
|
||||
|
||||
### CLI-50 — Python CLI `stream-alarms` default (`--max-messages 1`, collect-then-print) now returns only the provider-status preamble — the README's own example fetches no alarm data
|
||||
**Severity:** Low · **Category:** cross-client parity / CLI usability
|
||||
**Files:** `clients/python/src/zb_mom_ww_mxgateway_cli/commands.py:514` (`--max-messages` default 1), `:1539-1551` (`_collect_alarm_messages` — bounded collect, printed once at the end), `:1122-1134`; `clients/python/README.md:451` (`mxgw-py stream-alarms --max-messages 1 --json`); feed order per `docs/Grpc.md`/`c3c603f`: `provider_status` → `snapshot_status` → cached `active_alarm` → `snapshot_complete`
|
||||
The other four CLIs stream rows as they arrive and default to unbounded (`--limit 0` / `--max-events 0`). Python is the outlier twice over: it buffers then prints (so it cannot tail a live feed at all), and its default of 1 — reasonable when the first frame was an alarm — now returns exactly one `provider_status` frame, since the feed gained a two-frame status preamble. The README example therefore demonstrates a command that yields no alarm rows. Pre-existing shape, but the preamble growth this cycle changed what the default actually returns.
|
||||
**Recommendation:** raise the default (or document that N status frames precede the data) and update the README example; longer term consider printing rows incrementally like the other four CLIs.
|
||||
|
||||
## Severity roll-up (new findings)
|
||||
|
||||
| Severity | Count | IDs |
|
||||
|---|---|---|
|
||||
| High | 0 | — |
|
||||
| Medium | 1 | CLI-46 |
|
||||
| Low | 4 | CLI-47, CLI-48, CLI-49, CLI-50 |
|
||||
|
||||
## Cross-client parity note (updated)
|
||||
|
||||
The 2026-07-12 divergence table has largely collapsed. Converged this cycle, with shared fixtures pinning the bytes: **HRESULT `< 0`** (CLI-38), **status verdict = `category != OK`** (CLI-37), **exact-secret scrub incl. echoed diagnostics** (CLI-40), **malformed-reply ladder** (CLI-41), **CLI ReplayGap row** (CLI-35/36 + NEXT-02, all five, same keys, numeric cursors), **credential env-var contract** (CLI-45). What remains divergent:
|
||||
|
||||
| Behavior | .NET | Go | Java | Python | Rust |
|
||||
|---|---|---|---|---|---|
|
||||
| Version vs published artifact (CLI-46) | 0.2.0 = shipped | 0.2.0 = shipped | 0.2.1 = shipped | 0.2.0 = shipped (**broken `connect()` on pypi**) | 0.2.0 = shipped |
|
||||
| CLI text render of snapshot/provider status (CLI-47) | proto-JSON, flag invisible when false | explicit `truncated=…` | explicit | n/a (JSON only) | explicit ✓ |
|
||||
| CLI JSON shape of `snapshot_status` (CLI-47) | omits false | omits false | omits false | omits false | always explicit |
|
||||
| New renderer arms tested (CLI-48) | no | no (no stream-alarms test) | no | n/a (generic dump) | yes ✓ |
|
||||
| `stream-alarms` consumption model (CLI-50) | stream, unbounded default | stream, unbounded | stream, unbounded | **collect, default 1** | stream, unbounded |
|
||||
| Session re-attach (CLI-05) | **No** | Yes | Yes | Yes | Yes |
|
||||
| Client-side bulk cap (CLI-32) | No | Yes | No | Yes | Yes |
|
||||
| Event backpressure (CLI-13/33) | unbuffered | 16+1, loud, true root cause ✓ | 16, cancel+error | unbuffered | unbuffered |
|
||||
| TLS default (CLI-17) | accept-all | accept-all | accept-all | TOFU pin | strict pin |
|
||||
| Typed auth errors (CLI-09) | Yes | **No** | Yes | Yes | Yes |
|
||||
| Real-wire test harness | Kestrel h2c ✓ (TST-24) | bufconn ✓ | InProcessServer ✓ | grpc.aio loopback ✓ (TST-24) | tonic loopback ✓ |
|
||||
|
||||
CLI subcommand coverage remains uneven by design and is now honestly specified (`docs/CrossLanguageSmokeMatrix.md:163-186`: .NET all nine, Rust + unregister/credential pair, Go/Python/Java credential pair only) — verified accurate against all five dispatch tables at HEAD, including the note that every gap is CLI-surface-only (all five libraries carry all typed helpers, confirmed for the Rust bulk family at `session.rs:605-654`).
|
||||
|
||||
## What is good
|
||||
|
||||
- **The remediation train executed at design fidelity.** All eleven 2026-07-12 findings landed exactly as their remediation designs specified — including the parts prior cycles skipped (the CLI-21 tag guard, the CLI-02/42 doc half, the CLI-26 pyproject-drift test). The conformance findings (CLI-37/38/40/41) all went fixtures-first: ten shared command-reply fixtures under `clients/proto/fixtures/behavior/command-replies/`, each consumed by all five suites, so the canonical rules are pinned in bytes, not prose.
|
||||
- **The publish pipeline is now guarded end to end**: version-vs-tag check in `tag-go-module.ps1`, registry-collision hard-stop in `pack-clients.ps1`, publish coordinates and the Java 0.2.1 exception recorded in `docs/ClientPackaging.md` with the reasoning ("these are the numbers a future release bumps off").
|
||||
- **TST-24 paid for itself immediately**: the first real-wire run of the Python client found `connect()` could never work outside stubs — a defect every mock-based test had actively asserted as correct. All five clients now have genuine-transport tests (framing, metadata, status codes, ReplayGap, PERMISSION_DENIED mapping), and CI runs all five suites plus a Java generated-tree drift check (`.gitea/workflows/ci.yml:84-142`).
|
||||
- **Generated-code hygiene is the best it has been**: vendored Rust protos byte-identical to Contracts (verified, all three files); check-codegen Check 3 sweeps both directions; the Java toolchain pin eliminated the spurious ~64k-line protobuf churn (per `build.gradle:61-70`, IPC-24) and `checkGeneratedClean` survived Gradle 9 (`df45cb4`); Python regen stays on the pinned grpcio-tools 1.80.0 / protobuf 6.31.1 baseline (`pyproject.toml:41-44`, generated header confirms 6.31.1); even a proto-comment-only change (`c3c603f`) regenerated every downstream tree in the same commit.
|
||||
- **Parity discipline held through the alarm epic**: no client wraps or reinterprets `AlarmFeedMessage`/`ActiveAlarmSnapshot` — the truncation and status flags pass through the generated types untouched in all five, and every README explains the `from_truncated_snapshot`/`snapshot_status` semantics with the same don't-reconcile-deletions guidance, explicitly disambiguating them from the per-record `degraded` flag.
|
||||
- **The smoke-matrix doc is a model of honest divergence-recording**: where the CLIs still differ (subcommand coverage, JSON formatting quirks per language), the doc specifies the delta instead of leaving it to be discovered — the exact practice this review series asked for two cycles ago.
|
||||
@@ -0,0 +1,104 @@
|
||||
# Testing, Documentation & Underdeveloped Areas — Re-Review (cycle 3)
|
||||
|
||||
- **Date:** 2026-08-19
|
||||
- **Commit:** `8b3e3b3` (`main`, "Merge feat/sweep-reread-tag-summary…")
|
||||
- **Baseline:** prior report `archreview/2026-07-12/60-testing-docs-gaps.md` @ `4f5371f` (2026-07-12); ~288 commits since.
|
||||
- **Method:** macOS static review — reading only, no builds, no test runs, no remote hosts. One read-only script executed (`python3 code-reviews/regen-readme.py --check`, sanctioned by REVIEW-PROCESS.md). All paths relative to `/Users/dohertj2/Desktop/MxAccessGateway`.
|
||||
|
||||
## Prior-finding verification
|
||||
|
||||
### Cycle-2 findings (TST-25..30)
|
||||
|
||||
| ID | Claimed status | Verified? | Evidence (file:line) | Notes |
|
||||
|----|----------------|-----------|----------------------|-------|
|
||||
| TST-25 | Done | **Yes** | `.gitea/workflows/ci.yml:144-165` (`windows-x86` per-push job, Linux runner → SSH to windev, exit-code propagation) and `:167-201` (`nightly-windev`, `if: github.event_name == 'schedule'`, `run-windev-ci.sh live`, on-failure Gitea-issue step); `scripts/ci/run-windev-ci.sh` (SHA-shape validation `:46-49`, base64 key decode `:70-78`, bootstrap holds the worktree lock `:91-119`); `scripts/ci/windev-worker-ci.ps1` (lock reuse via `MXGW_CI_LOCK_HELD` `:77-94`, re-checkout under lock + HEAD verification `:99-111`, cumulative build/test/live modes `:119-139`) | The whole design shipped as specified, including the acceptance-check hardening: the CI key is stored base64 single-line so Gitea's line-oriented masker redacts it (`ci.yml:156-159`), the bootstrap fetch/checkout races were serialized under the mkdir lock, and the nightly failure issue carries a browser-reachable link via `PUBLIC_SERVER_URL` (`ci.yml:174-181`, Check 6 closed 2026-08-10 per `docs/runbooks/TST-30-second-ci-runner.md`). Ground truth from the operator confirms the tier runs green. |
|
||||
| TST-26 | Done | **Yes** | `docs/GatewayTesting.md:624-657` describes exactly the four jobs that exist (`portable`, `java`, `windows-x86`, `nightly-windev`) incl. the "red can mean the tier is down" triage (`:649-652`); `scripts/check-codegen.ps1:26-27` now attributes the x86 guard to "the SSH-driven `windows-x86` CI job"; `ci.yml:14-18` cron comment matches what the schedule runs | Guard attribution is correct both ways: primary = Check 2 regeneration diff in `portable`, secondary = the windows-x86 net48 compile (`GatewayTesting.md:699-703`). No named job is missing and no phantom job survives. |
|
||||
| TST-27 | Done | **Yes** | `docs/GatewayConfiguration.md:195` — the `ShowTagValues` row now describes all three seams (events-hub mirror, alarms hub, `/browse` live values incl. the successful-read-only nuance from `e50ec49`), the clone-don't-mutate guarantee, and the ACL layering | "Reserved" is gone; the row was rewritten a second time when TST-16's residuals shipped (2026-08-17), so it documents strictly more than the finding asked. |
|
||||
| TST-28 | Done | **Yes** | `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs:46` — `StartAsync_SendsGatewayHelloWithConfiguredMaxFrameBytes(int maxMessageBytes)` theory, in the CI-run portable suite | Exactly the designed test (default + override), pinning the gateway half of the IPC-02 negotiation on Linux CI. |
|
||||
| TST-29 | Done | **Yes** | `oldtasks.md` absent from the tree; the five `*-docs-*.md` root artifacts absent; `docs/DesignDecisions.md:264` "Session-Resilience Epic Scope" carries the Phase-5 deferred-not-planned record and `:295` the "`EnableOrphanReattach` does not exist and must not be referenced" rule; CLAUDE.md's invariant paragraph points at DesignDecisions | Governance record migrated intact before deletion, exactly per the design. Residual: `stillpending.md` (also a TST-14 root artifact) still contradicts the migrated record — see new finding TST-33. |
|
||||
| TST-30 | Done | **Yes, with one pending operator action** | `docs/runbooks/TST-30-second-ci-runner.md:1-13` (Executed 2026-08-07: `gitea-runner-2`, id 5, capacity 2, `container.network: traefik`, token from a `0600` file mount; concurrency verified across three simultaneous runs) plus the honest correction that the per-repo runners endpoint legitimately returns 0 (instance-level registration); `docs/GatewayTesting.md:659-685` documents the shared-capacity/no-cancel reality and the `run-windev-ci.sh` bypass | The runner exists and the docs match the executed state. The **follow-up it surfaced is still open**: the shared instance registration token (world-readable ~5 months, proven still live by a probe registration) needs a web-UI reset — promoted to new finding TST-32 so it stops living only inside a closed finding's runbook. |
|
||||
|
||||
**Done claims that failed verification: none.** All six hold with file:line evidence; TST-30 carries the one explicitly-tracked operator residual (TST-32 below).
|
||||
|
||||
### Cycle-1 leftovers still tracked open (spot-verified at HEAD)
|
||||
|
||||
| ID | Tracker status | State at HEAD |
|
||||
|----|----------------|---------------|
|
||||
| TST-05 | Partially done | **Residual confirmed open.** The live suite is 8 `[LiveMxAccessFact]`s (`WorkerLiveMxAccessSmokeTests.cs:34-920`) with zero references to `Ping`/`GetSessionState`/`GetWorkerInfo`/`DrainEvents`/`ShutdownWorker`; the gap is honestly documented at `docs/GatewayTesting.md:90-95`. The scheduling half stays closed (nightly-windev runs the suite). The two control-command live facts specified in `archreview/remediation/60-testing-docs-gaps.md` remain unauthored (windev-only work). |
|
||||
| TST-06 | Not started | **Materially narrowed, tracker stale.** `Tests/Gateway/Dashboard/DashboardLiveDataServiceTests.cs`, `EventsHubTests.cs`, `EventsHubViewerRegistryTests.cs`, `AlarmsHubPublisherTests.cs`, `SessionDetailsPageEventAclTests.cs` all exist now (mostly landed with SEC-25/TST-16 closeouts and the in-process dashboard-feed work). The cycle-1 row deserves a re-audit-and-close pass rather than "Not started". |
|
||||
| TST-07 | Not started | Partially addressed: the `WorkerClientTests` 150 ms sleep-then-negative-assert is gone; `SessionManagerTests.cs:491` still has a real-clock `Task.Delay(50)`. |
|
||||
| TST-09 | Not started | Partially addressed: `GatewayApplication.cs:117-124` now registers `AuthStoreHealthCheck` **and** `SessionHealthCheck` (the `/health/active` probe with off-repo consumers). Still no Galaxy SQL / LDAP / worker-executable checks. |
|
||||
| TST-10 | Not started | Still no `docs/Deployment.md`; partial mitigation via `docs/runbooks/IdentifyingADeployedBuild.md` and the SEC-36 rotation runbook, but deploy/upgrade mechanics (dated `Server-YYYYMMDD` dirs, NSSM repoint, env-var config) still live in operator memory notes, and the 2026-08-09 credential-store-renamed-away outage (`docs/GatewayConfiguration.md:100`) shows the cost. |
|
||||
| TST-14 | Not started | Mostly discharged by TST-29 (oldtasks + the five untracked artifacts gone). Remaining root-file triage: `stillpending.md` (now the stale one — see TST-33) and `A2-galaxyrepository-adoption-handoff.md` (carries a current "ADOPTED" banner; acceptable as a handoff record). |
|
||||
| TST-17 | Not started | **Materially changed — row needs refresh by the worker domain.** `WnWrapAlarmConsumer.cs:309-327` now passes `szDomainName`/`szOprFullName` into the COM call (the `_ = ackOperatorDomain` discard is gone), and the 2026-08 probe cycle recorded the vendor 8-arg leg as "unavailable by configuration" with evidence (`docs/AlarmProbeFindings.md`, commits `d1ae43d`, `540a118`). Whether a caller-visible degrade signal exists is the worker reviewer's call, but "silent drop, unchanged" is no longer an accurate row. |
|
||||
| TST-18 | Not started | Confirmed open — no `*HostedService*` tests under `src/ZB.MOM.WW.MxGateway.Tests`. |
|
||||
| TST-19 | Not started | Confirmed open — `FakeWorkerHarness.cs` still has no lockstep note / `WorkerPipeSession` pointer. Mitigation since: the harness's canned control replies are no longer the *only* automation (windows-x86 runs the real worker suite per push), but the drift risk the finding names is intact. |
|
||||
| TST-20 | Not started | Confirmed open — `run-client-e2e-tests.ps1:48` `DrainEveryTags = 15` workaround intact. |
|
||||
| TST-21 | Not started | Confirmed open — `Server/appsettings.json:10` daily rolling file sink, no size cap / retained-count. |
|
||||
| TST-22 | Not started | Confirmed open — the shape block (`docs/GatewayConfiguration.md:12-88`) is maintained (gained `GroupToTag`, `UntaggedSessionVisibility`, `RecentFaultLimit`/`RecentSessionLimit`) but still omits documented keys (`DisableLogin`, `AutoLoginUser`, `CookieName`, `Tls`, `Ldap`, `Security`, `Alarms:Fallback`, `Worker:WriteCompletionWaitMilliseconds`, …). The option **tables** below it are complete and current — the omissions are only in the illustrative block. |
|
||||
|
||||
## New-code / new-docs review since `4f5371f`
|
||||
|
||||
**CI workflow evolution — verified sound.** `ci.yml` at HEAD is a four-job pipeline whose comments are accurate down to the failure semantics. Details worth recording: the portable job now runs the .NET client *tests* (TST-24 wire tests included, `ci.yml:83-90`), installs pwsh and the pinned Check-4 generators explicitly (`:58-72`), and pins protoc 34.1; the java job's `git diff --exit-code` is the true drift gate after IPC-24 deleted the churn-revert; `run-windev-ci.sh` validates `CI_SHA` shape before it reaches a remote shell (`:46-49`), never echoes the key, and pins host keys from the committed `windev.known_hosts`; `windev-worker-ci.ps1` re-checks-out under the lock, verifies `HEAD == Sha`, `git clean -fdx -e ci.log`, and is deliberately PS5.1-safe. The nightly's issue-on-failure path uses the runner-resolvable `github.server_url` for the API call but `PUBLIC_SERVER_URL` for the human link — the exact defect Check 6 caught, fixed in `30c92e8`.
|
||||
|
||||
**check-codegen.ps1 —** now four checks with the Check 3 reverse sweep (`:98-103`, canonical proto missing a vendored copy fails — the drift class no in-repo build can surface, `711793f`) and Check 4 Windows support (`.exe` banner normalization in the per-client `generate-proto.ps1` probes, `c94c4d4`; PS5.1-safe native-arg quoting, `8ae0c2f`). The header comment (`:2-27`) is accurate.
|
||||
|
||||
**fca978d marker sweep → f82dac1 restoration — assessed complete on sampled evidence.** The sweep (2026-07-07, 203 files) deleted ~1,100 prose-bearing comment lines alongside the tracking markers; the 2026-08-18 re-read narrowed those to 816 residual lines, judged each, and restored 17 collateral deletions across 10 files (`f82dac1`, merged `8b3e3b3`). Spot-checks of four substantive deletions **outside** the restored set all found surviving equivalents at HEAD: the `SessionEventDistributor` mode-vs-count FailFast race rationale survives in `docs/Sessions.md:206` (including the "fixed mode rather than a live subscriber-count snapshot" decision) and `docs/GatewayConfiguration.md:159`; the `MxAccessStaSession` off-STA-deadlock affinity-guard rationale survives in `AlarmCommandHandler.cs:68-80`; the subtag source's own-COM-object isolation rationale survives in `LmxSubtagAlarmSource.cs` class remarks; the watchdog `HeartbeatStuckCeiling` rationale survives in `WorkerPipeSessionOptions.cs:35+`. No residual collateral damage found. The companion `d3ac527` audit (re-verifying 20 doc-only `Server-0xx` resolutions, re-fixing the two the sweep had silently reverted — Server-040 among them) closes the loop on the sweep's other damage class. This is a model recovery process.
|
||||
|
||||
**Doc currency at HEAD — high.** Sampled claims all match code: `docs/Sessions.md:17` pipe-name prose matches `SessionManager.cs:536` (`mxgw-{pid}-{uid}`, with the sun_path rationale); `docs/Sessions.md:248` owner-scoped attach matches `EventStreamService`; `docs/GatewayConfiguration.md` documents the new `WriteCompletionWaitMilliseconds` (`:121`, with the OtOpcUa 2 s budget interaction), `EventQueueCapacity` (`:122`), alarm poll/fetch ceilings incl. truncation semantics (`:435-436`), and the `Fallback` block (`:448-451`); `docs/GatewayTesting.md` is the standout — the windev sections (`:488-622`) record the 879 baseline, the two "environmental" failures that were test bugs, the load-sensitivity list, and a first-class post-mortem of the zero-buffer-pipe testhost hang with the structural fixes (`TestNamedPipe.CreateServer`, bounded `PipePair.WriteAsync`). `docs/Authentication.md` was made runnable-as-written (`2b2748b`); `docs/AlarmProbeFindings.md` states the probe questions and their evidence honestly (ack leg unavailable-by-configuration, GUID leg still assumed). `python3 code-reviews/regen-readme.py --check` passes at HEAD ("up to date and consistent").
|
||||
|
||||
**New-feature test placement — correct layers.** Alarm truncation / `snapshot_status`: proto-level flag plumbed with direct tests (`b9fb0dd` atomic snapshot+truncation read), gateway feed frame (`AlarmTruncationSignalTests.cs`), dashboard banner (`AlarmsPageTruncationBannerTests.cs` incl. the mid-truncation attach case, `f57a6ae`), and all five clients regenerated with render coverage (`4d4f443`). SEC-25 tag ACL: `DashboardSessionAclTests`, both subscribe seams covered (`EventsHubTests`, `SessionDetailsPageEventAclTests`), decision-table corners pinned (`7ec0b35`), plus live-LDAP ACL coverage (`1d8a4a6`). Drain cluster (WRK-21/IPC-23/IPC-30): worker-side byte-budget tests run per-push via windows-x86; the gateway side keeps the drain-to-empty walk inside the pipe-harness envelope (`a256560`). Write-completion correlation: worker executor tests (`b0e2b8b`, `c867aca`) plus gateway-side config plumb tests (`GatewayOptionsTests`, `WorkerProcessLauncherTests`). In-process wire tests (TST-24) exist for all five clients and run in CI (`docs/GatewayTesting.md:280-321`).
|
||||
|
||||
## New findings
|
||||
|
||||
### TST-31 — `glauth.md`'s pre-provisioned-users table contradicts the directory, and its wrong credential propagated into the live-test surface — **Medium** · documentation currency / test fixtures (promotes NEXT-10)
|
||||
|
||||
- **Files:** `glauth.md:55` (ReadOnly gid `5501` vs actual `5601`), `:65-78` (pre-provisioned users table: `readonly`/`readonly123` does not exist in `scadaproj/infra/glauth/config.toml`; `admin` carries the standard dev password, not `admin123`), `:124` (`Password: admin123` in a worked example); propagation: `docs/GatewayTesting.md:119` (`MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD` default `admin123`) and `src/ZB.MOM.WW.MxGateway.IntegrationTests/WorkerLiveMxAccessSmokeTests.cs:1432` (`?? "admin123"`, with `:517` attributing the identity to the GLAuth-bridged ArchestrA `admin`/`admin123`).
|
||||
- **Description:** Carried over from `archreview/2026-07-12/remediation/90-candidate-findings-next-cycle.md` NEXT-10, which was deliberately deferred pending the OPC-UA group-taxonomy sweep and is still fully present at HEAD. The table was already the proven root cause of one defect (the NEXT-06 live-LDAP fixture drift, fixed 2026-08-07 — the *fixtures* were realigned but the table that misled them was not). The remaining propagation has a quiet coverage cost: the nightly live smoke's `WriteSecured` parity test authenticates with the defaulted `admin`/`admin123`, and because `AuthenticateUser` legitimately returns a real MXAccess *failure* reply on bad credentials (parity, not `InvalidRequest`), a wrong default does not fail the test — it silently degrades the WriteSecured leg from "authenticated round-trip" to "failure-reply shape check" every night. A reader also cannot tell which half of `glauth.md` to trust, since its dashboard section is correct.
|
||||
- **Recommendation:** Reconcile the table against `scadaproj/infra/glauth/config.toml` as one unit with the OPC-UA group taxonomy (the reason it was deferred), or — if that sweep stays parked — fix the two provably-wrong rows and the gid now and leave the taxonomy note. In the same change, re-derive the `MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD` default and the `WorkerLiveMxAccessSmokeTests` literal from whatever the rig's ArchestrA-bridged identity actually is, and make the live test log (not assert) whether `AuthenticateUser` succeeded, so a credential regression in the nightly is at least visible.
|
||||
|
||||
### TST-32 — The shared Gitea runner registration token (world-readable for ~5 months, proven live) still awaits its web-UI reset — **Medium** · CI/ops security
|
||||
|
||||
- **Files:** `docs/runbooks/TST-30-second-ci-runner.md:33-42` ("Follow-up 2026-08-07 — token hygiene"); `archreview/2026-07-12/remediation/90-candidate-findings-next-cycle.md:24` (the pending-operator-action row).
|
||||
- **Description:** Both CI runners share one **instance-scope** registration token that sat world-readable in `/opt/gitea/docker-compose.yml` for roughly five months and is **still valid** — a probe registered runner id 6 with it and then deleted it. Host hygiene is done (token now a `0600` file mount, compose + backups locked down), but hygiene does not retire the credential: anyone who read it can register a rogue runner that receives this repo's jobs, including the `WINDEV_SSH_KEY` secret that grants shell on windev. Gitea 1.26.4 exposes no rotation via CLI or API (both are get-or-create), so the reset must be done in the admin web UI, then `/opt/gitea/runner_token` refreshed and the two token-bearing compose backups shredded. This has been a documented "pending operator action" since 2026-08-07 with no tracked finding forcing it to a close — precisely the state in which such actions rot.
|
||||
- **Recommendation:** Execute the reset per the runbook (minutes of work; re-registration of the two runners with the new token is the only follow-on). Track it here rather than only in a closed finding's runbook, and after the reset append the dated "Executed" note the runbook's own convention expects.
|
||||
|
||||
### TST-33 — `stillpending.md` has drifted into self-contradiction and now overstates what is pending — **Low** · repo hygiene / documentation currency
|
||||
|
||||
- **Files:** `stillpending.md:54` (reconnect "not safe to rely on end-to-end yet" — Tasks 13–15 shipped and were verified as cycle-2 TST-01/TST-02 with all-five-client consumers); `:56` (Phase 5 orphan reattach "**planned** (epic Phase 5, not started)" — directly contradicts the governance record TST-29 migrated to `docs/DesignDecisions.md:264-295`, which says **deferred, not planned**, and CLAUDE.md's invariant); `:111` ("WorkerLiveMxAccessSmokeTests.cs (6 `[LiveMxAccessFact]`)" — there are 8); `:118` ("Cross-language client↔gateway wire behavior — no per-client integration unit tests… All client wire behavior is unverified in default unit runs" — false since TST-24 closed 2026-08-10: all five clients have real-transport wire tests running in CI); `:150` (§8: per-session-ACL "still scheduled… not yet started") vs `:65` (§2: the same item "RESOLVED (2026-08-17)") — the file contradicts itself.
|
||||
- **Description:** The file is not a frozen snapshot — it has been annotated through 2026-08-17 — so readers reasonably treat it as live, and it is one of the two remaining root-level triage artifacts from TST-14. Its most dangerous staleness is `:56`: it re-asserts the "planned Phase 5 / `EnableOrphanReattach`" framing that TST-04/TST-29 spent two cycles extinguishing, in the exact file a future contributor greps when asking "what's still pending".
|
||||
- **Recommendation:** One currency pass: fix the five spots above (§2 reconnect → shipped; §2 Phase 5 → deferred-not-planned, pointing at DesignDecisions; §5 counts and the wire-test bullet; §8 ACL row → resolved, matching §2). Then either commit to keeping it current or stamp it "historical snapshot — see archreview trackers for live status" at the top, which is the cheaper honest option.
|
||||
|
||||
### TST-34 — The code-reviews README consistency check is "the intended hook for CI" but is wired into no CI job — **Low** · CI
|
||||
|
||||
- **Files:** `REVIEW-PROCESS.md:124-131` ("`regen-readme.py --check` exits non-zero if README.md is stale… The PowerShell wrapper `scripts/check-code-reviews-readme.ps1` runs that check and is the intended hook for CI or a pre-commit step"); `.gitea/workflows/ci.yml` (no reference to either script in any job); also `REVIEW-PROCESS.md:130-131` (the "bare `python3` alias resolves to the Windows Store stub — use `python`" note is Windows-only advice presented unconditionally; on the macOS tree `python` may be absent and `python3` is correct).
|
||||
- **Description:** The check passes at HEAD (verified this review), and the repo's process doc names CI as its intended home, but 13 months of CI evolution never picked it up — so a stale `code-reviews/README.md` or an inconsistent open-findings count merges silently until the next human runs the script. The portable job already installs Python 3.12, so the cost is one step.
|
||||
- **Recommendation:** Add a `Code-reviews README consistency` step to the `portable` job (`python code-reviews/regen-readme.py --check`), and qualify the interpreter note in REVIEW-PROCESS.md as Windows-specific (macOS/Linux: `python3`).
|
||||
|
||||
### TST-35 — The Windows-host run of the gateway (net10) suite is still manual-only, and its failure class recurred three times in August — **Low** · CI/coverage
|
||||
|
||||
- **Files:** `docs/GatewayTesting.md:488-498` ("The gateway suite… is not part of the CI Windows tier — `windows-x86` and `nightly-windev` run only the x86 Worker build and `Worker.Tests`. It is still run on windev by hand…"); evidence the class is real: `e2352d1` (two Windows-only gateway test bugs long dismissed as environmental), `bfcf829`/`02baf0c` (zero-buffer Windows pipe wedged the whole suite's testhost), `25f07f8` (SQLite pool sharing-violation on Windows-only cleanup).
|
||||
- **Description:** TST-25 restored the Worker tier, but the *gateway* suite's Windows-specific divergences (platform crypto formatting, Windows pipe buffer semantics, file-sharing rules) surface only when someone remembers the manual windev run. Every one of the three August finds was a real defect in test code that CI could have caught nightly; the suite's idle wall time on windev is ~31 s, and the nightly job already builds the full slnx there — the marginal cost of running the suite is small. Severity Low, not Medium: the same 879 tests run on Linux per push, the manual procedure is well documented with a current baseline, and known load-sensitivity (the suite flakes when run concurrently with the x86 tier, `GatewayTesting.md:530-555`) means naive stacking into the nightly needs care.
|
||||
- **Recommendation:** Extend `windev-worker-ci.ps1`'s `live` mode with a `dotnet test src/ZB.MOM.WW.MxGateway.Tests` step **after** the Worker tier finishes (the lock already serializes the box, which addresses the documented load sensitivity), and record the 879 expectation in the script log. If nightly wall time becomes a concern, a weekly cadence still beats "when someone remembers".
|
||||
|
||||
### TST-36 — Docs and CLAUDE.md refer to the "`event` scope"; the canonical scope string is `events:read` — **Low** · documentation
|
||||
|
||||
- **Files:** `CLAUDE.md:160` (twice: "Possessing the `event` scope…", "an `event`-scoped key"); `docs/Sessions.md:248` (twice); `docs/DesignDecisions.md:93`; `docs/Grpc.md:103` (mitigated — it parenthesizes `GatewayScopes.EventsRead`); canonical constants in `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayScopes.cs:10` (`"events:read"`).
|
||||
- **Description:** `f2a422b` ("canonical scope strings in the two remaining stale lists") fixed the scope *lists* but left the `event`-scope shorthand in running prose. There is no `event` scope: an operator who takes the prose literally and runs `apikey create-key --scopes event` mints a key that cannot stream. CLAUDE.md even uses the correct `events:read` forty lines earlier (`:158`), so the same file teaches both spellings.
|
||||
- **Recommendation:** Mechanical sweep: replace "`event` scope" with "`events:read` scope" in the four prose sites (Grpc.md's parenthesized form is acceptable as-is). One grep verifies completion: `git grep -n '\`event\` scope'` over `CLAUDE.md docs/` returns nothing.
|
||||
|
||||
## Severity roll-up (new findings)
|
||||
|
||||
| Severity | Count | IDs |
|
||||
|---|---|---|
|
||||
| High | 0 | — |
|
||||
| Medium | 2 | TST-31, TST-32 |
|
||||
| Low | 4 | TST-33, TST-34, TST-35, TST-36 |
|
||||
|
||||
## Still good / preserved
|
||||
|
||||
- **The windev CI tier is the best piece of infrastructure this domain has reviewed in three cycles.** It shipped exactly per design and then got hardened by its own acceptance checks: the cleartext-key leak was found and fixed by storing the key base64 (with the masking rationale documented in three places), the bootstrap lock race was closed with an explicit `MXGW_CI_LOCK_HELD` handoff, the SHA is shape-validated before it reaches a remote shell, an unreachable windev fails red instead of wedging, and the nightly's failure path was itself tested to failure (Check 6) and fixed. `scripts/ci/README.md` records the bring-up, the acceptance checks with dated outcomes, and the degraded mode.
|
||||
- **The codegen guard now covers every generated surface in both directions:** descriptor set, `Contracts/Generated`, Rust vendored protos (including the reverse sweep for never-vendored canonical protos — drift no in-repo build can surface), Go/Python bindings with pinned generators that fail rather than skip when missing, and the Java aggregates via the java job's git-diff gate. Check 4 was additionally made Windows-runnable, so windev regen workflows use the same guard.
|
||||
- **The prior cycle's High (TST-25) is not just closed — its unlocks were followed through:** old TST-24 closed with real-transport wire tests in all five clients running in CI, old TST-05's scheduling half closed by the nightly, the EventBurst flake root-caused and deflaked (flush-edge sampling), the two "windev-environmental" failures exposed as test bugs, and the months-old full-suite testhost hang root-caused to a zero-buffer Windows pipe with structural fixes (`TestNamedPipe.CreateServer`, bounded `PipePair.WriteAsync`) plus a first-class post-mortem in `docs/GatewayTesting.md` — the windev baseline now equals macOS at 879.
|
||||
- **Documentation discipline visibly recovered from its one big self-inflicted wound.** The fca978d sweep's collateral damage was found by a systematic re-read (816 lines judged, 17 restorations), the doc-only `Server-0xx` resolutions were re-audited with the two regressions re-fixed (`d3ac527`), and every doc-currency spot-check this review ran (Sessions, GatewayConfiguration, GatewayTesting, Authentication, alarm probes, pipe naming, write-completion) matched code at HEAD. The two currency findings this cycle (TST-31, TST-33) are inherited debris, not new drift from the ~288-commit period.
|
||||
- **New features landed with tests at the right layers**: alarm truncation is asserted from the worker flag through the proto, the gateway feed frame, the dashboard banner (including mid-truncation attach), and all five client renders; the SEC-25 tag ACL is pinned at both subscribe seams with decision-table corners and live-LDAP coverage; `regen-readme.py --check` passes; `code-reviews/README.md` is consistent with the per-module findings at HEAD.
|
||||
@@ -0,0 +1,238 @@
|
||||
# MxAccessGateway — Remediation Plan & Tracking (2026-08-19 review)
|
||||
|
||||
Source review: [`../00-overall.md`](../00-overall.md) (HEAD `8b3e3b3`, committed as `f4919ba`). Generated 2026-08-20.
|
||||
Tracks the 34 cycle-3 IDs (GWC-31+, WRK-32+, IPC-33+, SEC-37+, CLI-46+, TST-31+) plus the tracker-housekeeping actions the overall asked for. Cycle-1/2 trackers remain the record for their own IDs.
|
||||
|
||||
**Status legend:** `Not started` · `In progress` · `Done` · `Blocked (external)` · `Deferred (P2)` · `N/A`.
|
||||
|
||||
## Execution model
|
||||
|
||||
This plan is executed as **seven parallel batches**, each by one subagent working in an isolated git worktree on its own `fix/c3-<batch>` branch, merged back into `main` by the coordinator in the order listed (lowest conflict surface first). Every batch must: build the affected component(s) on macOS (`NonWindows.slnx` — the x86 worker cannot build here), run only the filtered tests it touches, update docs in the same commit, and cite finding IDs in commit messages (`fix(IPC-34): …`). Worker-side changes (batches E, F) are verified afterwards on windev via `windev-worker-ci.ps1 -Mode test`.
|
||||
|
||||
**Model assignment rationale:** `opus` for batches that change concurrency/authorization logic or require reasoning across several interacting files with regression tests; `sonnet` for contained, well-specified edits with a clear recipe (version bumps, doc reconciliation, a two-line worker fix); `haiku` for none — every batch either touches code invariants or needs judgement about stale prose, where the cost of a wrong mechanical sweep exceeds the saving.
|
||||
|
||||
| Batch | Branch | Findings | Model | Why this model | Status |
|
||||
|---|---|---|---|---|---|
|
||||
| A | `fix/c3-client-versions` | CLI-46 | sonnet | Mechanical five-client bump + README note + one script guard; recipe is fully specified | Done |
|
||||
| B | `fix/c3-glauth-doc` | TST-31 | sonnet | Reconcile a table against one TOML source of truth, re-derive one default, change an assert to a log | Done |
|
||||
| C | `fix/c3-tracker-hygiene` | GWC-31 + housekeeping (close GWC-16/IPC-08/IPC-31, refresh TST-06/TST-17, register WRK-31) | sonnet | Cross-file comment/ID sweep needs care about what each ID *means*, but no logic changes | Done (`3619152`) |
|
||||
| D | `fix/c3-dashboard-acl` | SEC-40, SEC-41, GWC-38 | opus | Authorization semantics, fail-closed decisions, claims re-resolution; each needs a regression test | **Done** (`b177636` GWC-38, `2ef7b93` SEC-41+SEC-40, `8257a03` comment fixup; branch not merged/pushed) |
|
||||
| E | `fix/c3-ipc-boundaries` | IPC-33, IPC-39, IPC-34, IPC-35 | opus | Changes the WorkerClient command lifecycle, the options validator, and the worker drain budget; race-sensitive | **Done** (`577a0db` IPC-33, `a71663a` IPC-39, `b27035b` IPC-34, `ab41362` IPC-35; branch `fix/c3-ipc-boundaries`, not merged/pushed) — worker half verified on windev (`f2ab9cc`, Worker.Tests 524/524) |
|
||||
| F | `fix/c3-worker-residuals` | WRK-32, WRK-33 (+WRK-34 opportunistic) | sonnet | Small, precisely located worker edits + doc prose; cannot be built on macOS so the agent's job is a careful, minimal diff | Done (windev x86 build + Worker.Tests 524/524 green on `f2ab9cc`) |
|
||||
| G | `fix/c3-session-lifecycle` | GWC-32, GWC-34, GWC-37 | opus | Distributor/dispose lifecycle races and an alarm-monitor re-resolve path; needs lock-discipline reasoning | Done (`058d8d5`, `282a0e7`, `1932217`) |
|
||||
|
||||
**Out of scope for this execution (recorded, not dropped):**
|
||||
|
||||
| ID | Status | Why |
|
||||
|---|---|---|
|
||||
| TST-32 | Blocked (external) | Gitea runner registration-token reset is a web-UI action on the Gitea admin page; no repo change. Operator to-do. |
|
||||
| SEC-37 | Blocked (external) | Family-wide: TLS on the shared GLAuth lives in `scadaproj/infra/glauth/` (separate repo + docker host). Plan separately; after it lands, move the deployed service to `Production`. |
|
||||
| CLI-47/48/49/50, IPC-36/37, TST-35 | Not started | Activated by the 2026-08-25 P2 continuation below. |
|
||||
| GWC-33/35, SEC-38/39 | Done | Closed by continuation batch I; regression counts and commits are recorded below. |
|
||||
| GWC-36, IPC-38, TST-33/34/36 | Done | Closed by continuation batch H; verification and commit are recorded below. |
|
||||
| WRK-34 | Done | Completed opportunistically in batch F (`3679e59`): watchdog faults use `CommandMethod=Watchdog` and carry the stuck correlation ID in the diagnostic message. Tracker correction recorded by the continuation audit. |
|
||||
|
||||
## Batch specifications
|
||||
|
||||
### A — CLI-46 client release-train bump (sonnet)
|
||||
- Bump .NET/Go/Python/Rust clients `0.2.0 → 0.2.1`, Java `0.2.1 → 0.2.2` in every version-bearing file (`pyproject.toml` + `version.py`, `Cargo.toml` workspace + any `version =` consumers, `Directory.Build.props`/csproj `<Version>`, Java `build.gradle`/`gradle.properties`, Go module is tag-driven — update any `Version` const if one exists). Follow `docs/ClientPackaging.md` for the authoritative file list.
|
||||
- `clients/python/README.md` + `docs/ClientPackaging.md`: note that published pypi 0.2.0 has a broken `connect()` (event-loop binding, fixed by `a8f86b5`) and is superseded by 0.2.1.
|
||||
- Add a PR-time check to `scripts/check-codegen.ps1` (or a new `scripts/check-client-versions.ps1` wired where the other checks run): fail if any `clients/<lang>` source differs from its last publish tag while the version constant still equals that tag's version. Document in `docs/ClientPackaging.md`.
|
||||
- Verify: `dotnet build clients/dotnet/...slnx`; `cargo check --workspace`; `python -m pytest` (clients/python); `JAVA_HOME=/opt/homebrew/opt/openjdk@17 gradle test` then **revert** spurious generated-proto churn; `go build ./... && go test ./...`.
|
||||
|
||||
### B — TST-31 glauth.md + live-test credential (sonnet)
|
||||
- Reconcile `glauth.md` against `~/Desktop/scadaproj/infra/glauth/config.toml`: ReadOnly gid, the pre-provisioned users table (drop `readonly/readonly123`, fix `admin` password), the worked example at `:124`. Keep the OPC-UA taxonomy note.
|
||||
- Re-derive `MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD` default in `docs/GatewayTesting.md` and the `?? "admin123"` literal in `WorkerLiveMxAccessSmokeTests.cs` from the config; make the test **log** (not assert) whether `AuthenticateUser` succeeded.
|
||||
- Verify: `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx` (IntegrationTests compiles on mac? if excluded, note it).
|
||||
|
||||
### C — GWC-31 + tracker housekeeping (sonnet)
|
||||
- Sweep code comments citing unregistered `GWC-30`/`GWC-31` (perf pass) in `WorkerClient.cs`, `GatewayMetrics.cs`, `WorkerClientTests.cs`, worker `WorkerFrameReader/Writer.cs` to `PERF-nn` keyed to task numbers in `docs/plans/2026-08-15-perf-review-remediation.md`; add a PERF-ID table to that plan. Do not touch the genuine GWC-30 (frame-reader prefix) citations if any exist.
|
||||
- `archreview/REVIEW-PROCESS.md` (or wherever the ID rule lives): state that GWC/WRK/IPC/SEC/CLI/TST prefixes are reserved for archreview registers.
|
||||
- Cycle-2 tracker (`archreview/2026-07-12/remediation/00-tracking.md` + domain docs): GWC-16 → Won't fix (rationale from cycle-3 10-gateway-core.md), IPC-08 → Done (`7c1ea12`, residual IPC-34), IPC-31 → Done (superseded by GWC-28), refresh TST-06/TST-17 rows, register WRK-31 (interim ID burned by `e913dab`) so it is never reused.
|
||||
- Verify: `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx`; `dotnet test --filter FullyQualifiedName~WorkerClientTests`.
|
||||
|
||||
### D — SEC-40, SEC-41, GWC-38 (opus)
|
||||
- SEC-41: `DashboardSessionAcl` re-resolves tags from `LdapGroupClaimType` claims via `DashboardGroupTagMapping.MapGroupsToTags` against current options (as `HubTokenService` does) instead of trusting login-stamped tag claims; keep stamped claims only as fallback when no group claims exist (document). Test: principal with stale tag claim + group whose grant was removed → denied.
|
||||
- SEC-40: decide **tighten**: `AlarmsHub` gets a dedicated policy requiring a real role claim (anonymous-loopback/Disabled bypass excluded) — and document in `docs/GatewayDashboardDesign.md` ACL section + `docs/GatewayConfiguration.md` GroupToTag row that alarm metadata remains galaxy-wide for authenticated Viewers (central monitor, no per-session provenance). Test for the policy.
|
||||
- GWC-38: `GatewayBrowseScopeProvider` throws `RpcException(PermissionDenied)` (and logs) when `Current` is null **and** API-key auth is enabled; unchanged when auth mode is Disabled. Test both branches.
|
||||
- Verify: build Server; `dotnet test --filter "FullyQualifiedName~DashboardSessionAcl|FullyQualifiedName~AlarmsHub|FullyQualifiedName~BrowseScope"` (adjust to real class names).
|
||||
|
||||
**Outcome (done, branch `fix/c3-dashboard-acl`, commits `b177636`, `2ef7b93`, `8257a03`; not merged, not pushed).** Decisions worth carrying forward:
|
||||
|
||||
- **SEC-41 fallback:** the ACL re-resolves from `mxgateway:ldap_group` claims when the principal has any, and falls back to the stamped `zb:dashboardtag` claims only when it has none. That is exactly the hub-token principal — `HubTokenService.Validate` rehydrates roles and tags but **not** groups, so re-resolution already happened at mint. The fallback cannot re-open the cookie gap, because `DashboardAuthenticator` stamps both claim kinds from one group list.
|
||||
- **SEC-40 requirement type:** new `AuthenticatedDashboardRoleRequirement` + `AuthenticatedDashboardRoleHandler` and a `MxGateway.Dashboard.Alarms` policy. `DashboardAuthorizationHandler` is `AuthorizationHandler<DashboardAuthorizationRequirement>`, so it is never invoked for the new type — the same structural device the Secrets UI's `secrets:manage` already relies on. Both hub schemes are retained on the policy.
|
||||
- **SEC-40 scope answer:** the `/alarms` Blazor page carries the SAME policy (it reads the central monitor in process — gating only the hub left the disclosure one navigation away), and the side-rail Alarms link is wrapped in an `AuthorizeView` on that policy. `AlarmsHubPublisher` is unchanged and stays unfiltered by design: audience is decided at hub admission and the feed carries no per-session provenance.
|
||||
- **GWC-38:** `PermissionDenied` + warning only when `Authentication:Mode=ApiKey`; `Mode=Disabled` still returns unscoped. `GalaxyRepositoryHostWiringTests` phase 1 was changed to push an *unconstrained identity* rather than none — that is production's real unscoped case.
|
||||
- New/changed tests: `AlarmFeedAuthorizationTests` (new, 7), `DashboardSessionAclTests` (+5), `GatewayBrowseScopeProviderTests` (+2), `SecretsNavRenderTests` (+2, and it must register the new policy — an unregistered policy name *throws* inside `AuthorizeView` rather than denying). Filtered run over the touched classes: 92 passed / 0 failed; `NonWindows.slnx` builds zero-warning.
|
||||
|
||||
### E — IPC-33, IPC-39, IPC-34, IPC-35 (opus)
|
||||
- IPC-33 option (a): `GatewayOptionsValidator` adds the upper coupling `Worker:MaxMessageBytes ≤ Protocol:MaxGrpcMessageBytes + 2×reserve`; worker `ResolveDrainReplyByteBudget` becomes `MaxMessageBytes − 2×DrainReplyFrameHeadroomBytes` (floor `MaxMessageBytes/2`); update `docs/GatewayConfiguration.md`, `docs/WorkerFrameProtocol.md`. Test: `GatewayOptionsTests` case for the new bound.
|
||||
- IPC-39: pre-check compares against `MaxMessageBytes − MaxSequenceEncodingBytes (11)`; fix the comment direction.
|
||||
- IPC-34: caller-cancellation arms call `TrySendCancelForAbandonedCommand` (renamed) gated on `removed`, skipped when `_stopCts` is the source; new `WorkerClientErrorCode.CallerCancelled` replacing the `GatewayShutdown` mislabel, mapped to `Cancelled` in the gRPC layer; update `docs/Grpc.md` mapping line, `gateway.md:913-919`, `docs/GatewayProcessDesign.md:596-602`. Test: twin of the timeout test driven by the caller token.
|
||||
- IPC-35: `ReadHandshakeEnvelopeAsync` special-cases `WorkerFault` → `WorkerClientException(WorkerFaulted, CreateWorkerFaultMessage(...))`. Test: fake writes `WorkerFault{MxaccessCreationFailed}` instead of `WorkerReady`, category text surfaces.
|
||||
- Verify: build Server; `dotnet test --filter "FullyQualifiedName~WorkerClientTests|FullyQualifiedName~GatewayOptions"`. Worker half → windev afterwards.
|
||||
|
||||
### F — WRK-32, WRK-33 (sonnet)
|
||||
- WRK-33: `ObserveFaultWhenever(ObserveCommandTaskAsync(commandTask));` at `WorkerPipeSession.cs:1235` (or catch-all-and-log in the observer).
|
||||
- WRK-32: one `pumpStep()` before capturing the `CurrentVersion` baseline in `MxAccessCommandExecutor.cs`; rewrite the executor doc comment (`:996-1003`) to state the recency-based contract; add the same to `docs/MxAccessWorkerInstanceDesign.md` write-completion section and the reply-field comment in `mxaccess_gateway.proto` (**comment-only proto edit → regenerate all generated artifacts in the same commit**, per the repo rule; if regen tooling is unavailable on macOS, skip the proto comment and say so).
|
||||
- WRK-34 if trivial: `CreateFault(StaHung, "Watchdog", …)` with the correlation id moved into the message.
|
||||
- Cannot build on macOS: keep diff minimal, net48-safe (no init-only/positional records). Verified on windev afterwards.
|
||||
|
||||
### G — GWC-32, GWC-34, GWC-37 (opus)
|
||||
- GWC-32: `DisposeAsync` transitions `_state` to `Closed` under `_syncRoot` before completing channels (or `if (_disposed || IsTerminalState()) return;` in `StageWorkerEvent`); update the XML doc to name all three completion paths. Test: dispose a Ready client mid-dispatch → no `ProtocolViolation` fault/metric.
|
||||
- GWC-34: `_eventDistributorDisposed` flag set under `_syncRoot` when nulling; `EnsureDistributorCreated` throws `SessionManagerException(SessionNotReady)` when set. Test: attach after dispose → throws, no silent hang.
|
||||
- GWC-37: on `ApplyProviderModeChangeAsync(toMode == Subtag)` re-resolve the watch list and re-subscribe; additionally surface "zero targets resolved while mode can need them" on the provider status (`Degraded` + reason). Update `docs/` alarm-subtag section. Test with the fake worker: GR discovery fails at start, succeeds at failover → watch list non-empty after failover.
|
||||
- Verify: build Server; `dotnet test --filter "FullyQualifiedName~WorkerClientTests|FullyQualifiedName~GatewaySession|FullyQualifiedName~AlarmMonitor|FullyQualifiedName~AlarmWatchList"`.
|
||||
- **Outcome (Done):** `058d8d5` GWC-32, `282a0e7` GWC-34, `1932217` GWC-37 on `fix/c3-session-lifecycle`. Filtered run 176/176 green; `NonWindows.slnx` clean.
|
||||
- GWC-37 **does not re-send** the watch-list on the failover edge. The worker's `AlarmCommandHandler.Subscribe` rejects a second `SubscribeAlarms` while subscribed, and the only expressible replacement (`UnsubscribeAlarms` + `SubscribeAlarms`) rebuilds `FailoverAlarmConsumer` — restarting it on the primary that just failed and discarding the warm standby — so it would flap the feed rather than repair it. The re-resolve happens and any drift is reported on the same status frame as the failover; a monitor restart is the supported way to apply a new list. Adding a watch-list-update worker command is a possible future item, not done here.
|
||||
- GWC-37 changes `IAlarmWatchListResolver.ResolveAsync` to return `AlarmWatchListResolution`, and gives `Degraded && Mode==ALARMMGR` a real meaning (empty/undiscoverable standby) with a new dashboard label `Alarm Manager (no subtag standby)`.
|
||||
- GWC-32's dispose path now also records `WorkerStopped("dispose")` via the existing once-latch, closing a `workers_running` gauge leak on the dispose-a-Ready-client path. GWC-12's double-dispose interlock got a natural home (the `_disposed` claim moved under `_syncRoot`) but is **not** claimed as fixed.
|
||||
|
||||
## Merge order
|
||||
C is merged **last** (its comment sweep touches the same files E and G edit). Order: A, B, F, D, G, E, C. After merge: full `NonWindows` build + one full gateway test run; then windev worker CI for E+F.
|
||||
|
||||
## Integration gate (2026-08-20)
|
||||
All seven branches merged into `main` (A `ee68cc3`, B `f59c34a`, C `f2ab9cc`, D `9f0c3fd`, E `cfbc995`, F `aa07497`, G `c6650c1`). Merged tree: `NonWindows.slnx` 0 warnings; full gateway suite **1183/1183** passed; windev `windev-worker-ci.ps1 -Sha f2ab9cc -Mode test`: x86 Worker build succeeded, `Worker.Tests` **524 passed / 11 skipped / 0 failed**. Follow-ups surfaced by the batches: a worker command to refresh a live alarm subscription's watch-list (would let GWC-37's re-resolve be applied, not just reported); WRK-32's proto reply-field comment deferred until the next proto regen wave; the five clients are bumped but not yet published.
|
||||
|
||||
## P2 continuation (planned 2026-08-25)
|
||||
|
||||
Current-state audit: refreshed `origin/main` and re-checked every deferred finding and integration-gate follow-up. Sixteen of the seventeen P2 findings remain open; WRK-34 was already completed by `3679e59`. All three follow-ups remain open. Exact target client coordinates are still absent from Gitea (HTTP 404), and `clients/go/v0.2.1` does not exist.
|
||||
|
||||
Approved design: [`../../../docs/plans/2026-08-25-cycle-3-p2-followups-design.md`](../../../docs/plans/2026-08-25-cycle-3-p2-followups-design.md) (`30548da`).
|
||||
|
||||
Execution plan: [`../../../docs/plans/2026-08-25-cycle-3-p2-followups.md`](../../../docs/plans/2026-08-25-cycle-3-p2-followups.md).
|
||||
|
||||
The continuation runs serially in the isolated `.worktrees/c3-p2-followups` worktree on `codex/c3-p2-followups`, with a tracker update at every batch boundary. Codex equivalents are selected from the implementation risk: `gpt-5.6-terra`/medium for contained documentation, CI, and client work; `gpt-5.6-sol`/high for concurrency, authorization, protocol, and live alarm lifecycle changes.
|
||||
|
||||
| Batch | Findings / follow-ups | Codex model | Status | Exit gate |
|
||||
|---|---|---|---|---|
|
||||
| H — tracker and currency | WRK-34 record, GWC-36, IPC-38, TST-33/34/36 | `gpt-5.6-terra` medium | Done | Currency greps clean; README check in portable CI; NonWindows build clean |
|
||||
| I — gateway correctness/security | GWC-33/35, SEC-38/39 | `gpt-5.6-sol` high | Done | Focused regression tests green; NonWindows build clean |
|
||||
| J — secondary contracts | IPC-36/37 | `gpt-5.6-sol` high | Done | Shared launch constants and descriptor parity tests green |
|
||||
| K — client alarm parity | CLI-47/48/49/50 | `gpt-5.6-terra` high | Done | All five client suites green |
|
||||
| L — live watch-list/proto wave | watch-list refresh follow-up + WRK-32 proto comment | `gpt-5.6-sol` high | Done | Codegen clean; portable alarm/contract tests green; windev Worker.Tests green |
|
||||
| M — Windows gateway CI | TST-35 | `gpt-5.6-terra` medium | Done | windev `live` mode proves Worker, gateway, full-slnx, and live smoke tiers |
|
||||
| N — integration and release | publish five clients + Contracts | `gpt-5.6-sol` high | In progress | Full local/windev gates; merged `main`; registry coordinates and Go tag verified |
|
||||
|
||||
### Continuation tasklist
|
||||
|
||||
#### H — tracker and currency
|
||||
|
||||
- [x] Correct WRK-34 from Deferred to Done with commit `3679e59`.
|
||||
- [x] GWC-36: correct `EventSubscriberReplayAttachment.OldestAvailableSequence` semantics.
|
||||
- [x] IPC-38: add public `DrainEvents` behavior to `docs/Grpc.md` and correct the cycle-2 omission rationale.
|
||||
- [x] TST-33: reconcile the five stale/self-contradictory `stillpending.md` claims and mark its live-status boundary.
|
||||
- [x] TST-34: run `code-reviews/regen-readme.py --check` in portable CI and qualify the interpreter note.
|
||||
- [x] TST-36: replace prose ``event`` scope references with canonical ``events:read``.
|
||||
- [x] Record batch-H commit and verification evidence here.
|
||||
|
||||
Batch-H verification (2026-08-25): `python3 code-reviews/regen-readme.py --check`
|
||||
reported the generated README current and consistent; the TST-33 and live-document
|
||||
TST-36 stale-pattern greps returned no matches; `docs/Grpc.md` contains the public
|
||||
`DrainEvents` contract; and `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx`
|
||||
succeeded with **0 warnings / 0 errors** in 1.59 s. Batch commit: `92aae28`.
|
||||
|
||||
#### I — gateway correctness and security
|
||||
|
||||
- [x] GWC-33: unregister terminal clients from the event-queue-depth gauge without shortening fault retention.
|
||||
- [x] GWC-35: repair acknowledged-to-active snapshot deltas with a Raise transition and instance identity.
|
||||
- [x] SEC-38: feed/consult the aggregate for valid key IDs that overflow the peer partition cap.
|
||||
- [x] SEC-39: prove a null effective key ID cannot reset the shared fallback partition.
|
||||
- [x] Run focused tests and clean NonWindows build; record counts and commits here.
|
||||
|
||||
Batch-I verification (2026-08-25): the combined `WorkerClientTests`,
|
||||
`GatewayAlarmMonitorAttachOrderTests`, and `ApiKeyFailureLimiterTests` filter passed
|
||||
**55/55** tests in 12 s; `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx`
|
||||
succeeded with **0 warnings / 0 errors** in 3.42 s. Finding commits: GWC-33
|
||||
`b0156c6`, GWC-35 `7807201`, and SEC-38/SEC-39 `1875426`.
|
||||
|
||||
#### J — secondary contracts
|
||||
|
||||
- [x] IPC-36: centralize all six worker-launch environment names in Contracts and consume them on both sides.
|
||||
- [x] IPC-36: document variable/config/default mapping and test launcher completeness.
|
||||
- [x] IPC-37: add normalized retained-vs-served Galaxy descriptor parity test.
|
||||
- [x] Run focused tests and clean NonWindows build; record counts and commits here.
|
||||
|
||||
Batch-J verification (2026-08-25): the combined `WorkerProcessLauncherTests` and
|
||||
`GalaxyContractParityTests` filter passed **10/10** tests in 1 s; the descriptor
|
||||
mutation proof identified the changed protobuf message; and `dotnet build
|
||||
src/ZB.MOM.WW.MxGateway.NonWindows.slnx` succeeded with **0 warnings / 0
|
||||
errors** in 2.20 s. IPC-36/IPC-37 commit: `d6c5a11`.
|
||||
|
||||
#### K — client alarm-feed parity
|
||||
|
||||
- [x] CLI-47: make .NET text status flags explicit and document deliberate JSON differences.
|
||||
- [x] CLI-48: test provider/snapshot status arms in .NET, Go, and Java; add Go stream-alarm scaffold.
|
||||
- [x] CLI-49: export Go aliases for `AlarmProviderStatus` and `AlarmSnapshotStatus`; fix five-arm documentation.
|
||||
- [x] CLI-50: stream Python alarm rows incrementally with an unbounded default and bounded opt-in.
|
||||
- [x] Run .NET, Go, Python, Rust, and Java client suites; record counts and commits here.
|
||||
|
||||
Batch-K verification (2026-08-25): `dotnet test
|
||||
clients/dotnet/ZB.MOM.WW.MxGateway.Client.slnx` passed **134/134** portable
|
||||
tests (1 live test skipped); `go test ./...` passed both tested packages;
|
||||
`python -m pytest` passed **169/169** portable tests (1 opt-in TLS test
|
||||
skipped); `cargo test --workspace` passed **108/108** tests; and
|
||||
`JAVA_HOME=/opt/homebrew/opt/openjdk@17 gradle test` passed **132/132** tests.
|
||||
CLI-47/CLI-48/CLI-49/CLI-50 commit: `5cdb3da`.
|
||||
|
||||
#### L — live watch-list refresh and protobuf wave
|
||||
|
||||
- [x] Add explicit fallback-enabled subscription semantics for an initially empty discovered list.
|
||||
- [x] Add and round-trip `UpdateAlarmWatchListCommand` and its `MxCommandKind`.
|
||||
- [x] Route the command through the worker executor and alarm handler on the STA.
|
||||
- [x] Build/subscribe/attach a replacement standby before swap; preserve the old standby on failure.
|
||||
- [x] Preserve current primary/subtag mode during a successful replacement.
|
||||
- [x] Re-resolve on the existing gateway reconciliation cadence and send only material changes/recovery.
|
||||
- [x] Refresh immediately at failover edge, update acknowledged fingerprint only after worker OK, and reconcile after success.
|
||||
- [x] Expand WRK-32's public reply-field comment to the sequential stale-callback/recency case.
|
||||
- [x] Replace IPC-36's internal env-var name in the proto comment with the public configuration key.
|
||||
- [x] Regenerate Contracts and all client generated trees once; run codegen and all affected suites.
|
||||
- [x] Verify x86 Worker build and Worker.Tests on windev; record SHA/counts here.
|
||||
|
||||
Batch-L verification (2026-08-25): `pwsh scripts/check-codegen.ps1` passed all
|
||||
four descriptor/C#/Rust/Go/Python freshness checks, and Java
|
||||
`checkGeneratedClean` passed. The exact portable alarm/contract/descriptor
|
||||
filter passed **77/77**; `NonWindows.slnx` built with **0 warnings / 0 errors**;
|
||||
the .NET, Go, Python, Rust, and Java client suites were green (respectively
|
||||
**134 passed / 1 live skip**, both Go packages, **169 passed / 1 opt-in TLS
|
||||
skip**, **108 passed**, and **132 passed**). Windev `test` mode at
|
||||
`986c6c0ce4f02c62b4f59ec44780de5af6d291b8` built the x86/net48 Worker with
|
||||
**0 warnings / 0 errors** and passed Worker.Tests **529 passed / 11 opt-in
|
||||
skips / 0 failed**. Feature/proto commit: `484028e`; Batch-J x86 compile
|
||||
follow-up: `986c6c0`.
|
||||
|
||||
#### M — Windows gateway CI
|
||||
|
||||
- [x] TST-35: add the serialized net10 gateway suite to windev `live` mode.
|
||||
- [x] Update `docs/GatewayTesting.md` and `scripts/ci/README.md` with order/failure semantics.
|
||||
- [x] Execute `live` mode on windev and record Worker, gateway, full-slnx, and live-smoke evidence here.
|
||||
|
||||
Batch-M verification (2026-08-25): windev `live` mode at exact commit
|
||||
`cc143b52cf04be0b2f08350922a987b2e250bc1d` completed the serialized sequence in
|
||||
**4m45s** with remote exit code 0. The x86/net48 Worker build succeeded with **0
|
||||
warnings / 0 errors** in 22.59 s; Worker.Tests passed **529**, skipped **11**
|
||||
opt-in probes, and failed **0**; the full Windows `slnx` build succeeded with **0
|
||||
warnings / 0 errors** in 43.24 s; the net10 gateway suite passed **1193/1193**
|
||||
in 53 s; and the live-MXAccess smoke passed **8/8** in 1m46s. TST-35 commit:
|
||||
`cc143b5`.
|
||||
|
||||
#### N — integration and release
|
||||
|
||||
- [x] Run `git diff --check`, codegen, client-version, README, NonWindows build, and full gateway suite gates.
|
||||
- [x] Run all five client suites on the final candidate.
|
||||
- [ ] Run final windev `live` gate at the exact candidate SHA.
|
||||
- [ ] Merge/push the verified continuation to `main`; wait for green CI.
|
||||
- [ ] Re-query all package coordinates and Go tag immediately before publish; abort on collision.
|
||||
- [ ] Publish .NET Client + Contracts 0.2.1, Python 0.2.1, Rust 0.2.1, Java 0.2.2.
|
||||
- [ ] Create/push `clients/go/v0.2.1` at the exact release commit.
|
||||
- [ ] Verify every registry coordinate and Go module/tag, update `docs/ClientPackaging.md`, and record immutable release evidence here.
|
||||
|
||||
Batch-N portable verification (2026-08-25): `git diff --check`, codegen freshness,
|
||||
client-version freshness, and the generated README check all passed without changing
|
||||
the worktree. `NonWindows.slnx` built with **0 warnings / 0 errors**; the gateway
|
||||
suite passed **1193/1193**, and the portable integration suite passed **5** with
|
||||
**19** live-only skips. The final .NET, Go, Python, Rust, and Java client gates were
|
||||
green (respectively **134 passed / 1 live skip**, all Go packages built/tested with
|
||||
clean `gofmt`, **169 passed / 1 opt-in TLS skip** from a clean virtual environment,
|
||||
**108 passed** plus clean `cargo fmt`/Clippy, and **132 passed / 0 failed** with the
|
||||
generated Java tree unchanged).
|
||||
@@ -74,7 +74,7 @@ Full design + implementation for each row lives in the linked domain doc under i
|
||||
| GWC-13 | Low | — | M | — | Not started | Worker-ready wait is a 25 ms poll loop |
|
||||
| GWC-14 | Low | P2 | M | — | Done | Replay ring is a `LinkedList` with a node alloc per event |
|
||||
| GWC-15 | Low | P2 | S | — | Done | Per-event gauge reads `ChannelReader.Count` every event |
|
||||
| GWC-16 | Low | — | S | — | Not started | `Invoke` resolves the session twice per command |
|
||||
| GWC-16 | Low | — | S | — | Won't fix | `Invoke` resolves the session twice per command — measured negligible, deliberately not collapsed (see domain doc) |
|
||||
| GWC-17 | Low | — | M | — | Not started | Sessions layer throws `Grpc.Core.RpcException` (layering leak) |
|
||||
| GWC-18 | Low | — | S | — | Not started | `GatewaySession` implements `DisposeAsync` without `IAsyncDisposable` |
|
||||
| GWC-19 | Low | — | S | — | Not started | Dead `GatewaySession.KillWorker(string)` + stale doc |
|
||||
@@ -119,7 +119,7 @@ Full design + implementation for each row lives in the linked domain doc under i
|
||||
| IPC-05 | Medium | P2 | S | — | Done | Redundant deep copies on command and event hot paths |
|
||||
| IPC-06 | Medium | P2 | S | — | Done | `gateway.md` Worker Envelope sketch no longer matches the contract |
|
||||
| IPC-07 | Medium | P2 | S | — | Done | `docs/Grpc.md` says six RPCs; there are seven |
|
||||
| IPC-08 | Medium | — | M | — | Not started | `WorkerCancel` defined and handled but never sent |
|
||||
| IPC-08 | Medium | — | M | — | Done | `WorkerCancel` defined and handled but never sent — resolved by `7c1ea12`/`a1a38b5`; residual (caller-cancellation leg) tracked as cycle-3 IPC-34 |
|
||||
| IPC-09 | Medium | P1 | M | IPC-01 | Done | Known codegen fragilities have no in-repo guards |
|
||||
| IPC-10 | Low | — | S | — | Not started | Envelope `sequence` is write-only; monotonicity never validated |
|
||||
| IPC-11 | Low | — | S | — | Not started | No protocol version negotiation despite `supported_protocol_version` name |
|
||||
@@ -218,7 +218,7 @@ Full design + implementation for each row lives in the linked domain doc under i
|
||||
| TST-03 | High | P1 | M | — | Done | No CI exists |
|
||||
| TST-04 | High | P2 | L | — | Done | Session-resilience epic 16/28 tasks unfinished |
|
||||
| TST-05 | Medium | P1 | S | TST-03 | Partially done | Real-worker control/COM paths verified opt-in only. **Scheduling half closed 2026-08-10** by the TST-25 `nightly-windev` job (`.gitea/workflows/ci.yml`, cron `0 6 * * *` → `scripts/ci/run-windev-ci.sh live`, which sets `MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`, runs `WorkerLiveMxAccessSmokeTests`, and opens a Gitea issue on failure) — "opt-in, run by memory" is now "runs nightly, reports failures". **Residual: the coverage-audit half.** The live suite's 8 facts cover all six late-added COM commands but none of the five control commands (`Ping`, `GetSessionState`, `GetWorkerInfo`, `DrainEvents`, `ShutdownWorker`), which real workers answer in `Worker/Ipc/WorkerPipeSession.cs` yet are still only exercised through `FakeWorkerHarness` canned replies — precisely the masking the finding named |
|
||||
| TST-06 | Medium | — | M | — | Not started | Dashboard live-data path untested |
|
||||
| TST-06 | Medium | — | M | — | Materially narrowed | Dashboard live-data path untested — `DashboardLiveDataServiceTests`, `EventsHubTests`, `EventsHubViewerRegistryTests`, `AlarmsHubPublisherTests`, `SessionDetailsPageEventAclTests` now exist (landed with SEC-25/TST-16); needs a re-audit-and-close pass, not re-implementation |
|
||||
| TST-07 | Medium | — | S | — | Not started | Real-clock sleeps with negative assertions are latent flakes |
|
||||
| TST-08 | Medium | P1 | M | — | Done | Full-suite orphaned testhost processes (does not reproduce; doc de-stale) |
|
||||
| TST-09 | Medium | — | M | — | Not started | Health checks cover only the auth store |
|
||||
@@ -229,7 +229,7 @@ Full design + implementation for each row lives in the linked domain doc under i
|
||||
| TST-14 | Medium | P2 | S | — | Not started | Repo-root working artifacts need triage |
|
||||
| TST-15 | Medium | P2 | M | TST-04 | Done | Dashboard EventsHub has no per-session ACL |
|
||||
| TST-16 | Medium | P1 | S | — | Done | `Dashboard:ShowTagValues` is a dead flag |
|
||||
| TST-17 | Medium | — | S | — | Not started | Vendor-gated alarm parity residuals silently lossy |
|
||||
| TST-17 | Medium | — | S | — | Materially changed | Vendor-gated alarm parity residuals silently lossy — `WnWrapAlarmConsumer.cs:309-327` now passes `szDomainName`/`szOprFullName`; the vendor 8-arg leg is recorded "unavailable by configuration" with evidence (`docs/AlarmProbeFindings.md`, `d1ae43d`, `540a118`); "silent drop, unchanged" is no longer accurate — needs a worker-domain refresh pass |
|
||||
| TST-18 | Low | — | S | — | Not started | Hosted-service wrappers untested |
|
||||
| TST-19 | Low | — | S | — | Not started | Keep FakeWorkerHarness canned replies in lockstep |
|
||||
| TST-20 | Low | — | S | — | Not started | E2E script papers over a real advise-without-consumer sharp edge |
|
||||
|
||||
@@ -330,6 +330,8 @@ Open question the review cannot settle: whether a future probe (COM-init readine
|
||||
|
||||
**Verification.** `dotnet build src/ZB.MOM.WW.MxGateway.Server`; `dotnet test src/ZB.MOM.WW.MxGateway.Tests --filter "FullyQualifiedName~SessionManager"`.
|
||||
|
||||
**Resolution: Won't fix (2026-08-20, closed by cycle-3 verification).** `Grpc/MxAccessGatewayService.cs:111-115` carries a `PERF(followup)` comment documenting the double session lookup as measured-negligible and deliberately not collapsed — the fix would duplicate `InvokeAsync`'s fault mapping at a second entry point, trading a negligible per-command cost for a maintenance liability. Per the cycle-3 gateway-core review (`archreview/2026-08-19/10-gateway-core.md`, GWC-16 row), the rationale is sound; tracker flipped from `Not started` to `Won't fix`.
|
||||
|
||||
---
|
||||
|
||||
## GWC-17 — Sessions layer throws `Grpc.Core.RpcException` `Low` · `—`
|
||||
|
||||
@@ -175,6 +175,8 @@ The frame protocol and proto evolution hygiene are sound; the remediable risk cl
|
||||
|
||||
**Verification.** `dotnet build src/ZB.MOM.WW.MxGateway.Server` + `dotnet test ...Tests --filter FullyQualifiedName~WorkerClient`; worker side `-p:Platform=x86`.
|
||||
|
||||
**Resolution: Done (2026-08-20, closed by cycle-3 verification, residual filed).** Wired up substantially as designed: `InvokeAsync` forwards a `WorkerCancel` when the gateway's own command timeout fires and it owned the removal (`Server/Workers/WorkerClient.cs:269-291`), `TrySendCancelForTimedOutCommand` is structurally no-throw (`:835-870`), the worker routes it to `CancelCommand` (`Worker/Ipc/WorkerPipeSession.cs:755-756` → `MxAccess/MxAccessStaSession.cs:529-532` → `Sta/StaCommandDispatcher.CancelQueuedCommand:151`), regression test `WorkerClientTests.InvokeAsync_WhenCommandTimesOut_SendsWorkerCancelForThatCorrelation` exists, and `docs/GatewayProcessDesign.md:596-602` documents it. Landed in `7c1ea12`/`a1a38b5`. **Residual, not covered by this closure:** the caller-cancellation leg (context deadline / client disconnect, as opposed to the gateway's own `CommandTimeout`) sends nothing — filed as cycle-3 **IPC-34**. Per `archreview/2026-08-19/30-contracts-ipc.md` (IPC-08 row), tracker flipped from `Not started` to `Done` with the residual cross-referenced.
|
||||
|
||||
---
|
||||
|
||||
## IPC-09 — Known codegen fragilities have no in-repo guards `Medium` · `P1`
|
||||
|
||||
@@ -191,6 +191,8 @@ This finding is the umbrella; TST-01/02/15 are its actionable slices. The remedi
|
||||
|
||||
**Verification.** `dotnet test ... --filter FullyQualifiedName~DashboardLiveDataServiceTests`.
|
||||
|
||||
**Status refresh (2026-08-20, per cycle-3 verification).** Materially narrowed — the tracker row was stale. `Tests/Gateway/Dashboard/DashboardLiveDataServiceTests.cs`, `EventsHubTests.cs`, `EventsHubViewerRegistryTests.cs`, `AlarmsHubPublisherTests.cs`, and `SessionDetailsPageEventAclTests.cs` all exist now (mostly landed with the SEC-25/TST-16 closeouts and the in-process dashboard-feed work). Per `archreview/2026-08-19/60-testing-docs-gaps.md` (TST-06 row), this needs a re-audit-and-close pass against the current coverage rather than fresh implementation — left open pending that audit, tracker status changed from `Not started` to `Materially narrowed` to reflect the actual state.
|
||||
|
||||
---
|
||||
|
||||
## TST-07 — Real-clock sleeps with negative assertions are latent flakes `Medium` · `—`
|
||||
@@ -371,6 +373,8 @@ If TST-02's interim mitigation (flip retention off) is chosen instead of impleme
|
||||
|
||||
**Verification.** `dotnet build src/ZB.MOM.WW.MxGateway.Worker -p:Platform=x86`; `dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/... -p:Platform=x86` (Windows).
|
||||
|
||||
**Status refresh (2026-08-20, per cycle-3 verification).** Materially changed — the tracker row needs a refresh from the worker domain. `WnWrapAlarmConsumer.cs:309-327` now passes `szDomainName`/`szOprFullName` into the COM call (the `_ = ackOperatorDomain` discard is gone), and the 2026-08 probe cycle recorded the vendor 8-arg leg as "unavailable by configuration" with evidence (`docs/AlarmProbeFindings.md`, commits `d1ae43d`, `540a118`). Per `archreview/2026-08-19/60-testing-docs-gaps.md` (TST-17 row), "silent drop, unchanged" is no longer an accurate description of the row; whether a caller-visible degrade signal now exists is the worker reviewer's call to close out. Tracker status changed from `Not started` to `Materially changed`.
|
||||
|
||||
---
|
||||
|
||||
## TST-18 — Hosted-service wrappers untested `Low` · `—`
|
||||
|
||||
@@ -166,6 +166,11 @@ when the gateway's alarm monitor restarts and drops a truncated verdict. Track
|
||||
it if you need set completeness on a live feed without polling
|
||||
`QueryActiveAlarmsAsync`.
|
||||
|
||||
The CLI's text rows always spell out `degraded=false` and `truncated=false` so
|
||||
healthy and clearing frames remain visible. Its protobuf-JSON modes retain the
|
||||
standard proto3 omission rule, so those same false-valued payloads appear as
|
||||
`{"providerStatus": {}}` and `{"snapshotStatus": {}}`; absent means false.
|
||||
|
||||
`MxGatewaySession.CloseAsync` is explicit and idempotent. Repeated calls return
|
||||
the first `CloseSessionReply` instead of sending another close request.
|
||||
|
||||
@@ -285,7 +290,7 @@ dotnet run --project clients/dotnet/ZB.MOM.WW.MxGateway.Client.Cli -- advise --s
|
||||
dotnet run --project clients/dotnet/ZB.MOM.WW.MxGateway.Client.Cli -- write --session-id <id> --server-handle 1 --item-handle 1 --type int32 --value 123 --json
|
||||
dotnet run --project clients/dotnet/ZB.MOM.WW.MxGateway.Client.Cli -- write2 --session-id <id> --server-handle 1 --item-handle 1 --type int32 --value 123 --timestamp 2026-01-01T00:00:00Z --json
|
||||
dotnet run --project clients/dotnet/ZB.MOM.WW.MxGateway.Client.Cli -- stream-events --session-id <id> --max-events 1 --json
|
||||
dotnet run --project clients/dotnet/ZB.MOM.WW.MxGateway.Client.Cli -- stream-alarms --filter-prefix Area001 --max-events 1 --json
|
||||
dotnet run --project clients/dotnet/ZB.MOM.WW.MxGateway.Client.Cli -- stream-alarms --filter-prefix Area001 --max-events 4 --json
|
||||
dotnet run --project clients/dotnet/ZB.MOM.WW.MxGateway.Client.Cli -- acknowledge-alarm --reference "\\Galaxy\Area001.Pump001.PumpFault" --comment "ack from cli" --operator operator1 --json
|
||||
dotnet run --project clients/dotnet/ZB.MOM.WW.MxGateway.Client.Cli -- smoke --endpoint http://localhost:5000 --api-key-env MXGATEWAY_API_KEY --item Area001.Pump001.Speed --json
|
||||
```
|
||||
|
||||
@@ -1561,9 +1561,9 @@ public static class MxGatewayClientCli
|
||||
AlarmFeedMessage.PayloadOneofCase.Transition =>
|
||||
$"transition {ProtobufJsonFormatter.Format(feedMessage.Transition)}",
|
||||
AlarmFeedMessage.PayloadOneofCase.ProviderStatus =>
|
||||
$"provider-status {ProtobufJsonFormatter.Format(feedMessage.ProviderStatus)}",
|
||||
$"provider-status mode={feedMessage.ProviderStatus.Mode} degraded={feedMessage.ProviderStatus.Degraded.ToString().ToLowerInvariant()} reason=\"{feedMessage.ProviderStatus.Reason}\"",
|
||||
AlarmFeedMessage.PayloadOneofCase.SnapshotStatus =>
|
||||
$"snapshot-status {ProtobufJsonFormatter.Format(feedMessage.SnapshotStatus)}",
|
||||
$"snapshot-status truncated={feedMessage.SnapshotStatus.Truncated.ToString().ToLowerInvariant()}",
|
||||
_ => $"unknown-payload {feedMessage.PayloadCase}",
|
||||
};
|
||||
}
|
||||
|
||||
@@ -705,6 +705,50 @@ public sealed class MxGatewayClientCliTests
|
||||
Assert.Equal(string.Empty, error.ToString());
|
||||
}
|
||||
|
||||
/// <summary>Verifies that false alarm-feed status flags remain explicit in text while protobuf JSON omits defaults.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_StreamAlarms_StatusFramesDistinguishTextFromProtobufJson()
|
||||
{
|
||||
FakeCliClient fakeClient = new();
|
||||
fakeClient.AlarmFeedMessages.Add(new AlarmFeedMessage
|
||||
{
|
||||
ProviderStatus = new AlarmProviderStatus { Degraded = false },
|
||||
});
|
||||
fakeClient.AlarmFeedMessages.Add(new AlarmFeedMessage
|
||||
{
|
||||
SnapshotStatus = new AlarmSnapshotStatus { Truncated = false },
|
||||
});
|
||||
|
||||
using var textOutput = new StringWriter();
|
||||
using var textError = new StringWriter();
|
||||
int textExit = await MxGatewayClientCli.RunAsync(
|
||||
["stream-alarms", "--endpoint", "http://localhost:5000", "--api-key", "test-api-key", "--max-events", "2"],
|
||||
textOutput,
|
||||
textError,
|
||||
_ => fakeClient);
|
||||
|
||||
Assert.Equal(0, textExit);
|
||||
Assert.Equal(
|
||||
["provider-status mode=Unspecified degraded=false reason=\"\"", "snapshot-status truncated=false"],
|
||||
textOutput.ToString().Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries));
|
||||
Assert.Equal(string.Empty, textError.ToString());
|
||||
|
||||
using var jsonOutput = new StringWriter();
|
||||
using var jsonError = new StringWriter();
|
||||
int jsonExit = await MxGatewayClientCli.RunAsync(
|
||||
["stream-alarms", "--endpoint", "http://localhost:5000", "--api-key", "test-api-key", "--max-events", "2", "--jsonl"],
|
||||
jsonOutput,
|
||||
jsonError,
|
||||
_ => fakeClient);
|
||||
|
||||
Assert.Equal(0, jsonExit);
|
||||
Assert.Equal(
|
||||
["{ \"providerStatus\": { } }", "{ \"snapshotStatus\": { } }"],
|
||||
jsonOutput.ToString().Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries));
|
||||
Assert.Equal(string.Empty, jsonError.ToString());
|
||||
}
|
||||
|
||||
/// <summary>Verifies that acknowledge-alarm builds a request and prints the JSON reply.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
|
||||
@@ -19,7 +19,7 @@
|
||||
<PropertyGroup>
|
||||
<IsPackable>true</IsPackable>
|
||||
<PackageId>ZB.MOM.WW.MxGateway.Client</PackageId>
|
||||
<Version>0.2.0</Version>
|
||||
<Version>0.2.1</Version>
|
||||
<Description>.NET 10 gRPC client for the MxAccessGateway service. Provides typed wrappers, retry, and a lazy-browse walker over the Galaxy Repository hierarchy.</Description>
|
||||
<PackageReadmeFile>README.md</PackageReadmeFile>
|
||||
<!-- Only the shipped library generates XML docs (matching src/Contracts). The Cli and
|
||||
|
||||
@@ -162,6 +162,11 @@ on every verdict change — including the clearing frame sent when the gateway's
|
||||
alarm monitor restarts and drops a truncated verdict. Track it if you need set
|
||||
completeness on a live feed without polling `QueryActiveAlarms`.
|
||||
|
||||
The CLI's text rows always spell out `degraded=false` and `truncated=false`.
|
||||
Its protobuf-JSON mode retains the standard proto3 omission rule, so those
|
||||
false-valued payloads appear as `{"providerStatus":{}}` and
|
||||
`{"snapshotStatus":{}}`; absent means false.
|
||||
|
||||
## Write Semantics And Common Pitfalls
|
||||
|
||||
These are MXAccess parity behaviors that surprise new callers. The gateway
|
||||
@@ -439,7 +444,7 @@ go run ./cmd/mxgw-go write -session-id <id> -server-handle 1 -item-handle 1 -typ
|
||||
go run ./cmd/mxgw-go write-bulk -session-id <id> -server-handle 1 -item-handles 1,2 -values 10,20 -type int32 -plaintext -json
|
||||
go run ./cmd/mxgw-go read-bulk -session-id <id> -item-handles 1,2 -plaintext -json
|
||||
go run ./cmd/mxgw-go stream-events -session-id <id> -plaintext -json
|
||||
go run ./cmd/mxgw-go stream-alarms -plaintext -json
|
||||
go run ./cmd/mxgw-go stream-alarms -plaintext -limit 4 -json
|
||||
go run ./cmd/mxgw-go smoke -item Area001.Tag.Value -plaintext -json
|
||||
go run ./cmd/mxgw-go galaxy-test-connection -plaintext -json
|
||||
go run ./cmd/mxgw-go galaxy-last-deploy -plaintext -json
|
||||
|
||||
@@ -283,6 +283,68 @@ type pingFakeGateway struct {
|
||||
pb.UnimplementedMxAccessGatewayServer
|
||||
}
|
||||
|
||||
// alarmStatusFakeGateway emits both status arms with their proto3 boolean
|
||||
// defaults so the CLI contract covers values that protobuf JSON omits.
|
||||
type alarmStatusFakeGateway struct {
|
||||
pb.UnimplementedMxAccessGatewayServer
|
||||
}
|
||||
|
||||
func (g *alarmStatusFakeGateway) StreamAlarms(
|
||||
_ *pb.StreamAlarmsRequest,
|
||||
stream grpc.ServerStreamingServer[pb.AlarmFeedMessage],
|
||||
) error {
|
||||
if err := stream.Send(&pb.AlarmFeedMessage{
|
||||
Payload: &pb.AlarmFeedMessage_ProviderStatus{
|
||||
ProviderStatus: &pb.AlarmProviderStatus{Degraded: false},
|
||||
},
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return stream.Send(&pb.AlarmFeedMessage{
|
||||
Payload: &pb.AlarmFeedMessage_SnapshotStatus{
|
||||
SnapshotStatus: &pb.AlarmSnapshotStatus{Truncated: false},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func startAlarmStatusGateway(t *testing.T) string {
|
||||
t.Helper()
|
||||
listener, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatalf("listen: %v", err)
|
||||
}
|
||||
server := grpc.NewServer()
|
||||
pb.RegisterMxAccessGatewayServer(server, &alarmStatusFakeGateway{})
|
||||
go func() { _ = server.Serve(listener) }()
|
||||
t.Cleanup(func() {
|
||||
server.Stop()
|
||||
_ = listener.Close()
|
||||
})
|
||||
return listener.Addr().String()
|
||||
}
|
||||
|
||||
func TestRunStreamAlarmsPrintsStatusArmsInTextAndProtobufJSON(t *testing.T) {
|
||||
endpoint := startAlarmStatusGateway(t)
|
||||
baseArgs := []string{"stream-alarms", "-endpoint", endpoint, "-plaintext", "-api-key", "test", "-limit", "2"}
|
||||
|
||||
var textOut, textErr bytes.Buffer
|
||||
if err := runWithIO(t.Context(), baseArgs, &textOut, &textErr); err != nil {
|
||||
t.Fatalf("text stream-alarms error = %v; stderr = %s", err, textErr.String())
|
||||
}
|
||||
if got, want := strings.TrimSpace(textOut.String()), "provider-status mode=ALARM_PROVIDER_MODE_UNSPECIFIED degraded=false reason=\"\"\nsnapshot-status truncated=false"; got != want {
|
||||
t.Fatalf("text stream-alarms output = %q, want %q", got, want)
|
||||
}
|
||||
|
||||
var jsonOut, jsonErr bytes.Buffer
|
||||
jsonArgs := append(append([]string{}, baseArgs...), "-json")
|
||||
if err := runWithIO(t.Context(), jsonArgs, &jsonOut, &jsonErr); err != nil {
|
||||
t.Fatalf("JSON stream-alarms error = %v; stderr = %s", err, jsonErr.String())
|
||||
}
|
||||
if got, want := strings.TrimSpace(jsonOut.String()), "{\"providerStatus\":{}}\n{\"snapshotStatus\":{}}"; got != want {
|
||||
t.Fatalf("JSON stream-alarms output = %q, want protobuf JSON %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func (g *pingFakeGateway) Invoke(_ context.Context, req *pb.MxCommandRequest) (*pb.MxCommandReply, error) {
|
||||
echo := "pong:" + req.GetCommand().GetPing().GetMessage()
|
||||
return &pb.MxCommandReply{
|
||||
|
||||
@@ -6,7 +6,10 @@ $ErrorActionPreference = 'Stop'
|
||||
# those header stamps, so a regeneration on an off-pin machine would churn the tree and make
|
||||
# check-codegen Check 4 false-fail (or mask real drift under churn). Assert the exact versions
|
||||
# so a regen is deterministic. protoc itself is warn-only (source_code_info is normalized out of
|
||||
# the committed bindings), matching publish-client-proto-inputs.ps1.
|
||||
# the committed bindings), matching publish-client-proto-inputs.ps1. On Windows a plugin reports
|
||||
# its argv[0] name, so the banner carries a trailing `.exe` (e.g. "protoc-gen-go.exe v1.36.11");
|
||||
# Get-NormalizedToolVersion strips that suffix before the pin compare so Check 4 runs the same on
|
||||
# Windows as everywhere else.
|
||||
$PinnedProtocGenGoVersion = 'protoc-gen-go v1.36.11'
|
||||
$PinnedProtocGenGoGrpcVersion = 'protoc-gen-go-grpc 1.6.2'
|
||||
$PinnedProtocVersion = 'libprotoc 34.1'
|
||||
@@ -16,6 +19,14 @@ $protoRoot = Join-Path $repoRoot 'src\ZB.MOM.WW.MxGateway.Contracts\Protos'
|
||||
$outputRoot = Join-Path $PSScriptRoot 'internal\generated'
|
||||
$modulePath = 'gitea.dohertylan.com/dohertj2/mxaccessgw/clients/go/internal/generated'
|
||||
|
||||
function Get-NormalizedToolVersion {
|
||||
# On Windows a plugin reports its argv[0] name, so the banner carries an `.exe`
|
||||
# suffix ("protoc-gen-go.exe v1.36.11"). Strip it so the pin compare is
|
||||
# host-independent; the version part must still match exactly.
|
||||
param([string]$RawBanner)
|
||||
return ($RawBanner -replace '\.exe(?=\s)', '')
|
||||
}
|
||||
|
||||
function Resolve-Tool {
|
||||
# Resolve a codegen tool from PATH first (portable), then the documented Windows fallbacks,
|
||||
# instead of the previous hard-coded per-machine paths. See docs/ToolchainLinks.md.
|
||||
@@ -51,17 +62,17 @@ $protocGenGoGrpc = Resolve-Tool -Names @('protoc-gen-go-grpc', 'protoc-gen-go-gr
|
||||
|
||||
# Assert the pinned plugin versions before generating so Check 4 cannot false-fail (or mask drift)
|
||||
# on an off-pin machine. protoc is warn-only.
|
||||
$protocGenGoVersion = (& $protocGenGo --version 2>&1 | Out-String).Trim()
|
||||
$protocGenGoVersion = Get-NormalizedToolVersion (& $protocGenGo --version 2>&1 | Out-String).Trim()
|
||||
if ($protocGenGoVersion -ne $PinnedProtocGenGoVersion) {
|
||||
throw "protoc-gen-go reports '$protocGenGoVersion', but regeneration is pinned to '$PinnedProtocGenGoVersion'. " +
|
||||
"Install the pin: go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11"
|
||||
}
|
||||
$protocGenGoGrpcVersion = (& $protocGenGoGrpc --version 2>&1 | Out-String).Trim()
|
||||
$protocGenGoGrpcVersion = Get-NormalizedToolVersion (& $protocGenGoGrpc --version 2>&1 | Out-String).Trim()
|
||||
if ($protocGenGoGrpcVersion -ne $PinnedProtocGenGoGrpcVersion) {
|
||||
throw "protoc-gen-go-grpc reports '$protocGenGoGrpcVersion', but regeneration is pinned to '$PinnedProtocGenGoGrpcVersion'. " +
|
||||
"Install the pin: go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.6.2"
|
||||
}
|
||||
$protocVersion = (& $protoc --version 2>&1 | Out-String).Trim()
|
||||
$protocVersion = Get-NormalizedToolVersion (& $protoc --version 2>&1 | Out-String).Trim()
|
||||
if ($protocVersion -ne $PinnedProtocVersion) {
|
||||
Write-Warning "protoc reports '$protocVersion', pin is '$PinnedProtocVersion'. Descriptor comments are normalized out of the committed Go bindings, so patch drift is tolerated; keep CI on the pin."
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -127,9 +127,14 @@ type (
|
||||
QueryActiveAlarmsRequest = pb.QueryActiveAlarmsRequest
|
||||
// StreamAlarmsRequest is the gateway StreamAlarms request message.
|
||||
StreamAlarmsRequest = pb.StreamAlarmsRequest
|
||||
// AlarmFeedMessage is one message on the StreamAlarms feed — an
|
||||
// active-alarm snapshot row, a snapshot-complete sentinel, or a transition.
|
||||
// AlarmFeedMessage is one message on the StreamAlarms feed — an active-alarm
|
||||
// snapshot row, snapshot-complete sentinel, transition, provider status, or
|
||||
// snapshot-completeness status.
|
||||
AlarmFeedMessage = pb.AlarmFeedMessage
|
||||
// AlarmProviderStatus reports the alarm provider mode and degradation state.
|
||||
AlarmProviderStatus = pb.AlarmProviderStatus
|
||||
// AlarmSnapshotStatus reports whether the cached alarm snapshot is truncated.
|
||||
AlarmSnapshotStatus = pb.AlarmSnapshotStatus
|
||||
// ActiveAlarmSnapshot is one row in a ConditionRefresh stream.
|
||||
ActiveAlarmSnapshot = pb.ActiveAlarmSnapshot
|
||||
// OnAlarmTransitionEvent is the body carried by alarm-transition MxEvents.
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
package mxgateway_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.dohertylan.com/dohertj2/mxaccessgw/clients/go/mxgateway"
|
||||
)
|
||||
|
||||
// TestAlarmFeedStatusAliasesArePublic is a compile-time consumer contract for
|
||||
// the two status payloads carried by AlarmFeedMessage.
|
||||
func TestAlarmFeedStatusAliasesArePublic(t *testing.T) {
|
||||
t.Helper()
|
||||
var provider mxgateway.AlarmProviderStatus
|
||||
var snapshot mxgateway.AlarmSnapshotStatus
|
||||
_ = provider
|
||||
_ = snapshot
|
||||
}
|
||||
@@ -3,7 +3,7 @@ package mxgateway
|
||||
const (
|
||||
// ClientVersion is the released semantic version of this Go client module.
|
||||
// Keep it in sync with the module tag applied by scripts/tag-go-module.ps1.
|
||||
ClientVersion = "0.2.0"
|
||||
ClientVersion = "0.2.1"
|
||||
|
||||
// GatewayProtocolVersion matches GatewayContractInfo.GatewayProtocolVersion
|
||||
// in the shared .NET contracts.
|
||||
|
||||
@@ -135,6 +135,11 @@ the clearing frame sent when the gateway's alarm monitor restarts and drops a
|
||||
truncated verdict. Track it if you need set completeness on a live feed without
|
||||
polling `queryActiveAlarms`.
|
||||
|
||||
The CLI's text rows always spell out `degraded=false` and `truncated=false`.
|
||||
Its protobuf-JSON mode retains the standard proto3 omission rule, so those
|
||||
false-valued payloads appear as `{"providerStatus":{}}` and
|
||||
`{"snapshotStatus":{}}`; absent means false.
|
||||
|
||||
## Write Semantics And Common Pitfalls
|
||||
|
||||
These are MXAccess parity behaviors that surprise new callers. The gateway
|
||||
@@ -411,7 +416,7 @@ gradle :zb-mom-ww-mxgateway-cli:run --args="advise-supervisory --endpoint localh
|
||||
gradle :zb-mom-ww-mxgateway-cli:run --args="authenticate-user --endpoint localhost:5000 --api-key-env MXGATEWAY_API_KEY --plaintext --session-id <id> --server-handle 1 --verify-user operator --password-env MXGATEWAY_VERIFY_PASSWORD --json"
|
||||
gradle :zb-mom-ww-mxgateway-cli:run --args="write-secured --endpoint localhost:5000 --api-key-env MXGATEWAY_API_KEY --plaintext --session-id <id> --server-handle 1 --item-handle 1 --type int32 --value 123 --current-user-id 100 --verifier-user-id 100 --json"
|
||||
gradle :zb-mom-ww-mxgateway-cli:run --args="stream-events --endpoint localhost:5000 --api-key-env MXGATEWAY_API_KEY --plaintext --session-id <id> --limit 1 --json"
|
||||
gradle :zb-mom-ww-mxgateway-cli:run --args="stream-alarms --endpoint localhost:5000 --api-key-env MXGATEWAY_API_KEY --plaintext --filter-prefix Galaxy --limit 1 --json"
|
||||
gradle :zb-mom-ww-mxgateway-cli:run --args="stream-alarms --endpoint localhost:5000 --api-key-env MXGATEWAY_API_KEY --plaintext --filter-prefix Galaxy --limit 4 --json"
|
||||
gradle :zb-mom-ww-mxgateway-cli:run --args="acknowledge-alarm --endpoint localhost:5000 --api-key-env MXGATEWAY_API_KEY --plaintext --reference \"\\Galaxy\Area001.Pump001.PumpFault\" --json"
|
||||
gradle :zb-mom-ww-mxgateway-cli:run --args="smoke --endpoint localhost:5000 --api-key-env MXGATEWAY_API_KEY --plaintext --item TestObject.TestInt --json"
|
||||
```
|
||||
@@ -483,7 +488,7 @@ repositories {
|
||||
}
|
||||
|
||||
dependencies {
|
||||
implementation 'com.zb.mom.ww.mxgateway:zb-mom-ww-mxgateway-client:0.2.1'
|
||||
implementation 'com.zb.mom.ww.mxgateway:zb-mom-ww-mxgateway-client:0.2.2'
|
||||
}
|
||||
````
|
||||
|
||||
|
||||
@@ -16,11 +16,14 @@ subprojects {
|
||||
// 0.2.0 was already published to the Gitea Maven feed on 2026-06-26,
|
||||
// before the CLI-37/38/40/41 conformance fixes changed the client's
|
||||
// observable behavior (status.category-based validation, hresult < 0,
|
||||
// exact-secret redaction, typed malformed-reply errors). Bump to 0.2.1
|
||||
// so the published coordinate matches the conformant behavior the other
|
||||
// four clients ship at 0.2.0 for the first time. See CLI-39 and the
|
||||
// exact-secret redaction, typed malformed-reply errors). Bumped to 0.2.1
|
||||
// so the published coordinate matched the conformant behavior the other
|
||||
// four clients shipped at 0.2.0 for the first time. See CLI-39 and the
|
||||
// "Versioning" section of docs/ClientPackaging.md.
|
||||
version = '0.2.1'
|
||||
// 0.2.1 was itself published (Maven, 2026-08-07); this is the next
|
||||
// release-train bump (CLI-46), one ahead of the other four clients'
|
||||
// 0.2.1 because the Java coordinate has always trailed by one number.
|
||||
version = '0.2.2'
|
||||
|
||||
pluginManager.withPlugin('java') {
|
||||
java {
|
||||
|
||||
+1796
-608
File diff suppressed because it is too large
Load Diff
+25
-2
@@ -27,6 +27,8 @@ import mxaccess_gateway.v1.MxaccessGateway.ActiveAlarmSnapshot;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.AddItemReply;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.AlarmConditionState;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.AlarmFeedMessage;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.AlarmProviderStatus;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.AlarmSnapshotStatus;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.AlarmTransitionKind;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.BulkReadResult;
|
||||
import mxaccess_gateway.v1.MxaccessGateway.BulkWriteResult;
|
||||
@@ -60,7 +62,7 @@ final class MxGatewayCliTests {
|
||||
|
||||
assertEquals(0, run.exitCode());
|
||||
assertEquals("", run.errors());
|
||||
assertTrue(run.output().contains("mxgateway-java 0.2.1"));
|
||||
assertTrue(run.output().contains("mxgateway-java 0.2.2"));
|
||||
assertTrue(run.output().contains("gatewayProtocolVersion=3"));
|
||||
assertTrue(run.output().contains("workerProtocolVersion=1"));
|
||||
}
|
||||
@@ -90,7 +92,7 @@ final class MxGatewayCliTests {
|
||||
CliRun run = execute(new FakeClientFactory(), "version", "--json");
|
||||
|
||||
assertEquals(0, run.exitCode());
|
||||
assertTrue(run.output().contains("\"clientVersion\":\"0.2.1\""));
|
||||
assertTrue(run.output().contains("\"clientVersion\":\"0.2.2\""));
|
||||
assertTrue(run.output().contains("\"gatewayProtocolVersion\":3"));
|
||||
}
|
||||
|
||||
@@ -685,6 +687,21 @@ final class MxGatewayCliTests {
|
||||
assertTrue(run.output().contains("\"snapshotComplete\""), run.output());
|
||||
}
|
||||
|
||||
@Test
|
||||
void streamAlarmsCommandPrintsStatusArmsInTextAndProtobufJson() {
|
||||
FakeClientFactory factory = new FakeClientFactory();
|
||||
|
||||
CliRun text = execute(factory, "stream-alarms");
|
||||
List<String> textRows = text.output().lines().filter(line -> !line.isBlank()).toList();
|
||||
assertEquals("provider-status mode=ALARM_PROVIDER_MODE_UNSPECIFIED degraded=false reason=", textRows.get(3));
|
||||
assertEquals("snapshot-status truncated=false", textRows.get(4));
|
||||
|
||||
CliRun json = execute(factory, "stream-alarms", "--json");
|
||||
List<String> jsonRows = json.output().lines().filter(line -> !line.isBlank()).toList();
|
||||
assertEquals("{\"providerStatus\":{}}", jsonRows.get(3));
|
||||
assertEquals("{\"snapshotStatus\":{}}", jsonRows.get(4));
|
||||
}
|
||||
|
||||
@Test
|
||||
void acknowledgeAlarmCommandForwardsOptionsAndPrintsReply() {
|
||||
FakeClientFactory factory = new FakeClientFactory();
|
||||
@@ -1409,6 +1426,12 @@ final class MxGatewayCliTests {
|
||||
.setTransitionKind(AlarmTransitionKind.ALARM_TRANSITION_KIND_ACKNOWLEDGE)
|
||||
.setSeverity(700))
|
||||
.build());
|
||||
observer.onNext(AlarmFeedMessage.newBuilder()
|
||||
.setProviderStatus(AlarmProviderStatus.newBuilder().setDegraded(false))
|
||||
.build());
|
||||
observer.onNext(AlarmFeedMessage.newBuilder()
|
||||
.setSnapshotStatus(AlarmSnapshotStatus.newBuilder().setTruncated(false))
|
||||
.build());
|
||||
observer.onCompleted();
|
||||
return new MxGatewayAlarmFeedSubscription();
|
||||
}
|
||||
|
||||
@@ -72,16 +72,18 @@ tasks.register('checkGeneratedClean') {
|
||||
group = 'verification'
|
||||
description = 'Fails if the committed generated Java tree differs from a fresh regeneration.'
|
||||
dependsOn 'generateProto'
|
||||
def generatedDir = 'clients/java/src/main/generated'
|
||||
def repoRoot = rootProject.projectDir.parentFile.parentFile
|
||||
// Project.exec was removed in Gradle 9; ProviderFactory.exec runs the git-status probe
|
||||
// lazily (captured at configuration time, evaluated in doLast) and works on both the
|
||||
// installed Gradle and Gradle 9.
|
||||
def gitStatus = providers.exec {
|
||||
workingDir(repoRoot)
|
||||
commandLine('git', 'status', '--porcelain', '--', generatedDir)
|
||||
ignoreExitValue = true
|
||||
}
|
||||
doLast {
|
||||
def generatedDir = 'clients/java/src/main/generated'
|
||||
def stdout = new ByteArrayOutputStream()
|
||||
def result = exec {
|
||||
workingDir = rootProject.projectDir.parentFile.parentFile
|
||||
commandLine 'git', 'status', '--porcelain', '--', generatedDir
|
||||
standardOutput = stdout
|
||||
ignoreExitValue = true
|
||||
}
|
||||
def dirty = stdout.toString().trim()
|
||||
def dirty = gitStatus.standardOutput.asText.get().trim()
|
||||
if (!dirty.isEmpty()) {
|
||||
throw new GradleException(
|
||||
"Generated Java is stale:\n${dirty}\n" +
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ package com.zb.mom.ww.mxgateway.client;
|
||||
public final class MxGatewayClientVersion {
|
||||
private static final int GATEWAY_PROTOCOL_VERSION = 3;
|
||||
private static final int WORKER_PROTOCOL_VERSION = 1;
|
||||
private static final String CLIENT_VERSION = "0.2.1";
|
||||
private static final String CLIENT_VERSION = "0.2.2";
|
||||
|
||||
private MxGatewayClientVersion() {
|
||||
}
|
||||
|
||||
Binary file not shown.
@@ -132,6 +132,14 @@ sent when the gateway's alarm monitor restarts and drops a truncated verdict.
|
||||
Track it if you need set completeness on a live feed without polling
|
||||
`query_active_alarms`.
|
||||
|
||||
`mxgw-py stream-alarms` writes one JSON row as each feed message arrives. The
|
||||
default `--max-messages 0` is unbounded; use a positive value for a finite
|
||||
sample, remembering that the provider-status and snapshot-status preamble both
|
||||
count toward it. `--timeout` ends the command after an idle wait and the stream
|
||||
is closed in every exit path. Protobuf JSON omits false-valued scalar defaults,
|
||||
so healthy/clearing frames appear as `{"providerStatus": {}}` and
|
||||
`{"snapshotStatus": {}}`; absent means false.
|
||||
|
||||
Canceling a Python task cancels the client-side gRPC call or stream wait. It
|
||||
does not abort an in-flight MXAccess COM call inside the worker process.
|
||||
|
||||
@@ -448,7 +456,7 @@ mxgw-py register --session-id <id> --client-name python-client --json
|
||||
mxgw-py add-item --session-id <id> --server-handle 1 --item Object.Attribute --json
|
||||
mxgw-py advise --session-id <id> --server-handle 1 --item-handle 2 --json
|
||||
mxgw-py stream-events --session-id <id> --max-events 1 --json
|
||||
mxgw-py stream-alarms --max-messages 1 --json
|
||||
mxgw-py stream-alarms --max-messages 4 --json
|
||||
mxgw-py acknowledge-alarm --reference "\\Galaxy\Area001.Pump001.PumpFault" --json
|
||||
mxgw-py write --session-id <id> --server-handle 1 --item-handle 2 --type int32 --value 123 --json
|
||||
```
|
||||
@@ -484,6 +492,15 @@ mxgw-py smoke --endpoint $env:MXGATEWAY_ENDPOINT --plaintext --api-key-env MXGAT
|
||||
|
||||
## Installing from the Gitea PyPI Feed
|
||||
|
||||
**The published `0.2.0` artifact cannot connect.** Its `connect()` built the
|
||||
`grpc.aio` channel inside `asyncio.to_thread`, and a `grpc.aio` channel binds
|
||||
to the event loop current on the *constructing* thread — so every
|
||||
non-stub connection raised `RuntimeError: There is no current event loop in
|
||||
thread`. Fixed in-tree by commit `a8f86b5` (`resolve_channel_security` now
|
||||
runs off-loop, `create_channel` runs on-loop). The fix is superseded by
|
||||
`0.2.1`; do not install `0.2.0` from the feed below — use `0.2.1` or later,
|
||||
or install from source at a commit at or after `a8f86b5`.
|
||||
|
||||
The client publishes to the internal Gitea PyPI feed:
|
||||
|
||||
````bash
|
||||
|
||||
@@ -36,7 +36,11 @@ function Resolve-Python {
|
||||
function Assert-GrpcioToolsVersion {
|
||||
param([string]$Python)
|
||||
|
||||
$version = (& $Python -c 'import grpc_tools; from importlib.metadata import version; print(version("grpcio-tools"))').Trim()
|
||||
# Windows PowerShell 5.1 strips embedded double quotes when passing an argument to a native
|
||||
# exe (pwsh 7 does not), so a Python literal quoted with " " here would arrive at Python as
|
||||
# print(version(grpcio-tools)) -> NameError. Use single quotes for the Python string literal
|
||||
# inside this double-quoted PowerShell string so the quoting survives on both hosts.
|
||||
$version = (& $Python -c "import grpc_tools; from importlib.metadata import version; print(version('grpcio-tools'))").Trim()
|
||||
if ($version -ne $PinnedGrpcioToolsVersion) {
|
||||
throw "grpcio-tools $version is installed, but regeneration is pinned to $PinnedGrpcioToolsVersion. " +
|
||||
"Install the pin (python -m pip install 'grpcio-tools==$PinnedGrpcioToolsVersion') before regenerating, " +
|
||||
|
||||
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
||||
|
||||
[project]
|
||||
name = "zb-mom-ww-mxaccess-gateway-client"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
description = "Async Python client for MXAccess Gateway."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12"
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1,3 +1,3 @@
|
||||
"""Package version information."""
|
||||
|
||||
__version__ = "0.2.0"
|
||||
__version__ = "0.2.1"
|
||||
|
||||
@@ -511,16 +511,15 @@ def stream_events(**kwargs: Any) -> None:
|
||||
@main.command("stream-alarms")
|
||||
@gateway_options
|
||||
@click.option("--filter-prefix", default="", help="Alarm-reference prefix filter.")
|
||||
@click.option("--max-messages", default=1, type=int, show_default=True)
|
||||
@click.option("--max-messages", default=0, type=int, show_default=True, help="Maximum rows to emit; 0 means unbounded.")
|
||||
@click.option("--timeout", default=5.0, type=float, show_default=True)
|
||||
@click.option("--correlation-id", default="", help="Client correlation id.")
|
||||
@click.option("--json", "output_json", is_flag=True, help="Emit JSON output.")
|
||||
def stream_alarms(**kwargs: Any) -> None:
|
||||
"""Stream a bounded number of messages from the gateway's central alarm feed."""
|
||||
"""Stream messages from the gateway's central alarm feed as they arrive."""
|
||||
|
||||
_run(
|
||||
_run_streaming(
|
||||
_stream_alarms(**kwargs),
|
||||
output_json=kwargs["output_json"],
|
||||
secrets=_secrets(kwargs),
|
||||
)
|
||||
|
||||
@@ -1119,19 +1118,39 @@ async def _stream_events(**kwargs: Any) -> dict[str, Any]:
|
||||
return {"events": [_event_row(event) for event in events]}
|
||||
|
||||
|
||||
async def _stream_alarms(**kwargs: Any) -> dict[str, Any]:
|
||||
async with await _connect(kwargs) as client:
|
||||
messages = await _collect_alarm_messages(
|
||||
client.stream_alarms(
|
||||
pb.StreamAlarmsRequest(
|
||||
client_correlation_id=kwargs["correlation_id"],
|
||||
alarm_filter_prefix=kwargs["filter_prefix"],
|
||||
),
|
||||
),
|
||||
max_messages=kwargs["max_messages"],
|
||||
timeout=kwargs["timeout"],
|
||||
async def _stream_alarms(**kwargs: Any) -> None:
|
||||
max_messages = int(kwargs["max_messages"])
|
||||
if max_messages > MAX_AGGREGATE_EVENTS:
|
||||
raise click.BadParameter(
|
||||
f"must be less than or equal to {MAX_AGGREGATE_EVENTS}",
|
||||
param_hint="--max-messages",
|
||||
)
|
||||
return {"messages": [_message_dict(message) for message in messages]}
|
||||
|
||||
async with await _connect(kwargs) as client:
|
||||
messages = client.stream_alarms(
|
||||
pb.StreamAlarmsRequest(
|
||||
client_correlation_id=kwargs["correlation_id"],
|
||||
alarm_filter_prefix=kwargs["filter_prefix"],
|
||||
),
|
||||
)
|
||||
iterator = messages.__aiter__()
|
||||
emitted = 0
|
||||
try:
|
||||
while max_messages == 0 or emitted < max_messages:
|
||||
try:
|
||||
message = await asyncio.wait_for(
|
||||
iterator.__anext__(),
|
||||
timeout=kwargs["timeout"],
|
||||
)
|
||||
except (StopAsyncIteration, asyncio.TimeoutError):
|
||||
break
|
||||
_emit(_message_dict(message), output_json=kwargs["output_json"])
|
||||
sys.stdout.flush()
|
||||
emitted += 1
|
||||
finally:
|
||||
close = getattr(iterator, "aclose", None)
|
||||
if close is not None:
|
||||
await close()
|
||||
|
||||
|
||||
async def _acknowledge_alarm(**kwargs: Any) -> dict[str, Any]:
|
||||
@@ -1489,6 +1508,17 @@ def _run(
|
||||
_emit(payload, output_json=output_json)
|
||||
|
||||
|
||||
def _run_streaming(
|
||||
awaitable: Awaitable[None],
|
||||
*,
|
||||
secrets: list[str | None],
|
||||
) -> None:
|
||||
try:
|
||||
asyncio.run(awaitable)
|
||||
except MxGatewayError as error:
|
||||
raise click.ClickException(redact_secret(str(error), secrets)) from error
|
||||
|
||||
|
||||
def _emit(
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
@@ -1536,34 +1566,6 @@ async def _collect_events(
|
||||
return collected
|
||||
|
||||
|
||||
async def _collect_alarm_messages(
|
||||
messages: Any,
|
||||
*,
|
||||
max_messages: int,
|
||||
timeout: float,
|
||||
) -> list[pb.AlarmFeedMessage]:
|
||||
if max_messages > MAX_AGGREGATE_EVENTS:
|
||||
raise click.BadParameter(
|
||||
f"must be less than or equal to {MAX_AGGREGATE_EVENTS}",
|
||||
param_hint="--max-messages",
|
||||
)
|
||||
|
||||
collected: list[pb.AlarmFeedMessage] = []
|
||||
iterator = messages.__aiter__()
|
||||
try:
|
||||
while len(collected) < max_messages:
|
||||
collected.append(await asyncio.wait_for(iterator.__anext__(), timeout=timeout))
|
||||
except StopAsyncIteration:
|
||||
pass
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
finally:
|
||||
close = getattr(iterator, "aclose", None)
|
||||
if close is not None:
|
||||
await close()
|
||||
return collected
|
||||
|
||||
|
||||
async def _collect_deploy_events(
|
||||
events: Any,
|
||||
*,
|
||||
|
||||
@@ -7,6 +7,7 @@ fixed source.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import time as _time_module_ref
|
||||
@@ -296,6 +297,22 @@ class _AlarmFakeStream:
|
||||
self.cancelled = True
|
||||
|
||||
|
||||
class _IncrementalAlarmFakeStream(_AlarmFakeStream):
|
||||
def __init__(self, messages: list[pb.AlarmFeedMessage], emitted: list[str]) -> None:
|
||||
super().__init__(messages)
|
||||
self._emitted = emitted
|
||||
self._yielded = 0
|
||||
|
||||
async def __anext__(self) -> pb.AlarmFeedMessage:
|
||||
if self._yielded > 0:
|
||||
assert len(self._emitted) >= self._yielded, "previous alarm row was buffered"
|
||||
if not self._messages:
|
||||
await asyncio.Event().wait()
|
||||
raise AssertionError("unreachable")
|
||||
self._yielded += 1
|
||||
return self._messages.pop(0)
|
||||
|
||||
|
||||
class _BulkFakeUnary:
|
||||
def __init__(self, replies: list[Any]) -> None:
|
||||
self.replies = replies
|
||||
@@ -667,8 +684,15 @@ def test_cli_write_bulk_happy_path(monkeypatch) -> None:
|
||||
assert cmd.kind == pb.MX_COMMAND_KIND_WRITE_BULK
|
||||
|
||||
|
||||
def test_cli_stream_alarms_happy_path(monkeypatch) -> None:
|
||||
transitions = [
|
||||
def test_cli_stream_alarms_default_streams_status_cached_and_live_rows_incrementally(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
messages = [
|
||||
pb.AlarmFeedMessage(provider_status=pb.AlarmProviderStatus(degraded=False)),
|
||||
pb.AlarmFeedMessage(snapshot_status=pb.AlarmSnapshotStatus(truncated=False)),
|
||||
pb.AlarmFeedMessage(
|
||||
active_alarm=pb.ActiveAlarmSnapshot(alarm_full_reference="Tank01.Level.HiHi"),
|
||||
),
|
||||
pb.AlarmFeedMessage(
|
||||
transition=pb.OnAlarmTransitionEvent(
|
||||
alarm_full_reference="Tank01.Level.HiHi",
|
||||
@@ -676,7 +700,15 @@ def test_cli_stream_alarms_happy_path(monkeypatch) -> None:
|
||||
),
|
||||
),
|
||||
]
|
||||
stream = _AlarmFakeStream(transitions)
|
||||
emitted: list[str] = []
|
||||
real_echo = cli_commands.click.echo
|
||||
|
||||
def recording_echo(message=None, **kwargs) -> None:
|
||||
emitted.append(str(message))
|
||||
real_echo(message, **kwargs)
|
||||
|
||||
monkeypatch.setattr(cli_commands.click, "echo", recording_echo)
|
||||
stream = _IncrementalAlarmFakeStream(messages, emitted)
|
||||
stub = _BulkFakeStub()
|
||||
stub.set_alarm_stream(stream)
|
||||
_install_fake_connect(monkeypatch, stub)
|
||||
@@ -689,10 +721,8 @@ def test_cli_stream_alarms_happy_path(monkeypatch) -> None:
|
||||
"--endpoint",
|
||||
"localhost:5000",
|
||||
"--plaintext",
|
||||
"--max-messages",
|
||||
"1",
|
||||
"--timeout",
|
||||
"5.0",
|
||||
"0.01",
|
||||
"--filter-prefix",
|
||||
"Tank01.",
|
||||
"--json",
|
||||
@@ -700,8 +730,44 @@ def test_cli_stream_alarms_happy_path(monkeypatch) -> None:
|
||||
)
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
payload = json.loads(result.output)
|
||||
assert payload["messages"][0]["transition"]["alarmFullReference"] == "Tank01.Level.HiHi"
|
||||
rows = [json.loads(line) for line in result.output.splitlines()]
|
||||
assert [next(iter(row)) for row in rows] == [
|
||||
"providerStatus",
|
||||
"snapshotStatus",
|
||||
"activeAlarm",
|
||||
"transition",
|
||||
]
|
||||
assert stream.cancelled
|
||||
|
||||
|
||||
def test_cli_stream_alarms_positive_max_messages_remains_finite(monkeypatch) -> None:
|
||||
messages = [
|
||||
pb.AlarmFeedMessage(provider_status=pb.AlarmProviderStatus()),
|
||||
pb.AlarmFeedMessage(snapshot_status=pb.AlarmSnapshotStatus()),
|
||||
pb.AlarmFeedMessage(active_alarm=pb.ActiveAlarmSnapshot()),
|
||||
]
|
||||
stream = _AlarmFakeStream(messages)
|
||||
stub = _BulkFakeStub()
|
||||
stub.set_alarm_stream(stream)
|
||||
_install_fake_connect(monkeypatch, stub)
|
||||
|
||||
result = CliRunner().invoke(
|
||||
main,
|
||||
[
|
||||
"stream-alarms",
|
||||
"--endpoint",
|
||||
"localhost:5000",
|
||||
"--plaintext",
|
||||
"--max-messages",
|
||||
"2",
|
||||
"--json",
|
||||
],
|
||||
)
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
rows = [json.loads(line) for line in result.output.splitlines()]
|
||||
assert [next(iter(row)) for row in rows] == ["providerStatus", "snapshotStatus"]
|
||||
assert stream.cancelled
|
||||
|
||||
|
||||
def test_cli_acknowledge_alarm_happy_path(monkeypatch) -> None:
|
||||
|
||||
Generated
+2
-2
@@ -590,7 +590,7 @@ checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
|
||||
|
||||
[[package]]
|
||||
name = "mxgw-cli"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"futures-util",
|
||||
@@ -1490,7 +1490,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zb-mom-ww-mxgateway-client"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "zb-mom-ww-mxgateway-client"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
edition = "2021"
|
||||
authors = ["Joseph Doherty"]
|
||||
description = "Async Rust client for the MxAccessGateway gRPC service, including a lazy-browse walker over the Galaxy Repository hierarchy."
|
||||
@@ -25,7 +25,7 @@ resolver = "2"
|
||||
|
||||
[workspace.package]
|
||||
edition = "2021"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
authors = ["Joseph Doherty"]
|
||||
license = "Proprietary"
|
||||
repository = "https://gitea.dohertylan.com/dohertj2/mxaccessgw"
|
||||
|
||||
@@ -77,7 +77,7 @@ cargo run -p mxgw-cli -- register --session-id <session-id> --client-name mxgw-r
|
||||
cargo run -p mxgw-cli -- add-item --session-id <session-id> --server-handle 1 --item TestChildObject.TestInt --json
|
||||
cargo run -p mxgw-cli -- advise --session-id <session-id> --server-handle 1 --item-handle 1 --json
|
||||
cargo run -p mxgw-cli -- stream-events --session-id <session-id> --max-events 1 --json
|
||||
cargo run -p mxgw-cli -- stream-alarms --session-id <session-id> --max-messages 1 --json
|
||||
cargo run -p mxgw-cli -- stream-alarms --session-id <session-id> --max-messages 4 --json
|
||||
cargo run -p mxgw-cli -- acknowledge-alarm --session-id <session-id> --alarm-reference "\\Galaxy\Area001.Pump001.PumpFault" --json
|
||||
cargo run -p mxgw-cli -- write --session-id <session-id> --server-handle 1 --item-handle 1 --value-type int32 --value 123 --json
|
||||
```
|
||||
@@ -141,6 +141,11 @@ every verdict change — including the clearing frame sent when the gateway's
|
||||
alarm monitor restarts and drops a truncated verdict. Track it if you need set
|
||||
completeness on a live feed without polling `query_active_alarms`.
|
||||
|
||||
Unlike the protobuf-JSON formatters used by the other CLIs, the Rust CLI's
|
||||
hand-built JSON status rows always include `"degraded": false` and
|
||||
`"truncated": false`. This deliberate representational difference is
|
||||
documented in the cross-language smoke matrix; both shapes mean the same thing.
|
||||
|
||||
The session helpers keep MXAccess handles visible:
|
||||
|
||||
```rust
|
||||
|
||||
@@ -130,6 +130,7 @@ message MxCommand {
|
||||
WriteSecuredBulkCommand write_secured_bulk = 41;
|
||||
WriteSecured2BulkCommand write_secured2_bulk = 42;
|
||||
ReadBulkCommand read_bulk = 43;
|
||||
UpdateAlarmWatchListCommand update_alarm_watch_list = 44;
|
||||
PingCommand ping = 100;
|
||||
GetSessionStateCommand get_session_state = 101;
|
||||
GetWorkerInfoCommand get_worker_info = 102;
|
||||
@@ -174,6 +175,7 @@ enum MxCommandKind {
|
||||
MX_COMMAND_KIND_WRITE_SECURED_BULK = 32;
|
||||
MX_COMMAND_KIND_WRITE_SECURED2_BULK = 33;
|
||||
MX_COMMAND_KIND_READ_BULK = 34;
|
||||
MX_COMMAND_KIND_UPDATE_ALARM_WATCH_LIST = 35;
|
||||
MX_COMMAND_KIND_PING = 100;
|
||||
MX_COMMAND_KIND_GET_SESSION_STATE = 101;
|
||||
MX_COMMAND_KIND_GET_WORKER_INFO = 102;
|
||||
@@ -344,8 +346,8 @@ enum AlarmProviderMode {
|
||||
// Subscription expression follows the canonical
|
||||
// `\\<machine>\Galaxy!<area>` format (literal "Galaxy" provider). The
|
||||
// worker spins up a wnwrapConsumer-backed subscription on its STA on
|
||||
// first call; subsequent calls are an error (use UnsubscribeAlarms then
|
||||
// SubscribeAlarms to reconfigure).
|
||||
// first call; subsequent calls are an error. Use UpdateAlarmWatchList to
|
||||
// replace the standby targets without restarting the primary subscription.
|
||||
message SubscribeAlarmsCommand {
|
||||
string subscription_expression = 1;
|
||||
// UNSPECIFIED = auto-failover/failback. ALARMMGR/SUBTAG force one provider.
|
||||
@@ -354,6 +356,17 @@ message SubscribeAlarmsCommand {
|
||||
// alarmmgr mode; in subtag mode it bounds what the consumer can observe.
|
||||
repeated AlarmSubtagTarget watch_list = 3;
|
||||
AlarmFailoverConfig failover = 4;
|
||||
// True when auto mode should retain subtag-fallback capability even if the
|
||||
// gateway's initial discovery produced an empty watch list. False leaves an
|
||||
// empty list as an intentional alarm-manager-only subscription.
|
||||
bool fallback_enabled = 5;
|
||||
}
|
||||
|
||||
// Replace the live alarm subscription's subtag standby targets. The worker
|
||||
// constructs and attaches the replacement before swapping it into service, so
|
||||
// a setup failure leaves the existing standby and active provider unchanged.
|
||||
message UpdateAlarmWatchListCommand {
|
||||
repeated AlarmSubtagTarget watch_list = 1;
|
||||
}
|
||||
|
||||
// Tear down the worker's alarm consumer. No-op if no subscription is
|
||||
@@ -537,11 +550,14 @@ message MxCommandReply {
|
||||
MxValue return_value = 6;
|
||||
// Correlated per-item outcome rows. For WRITE / WRITE2 / WRITE_SECURED /
|
||||
// WRITE_SECURED2 replies the worker holds the reply for a bounded window
|
||||
// (default 1.5 s, MXGATEWAY_WORKER_WRITE_COMPLETION_WAIT_MS) waiting for
|
||||
// (default 1.5 s, MxGateway:Worker:WriteCompletionWaitMilliseconds) waiting for
|
||||
// the matching MXAccess OnWriteComplete callback and copies its status rows
|
||||
// here, so statuses[0] carries the real MXAccess commit outcome (success OR
|
||||
// failure) while protocol_status/hresult still describe command acceptance
|
||||
// only. Empty statuses on a write reply means the completion did not arrive
|
||||
// only. Callback correlation is recency-based per item handle: after the
|
||||
// bounded wait, a sequential write's callback supersedes a timed-out
|
||||
// predecessor, and a later stale callback for that predecessor is ignored.
|
||||
// Empty statuses on a write reply means the completion did not arrive
|
||||
// within the window — the write is unconfirmed, not failed. Correlation is
|
||||
// best-effort per (server_handle, item_handle): MXAccess's callback carries
|
||||
// no transaction id, so concurrent writes to the same item within the
|
||||
|
||||
@@ -13,7 +13,7 @@ pub mod mxaccess_gateway {
|
||||
/// `mxaccess_gateway.v1` package — the v1 wire contract surfaced by the
|
||||
/// gateway to language clients.
|
||||
pub mod v1 {
|
||||
#![allow(clippy::large_enum_variant)]
|
||||
#![allow(clippy::large_enum_variant, clippy::result_large_err)]
|
||||
|
||||
tonic::include_proto!("mxaccess_gateway.v1");
|
||||
}
|
||||
@@ -24,7 +24,7 @@ pub mod mxaccess_worker {
|
||||
/// `mxaccess_worker.v1` package — frame and envelope types used inside
|
||||
/// the named-pipe transport between gateway and worker.
|
||||
pub mod v1 {
|
||||
#![allow(clippy::large_enum_variant)]
|
||||
#![allow(clippy::large_enum_variant, clippy::result_large_err)]
|
||||
|
||||
tonic::include_proto!("mxaccess_worker.v1");
|
||||
}
|
||||
@@ -35,7 +35,7 @@ pub mod galaxy_repository {
|
||||
/// `galaxy_repository.v1` package — types for the Galaxy hierarchy
|
||||
/// discovery and deploy-event watch RPCs.
|
||||
pub mod v1 {
|
||||
#![allow(clippy::large_enum_variant)]
|
||||
#![allow(clippy::large_enum_variant, clippy::result_large_err)]
|
||||
|
||||
tonic::include_proto!("galaxy_repository.v1");
|
||||
}
|
||||
|
||||
@@ -259,7 +259,7 @@ closed. New findings filed against this pass: Server-051..053.
|
||||
|
||||
**Recommendation:** Add a startup hosted service that finds and kills stale worker processes (by executable path / a well-known argument or environment marker) before the server accepts sessions, or update the design docs if reattachment/cleanup is deliberately deferred.
|
||||
|
||||
**Resolution:** Resolved 2026-05-18. Confirmed against source: no code path enumerated or killed leftover workers. Added `IRunningProcessInspector` / `SystemRunningProcessInspector` (a testable seam over `Process.GetProcessesByName`/`Kill`), `OrphanWorkerTerminator` (kills processes matched by the configured worker executable path, or by image name when the x64 gateway cannot introspect the x86 worker's `MainModule`, skipping the current process and tolerating per-process kill failures), and `OrphanWorkerCleanupHostedService` (best-effort `IHostedService`). The hosted service is registered in `AddWorkerProcessLauncher` ahead of `AddGatewaySessions` so cleanup runs before the server accepts sessions. `gateway.md` updated to describe the implemented behavior. Regression tests: `OrphanWorkerTerminatorTests` (`KillsWorkerProcessesMatchingConfiguredExecutablePath`, `KillsImageNameMatchWhenExecutablePathUnreadable`, `DoesNotKillUnrelatedProcessSharingImageName`, `DoesNotKillCurrentProcess`, `ContinuesWhenOneKillThrows`).
|
||||
**Resolution:** Resolved 2026-05-18. Confirmed against source: no code path enumerated or killed leftover workers. Added `IRunningProcessInspector` / `SystemRunningProcessInspector` (a testable seam over `Process.GetProcessesByName`/`Kill`), `OrphanWorkerTerminator` (kills processes matched by the configured worker executable path, or by image name when the x64 gateway cannot introspect the x86 worker's `MainModule`, skipping the current process and tolerating per-process kill failures), and `OrphanWorkerCleanupHostedService` (best-effort `IHostedService`). The hosted service is registered in `AddWorkerProcessLauncher` ahead of `AddGatewaySessions` so cleanup runs before the server accepts sessions. `gateway.md` updated to describe the implemented behavior. Regression tests: `OrphanWorkerTerminatorTests` (`KillsWorkerProcessesMatchingConfiguredExecutablePath`, `KillsImageNameMatchWhenExecutablePathUnreadable`, `DoesNotKillUnrelatedProcessSharingImageName`, `DoesNotKillCurrentProcess`, `ContinuesWhenOneKillThrows`). Re-verified present 2026-08-18 (feat/followups-tickets) — doc sub-claim only; the `gateway.md` orphan-cleanup prose (`OrphanWorkerCleanupHostedService` running `OrphanWorkerTerminator` once on startup, before the server accepts sessions) is still there.
|
||||
|
||||
### Server-003
|
||||
|
||||
@@ -364,7 +364,7 @@ closed. New findings filed against this pass: Server-051..053.
|
||||
|
||||
**Recommendation:** Set `Pooling`, a non-zero `DefaultTimeout`/`busy_timeout`, and enable WAL (`PRAGMA journal_mode=WAL`) once at startup so concurrent readers/writers degrade gracefully.
|
||||
|
||||
**Resolution:** Resolved 2026-05-18. Confirmed against source: the connection string set only `DataSource` and `Mode`. `AuthSqliteConnectionFactory.CreateConnection` now also sets `Pooling = true` and a non-zero `DefaultTimeout`. A new `OpenConnectionAsync(CancellationToken)` opens the connection and applies `PRAGMA journal_mode=WAL` and `PRAGMA busy_timeout` (5 s); WAL is a persistent database-level setting so re-applying it per connection is a cheap no-op, while `busy_timeout` is per-connection state. All nine auth-store call sites (`SqliteApiKeyAdminStore`, `SqliteApiKeyAuditStore`, `SqliteApiKeyStore`, `SqliteAuthStoreMigrator`) were switched from `CreateConnection()` + `OpenAsync()` to `OpenConnectionAsync()`. `docs/Authentication.md` updated to describe the WAL/busy-timeout behavior. Regression test: `SqliteAuthStoreTests.OpenConnectionAsync_EnablesWalJournalModeAndBusyTimeout`.
|
||||
**Resolution:** Resolved 2026-05-18. Confirmed against source: the connection string set only `DataSource` and `Mode`. `AuthSqliteConnectionFactory.CreateConnection` now also sets `Pooling = true` and a non-zero `DefaultTimeout`. A new `OpenConnectionAsync(CancellationToken)` opens the connection and applies `PRAGMA journal_mode=WAL` and `PRAGMA busy_timeout` (5 s); WAL is a persistent database-level setting so re-applying it per connection is a cheap no-op, while `busy_timeout` is per-connection state. All nine auth-store call sites (`SqliteApiKeyAdminStore`, `SqliteApiKeyAuditStore`, `SqliteApiKeyStore`, `SqliteAuthStoreMigrator`) were switched from `CreateConnection()` + `OpenAsync()` to `OpenConnectionAsync()`. `docs/Authentication.md` updated to describe the WAL/busy-timeout behavior. Regression test: `SqliteAuthStoreTests.OpenConnectionAsync_EnablesWalJournalModeAndBusyTimeout`. Regressed or never applied; re-fixed 2026-08-18 in `docs/Authentication.md` (feat/followups-tickets) — the doc sub-claim did not survive: the Storage section was later rewritten to delegate schema/connection-factory detail to `ZB.MOM.WW.Auth.ApiKeys` and the WAL/busy-timeout paragraph went with it, leaving no prose for the behavior anywhere. The behavior itself is intact (the library's `AuthSqliteConnectionFactory.OpenConnectionAsync` still issues both pragmas — confirmed against the 0.2.1 package), so the fix is prose-only: a paragraph restating the pooled-connection + `journal_mode=WAL` + 5 s `busy_timeout` contract and why it is load-bearing for the per-request last-used stamp and the per-denial audit append.
|
||||
|
||||
### Server-010
|
||||
|
||||
@@ -379,7 +379,7 @@ closed. New findings filed against this pass: Server-051..053.
|
||||
|
||||
**Recommendation:** Either hide/disable the Rotate action for revoked keys in `ApiKeysPage.razor`, require an explicit confirmation, or have `RotateAsync` preserve `revoked_utc` and add a separate explicit "reactivate" operation.
|
||||
|
||||
**Resolution:** Resolved 2026-05-18. Confirmed against source: `ApiKeysPage.razor` rendered the Rotate button unconditionally while Revoke was already gated on `key.RevokedUtc is null`. Took the lowest-risk recommended option — the dashboard now renders the Rotate (and Revoke) actions only for keys whose status is `Active`; a revoked key shows a "No actions" placeholder, so an operator cannot un-revoke a deliberately disabled key as a side effect of a rotation. `RotateAsync`'s store-level behavior is unchanged (rotation by `key_id` still clears `revoked_utc`, which the CLI relies on); `docs/Authentication.md` updated to document both the store behavior and the dashboard restriction. No automated test added: the change is pure conditional Razor rendering and the test project has no bUnit component-rendering harness; the underlying `DashboardApiKeyManagementService` is already unit-tested.
|
||||
**Resolution:** Resolved 2026-05-18. Confirmed against source: `ApiKeysPage.razor` rendered the Rotate button unconditionally while Revoke was already gated on `key.RevokedUtc is null`. Took the lowest-risk recommended option — the dashboard now renders the Rotate (and Revoke) actions only for keys whose status is `Active`; a revoked key shows a "No actions" placeholder, so an operator cannot un-revoke a deliberately disabled key as a side effect of a rotation. `RotateAsync`'s store-level behavior is unchanged (rotation by `key_id` still clears `revoked_utc`, which the CLI relies on); `docs/Authentication.md` updated to document both the store behavior and the dashboard restriction. No automated test added: the change is pure conditional Razor rendering and the test project has no bUnit component-rendering harness; the underlying `DashboardApiKeyManagementService` is already unit-tested. Re-verified present 2026-08-18 (feat/followups-tickets) — `ApiKeysPage.razor` still gates both Rotate and Revoke on `key.RevokedUtc is null`, with the "Rotate clears revoked_utc" comment intact.
|
||||
|
||||
### Server-011
|
||||
|
||||
@@ -394,7 +394,7 @@ closed. New findings filed against this pass: Server-051..053.
|
||||
|
||||
**Recommendation:** Add the needed `using` directives, drop the inline fully-qualified names, and convert to a primary constructor for consistency.
|
||||
|
||||
**Resolution:** Resolved 2026-05-18. Confirmed against source. Converted `WorkerAlarmRpcDispatcher` to a primary constructor with the standard `?? throw new ArgumentNullException(...)` field-initializer guard; dropped the inline `System.Guid` / `System.ArgumentNullException` qualifications (using implicit `using System;`); removed redundant `using System.Collections.Generic;` / `System.Threading` / `System.Threading.Tasks;` directives (covered by `ImplicitUsings`); replaced the two `if (... is null) throw new System.ArgumentNullException(...)` checks with `ArgumentNullException.ThrowIfNull`. The stale class-level `<summary>`/`<remarks>` ("Replaces NotWiredAlarmRpcDispatcher once ... wired in", "partially wired", "returns an Unimplemented diagnostic") were corrected to describe the actual GUID-vs-`Provider!Group.Tag` handling — overlapping with Server-014. No behavior change, so no new test; existing `WorkerAlarmRpcDispatcherTests` continue to pass and the project builds warning-free under `TreatWarningsAsErrors`.
|
||||
**Resolution:** Resolved 2026-05-18. Confirmed against source. Converted `WorkerAlarmRpcDispatcher` to a primary constructor with the standard `?? throw new ArgumentNullException(...)` field-initializer guard; dropped the inline `System.Guid` / `System.ArgumentNullException` qualifications (using implicit `using System;`); removed redundant `using System.Collections.Generic;` / `System.Threading` / `System.Threading.Tasks;` directives (covered by `ImplicitUsings`); replaced the two `if (... is null) throw new System.ArgumentNullException(...)` checks with `ArgumentNullException.ThrowIfNull`. The stale class-level `<summary>`/`<remarks>` ("Replaces NotWiredAlarmRpcDispatcher once ... wired in", "partially wired", "returns an Unimplemented diagnostic") were corrected to describe the actual GUID-vs-`Provider!Group.Tag` handling — overlapping with Server-014. No behavior change, so no new test; existing `WorkerAlarmRpcDispatcherTests` continue to pass and the project builds warning-free under `TreatWarningsAsErrors`. Re-verified 2026-08-18 (feat/followups-tickets): not regressed but no longer verifiable in place — `WorkerAlarmRpcDispatcher.cs` was **deleted** (not renamed) by the `dc9c0c9` project-rename/alarm-rework commit, which replaced the whole `IAlarmRpcDispatcher` trio with `GatewayAlarmMonitor` / `IGatewayAlarmService`. A repo-wide grep for the prose this finding removed ("not yet wired", "PR A.6/A.7", "worker-pending", "dev-rig") returns nothing in Server source, so the correction is moot rather than lost. Nothing to re-fix.
|
||||
|
||||
### Server-012
|
||||
|
||||
@@ -409,7 +409,7 @@ closed. New findings filed against this pass: Server-051..053.
|
||||
|
||||
**Recommendation:** Update CLAUDE.md's scope list and the `apikey` example to the canonical `*:*` scope strings, per CLAUDE.md's own rule that docs change with the code.
|
||||
|
||||
**Resolution:** Resolved 2026-05-18. Confirmed against `GatewayScopes` (`session:open`, `session:close`, `invoke:read`, `invoke:write`, `invoke:secure`, `events:read`, `metadata:read`, `admin`). CLAUDE.md's Build/Test/Run `apikey create` example and the Authentication-section scope list were both updated to the canonical `*:*` strings. (Note: since finding Server-004 was resolved, the old example would now be actively rejected at create time rather than silently creating an unusable key, making the doc correction load-bearing.) Pure documentation change; no test.
|
||||
**Resolution:** Resolved 2026-05-18. Confirmed against `GatewayScopes` (`session:open`, `session:close`, `invoke:read`, `invoke:write`, `invoke:secure`, `events:read`, `metadata:read`, `admin`). CLAUDE.md's Build/Test/Run `apikey create` example and the Authentication-section scope list were both updated to the canonical `*:*` strings. (Note: since finding Server-004 was resolved, the old example would now be actively rejected at create time rather than silently creating an unusable key, making the doc correction load-bearing.) Pure documentation change; no test. Regression re-fixed 2026-08-18 on feat/followup-closeout (a5f843c, f2a422b); scope lists re-verified present. (Both CLAUDE.md sites confirmed on feat/followups-tickets: the Build/Test/Run `apikey create-key` example carries the full canonical `session:open,session:close,invoke:read,invoke:write,invoke:secure,events:read,metadata:read,admin` set, and the Authentication-section list names the same eight strings.)
|
||||
|
||||
### Server-013
|
||||
|
||||
@@ -439,7 +439,7 @@ closed. New findings filed against this pass: Server-051..053.
|
||||
|
||||
**Recommendation:** Update the `AcknowledgeAlarm`/`QueryActiveAlarms` remarks to reflect that `WorkerAlarmRpcDispatcher` is the wired default, and describe its actual GUID-vs-`Provider!Group.Tag` handling.
|
||||
|
||||
**Resolution:** Resolved 2026-05-18. Confirmed against source: `SessionServiceCollectionExtensions` registers `WorkerAlarmRpcDispatcher` as `IAlarmRpcDispatcher`, so the "not yet wired" / "empty stream until PR A.2" / "PR A.6/A.7 follow-up" prose in the `AcknowledgeAlarm` and `QueryActiveAlarms` `<remarks>` and inline comments was stale. Rewrote both `<remarks>` blocks and both inline comments to state that DI binds the production `WorkerAlarmRpcDispatcher`, that it routes over the worker pipe IPC, and that `AcknowledgeAlarm` handles a canonical-GUID reference (→ `AcknowledgeAlarmCommand`) and a `Provider!Group.Tag` reference (→ `AcknowledgeAlarmByNameCommand`), with `NotWiredAlarmRpcDispatcher` being only the null fallback. The matching stale `WorkerAlarmRpcDispatcher` class-level XML doc was corrected as part of Server-011. Pure documentation/comment change; no test.
|
||||
**Resolution:** Resolved 2026-05-18. Confirmed against source: `SessionServiceCollectionExtensions` registers `WorkerAlarmRpcDispatcher` as `IAlarmRpcDispatcher`, so the "not yet wired" / "empty stream until PR A.2" / "PR A.6/A.7 follow-up" prose in the `AcknowledgeAlarm` and `QueryActiveAlarms` `<remarks>` and inline comments was stale. Rewrote both `<remarks>` blocks and both inline comments to state that DI binds the production `WorkerAlarmRpcDispatcher`, that it routes over the worker pipe IPC, and that `AcknowledgeAlarm` handles a canonical-GUID reference (→ `AcknowledgeAlarmCommand`) and a `Provider!Group.Tag` reference (→ `AcknowledgeAlarmByNameCommand`), with `NotWiredAlarmRpcDispatcher` being only the null fallback. The matching stale `WorkerAlarmRpcDispatcher` class-level XML doc was corrected as part of Server-011. Pure documentation/comment change; no test. Re-verified 2026-08-18 (feat/followups-tickets): not regressed but superseded — `MxAccessGatewayService.AcknowledgeAlarm` / `QueryActiveAlarms` still exist but were rewritten onto `IGatewayAlarmService` (the `IAlarmRpcDispatcher` seam is gone), and both now carry a bare `<inheritdoc />` rather than the rewritten `<remarks>`. The stale prose this finding removed is absent repo-wide, so there is nothing to re-fix; the rewritten remarks describing a now-deleted dispatcher would be wrong if restored.
|
||||
|
||||
### Server-015
|
||||
|
||||
@@ -454,7 +454,7 @@ closed. New findings filed against this pass: Server-051..053.
|
||||
|
||||
**Recommendation:** Make `CloseAsync` mutate `_state` through the existing `TransitionTo(...)` helper (or acquire `_syncRoot` around the reads/writes) so all `_state` access uses the same lock. Either extend `TransitionTo` to accept the `Closing` and `Closed` transitions (it already handles `Faulted`/`Closed` precedence) or refactor `CloseAsync` to call a private `TrySetClosing()` / `MarkClosed()` that locks `_syncRoot`. Add a regression test that forces a `TransitionTo(Ready)` after `CloseAsync` has set `Closing` and asserts the session does not flip back to `Ready`.
|
||||
|
||||
**Resolution:** 2026-05-20 — Unified the close path on `_syncRoot`. `GatewaySession.CloseAsync` (`src/MxGateway.Server/Sessions/GatewaySession.cs`) now mutates `_state` only through two private `_syncRoot`-locked helpers — `TryBeginClose` (writes `Closing`, returns the prior `_closeStarted`) and `MarkClosed` (writes `Closed`) — so every `_state` read/write in the session uses the same lock; `_closeLock` keeps its role of serializing concurrent close attempts. `TransitionTo` was tightened to refuse a transition out of `Closing` to anything other than `Closed`/`Faulted` so a late lifecycle callback cannot walk a closing session back to `Ready`. `docs/Sessions.md` updated to describe the unified lock discipline and the extended terminal precedence. Regression tests in `src/MxGateway.Tests/Gateway/Sessions/GatewaySessionTests.cs`: `TransitionTo_AfterCloseStarted_DoesNotOverwriteClosing` (the named scenario — `BlockingShutdownWorkerClient` parks the close inside `worker.ShutdownAsync` so the test can call `TransitionTo(Ready)` between the `Closing` and `Closed` writes and assert the state stays `Closing`) and `MarkFaulted_AfterCloseCompletes_DoesNotResurrectSession`.
|
||||
**Resolution:** 2026-05-20 — Unified the close path on `_syncRoot`. `GatewaySession.CloseAsync` (`src/MxGateway.Server/Sessions/GatewaySession.cs`) now mutates `_state` only through two private `_syncRoot`-locked helpers — `TryBeginClose` (writes `Closing`, returns the prior `_closeStarted`) and `MarkClosed` (writes `Closed`) — so every `_state` read/write in the session uses the same lock; `_closeLock` keeps its role of serializing concurrent close attempts. `TransitionTo` was tightened to refuse a transition out of `Closing` to anything other than `Closed`/`Faulted` so a late lifecycle callback cannot walk a closing session back to `Ready`. `docs/Sessions.md` updated to describe the unified lock discipline and the extended terminal precedence. Re-verified present 2026-08-18 (feat/followups-tickets) — doc sub-claim only; `docs/Sessions.md` still states that both close-related writes go through `_syncRoot` via `TryBeginClose`/`MarkClosed` and that `_closeLock` only serializes concurrent close attempts. Regression tests in `src/MxGateway.Tests/Gateway/Sessions/GatewaySessionTests.cs`: `TransitionTo_AfterCloseStarted_DoesNotOverwriteClosing` (the named scenario — `BlockingShutdownWorkerClient` parks the close inside `worker.ShutdownAsync` so the test can call `TransitionTo(Ready)` between the `Closing` and `Closed` writes and assert the state stays `Closing`) and `MarkFaulted_AfterCloseCompletes_DoesNotResurrectSession`.
|
||||
|
||||
### Server-016
|
||||
|
||||
@@ -484,7 +484,7 @@ closed. New findings filed against this pass: Server-051..053.
|
||||
|
||||
**Recommendation:** Add explicit arms to `ResolveRequiredScope`: map `AcknowledgeAlarmRequest` to `GatewayScopes.InvokeWrite` (parity with other write actions; ack changes alarm state) and `QueryActiveAlarmsRequest` to `GatewayScopes.MetadataRead` or `GatewayScopes.InvokeRead`. Update `docs/Authorization.md` to list both. Extend `GatewayGrpcScopeResolverTests` with the new mappings and an assertion that every request type defined by `mxaccess_gateway.proto` is named in the resolver (the test can enumerate the assembly's request types so a future RPC cannot quietly add itself only via the admin fallback).
|
||||
|
||||
**Resolution:** 2026-05-20 — Added explicit `AcknowledgeAlarmRequest => GatewayScopes.InvokeWrite` and `QueryActiveAlarmsRequest => GatewayScopes.EventsRead` arms to `GatewayGrpcScopeResolver.ResolveRequiredScope` (`src/MxGateway.Server/Security/Authorization/GatewayGrpcScopeResolver.cs:21-22`). `InvokeWrite` matches the existing `MxCommandKind.Write*` mapping because ack mutates alarm state; `EventsRead` matches `StreamEventsRequest` and `MxCommandKind.DrainEvents` because querying active alarms reads the same alarm/event surface. Extended `GatewayGrpcScopeResolverTests` with two new `InlineData` rows covering both request types (`src/MxGateway.Tests/Security/Authorization/GatewayGrpcScopeResolverTests.cs:16-17`) and added four interceptor-level cases in `GatewayGrpcAuthorizationInterceptorTests` (`UnaryServerHandler_AcknowledgeAlarmMissingScope_ReturnsPermissionDenied`, `UnaryServerHandler_AcknowledgeAlarmWithScope_RunsHandler`, `ServerStreamingServerHandler_QueryActiveAlarmsMissingScope_ReturnsPermissionDenied`, `ServerStreamingServerHandler_QueryActiveAlarmsWithScope_RunsHandler`) proving each new RPC denies callers lacking the chosen scope and runs the handler when the scope is held. Updated `docs/Authorization.md` (resolver snippet and Scope Catalog table) to list both RPCs against their scopes. `dotnet test ... --filter FullyQualifiedName~GatewayGrpcAuthorizationInterceptorTests` → 14 passed, 0 failed; resolver tests 28 passed, 0 failed.
|
||||
**Resolution:** 2026-05-20 — Added explicit `AcknowledgeAlarmRequest => GatewayScopes.InvokeWrite` and `QueryActiveAlarmsRequest => GatewayScopes.EventsRead` arms to `GatewayGrpcScopeResolver.ResolveRequiredScope` (`src/MxGateway.Server/Security/Authorization/GatewayGrpcScopeResolver.cs:21-22`). `InvokeWrite` matches the existing `MxCommandKind.Write*` mapping because ack mutates alarm state; `EventsRead` matches `StreamEventsRequest` and `MxCommandKind.DrainEvents` because querying active alarms reads the same alarm/event surface. Extended `GatewayGrpcScopeResolverTests` with two new `InlineData` rows covering both request types (`src/MxGateway.Tests/Security/Authorization/GatewayGrpcScopeResolverTests.cs:16-17`) and added four interceptor-level cases in `GatewayGrpcAuthorizationInterceptorTests` (`UnaryServerHandler_AcknowledgeAlarmMissingScope_ReturnsPermissionDenied`, `UnaryServerHandler_AcknowledgeAlarmWithScope_RunsHandler`, `ServerStreamingServerHandler_QueryActiveAlarmsMissingScope_ReturnsPermissionDenied`, `ServerStreamingServerHandler_QueryActiveAlarmsWithScope_RunsHandler`) proving each new RPC denies callers lacking the chosen scope and runs the handler when the scope is held. Updated `docs/Authorization.md` (resolver snippet and Scope Catalog table) to list both RPCs against their scopes. Re-verified present 2026-08-18 (feat/followups-tickets) — doc sub-claim only; the snippet still shows `AcknowledgeAlarmRequest => GatewayScopes.InvokeWrite` / `QueryActiveAlarmsRequest => GatewayScopes.EventsRead` and both RPCs still appear in the Scope Catalog rows. `dotnet test ... --filter FullyQualifiedName~GatewayGrpcAuthorizationInterceptorTests` → 14 passed, 0 failed; resolver tests 28 passed, 0 failed.
|
||||
|
||||
### Server-018
|
||||
|
||||
@@ -559,7 +559,7 @@ closed. New findings filed against this pass: Server-051..053.
|
||||
|
||||
**Recommendation:** Rewrite the `IAlarmRpcDispatcher` `<remarks>` block to match the language now used on `WorkerAlarmRpcDispatcher` and on the gRPC service: DI binds `WorkerAlarmRpcDispatcher` by default; `NotWiredAlarmRpcDispatcher` is only the null fallback for tests/DI omission. Drop the "PR A.6 / A.7" prefix from the `<summary>` — the interface is now the public alarm-RPC seam.
|
||||
|
||||
**Resolution:** 2026-05-20 — Rewrote `IAlarmRpcDispatcher`'s `<summary>` and `<remarks>` (`src/MxGateway.Server/Sessions/IAlarmRpcDispatcher.cs`) to match the language now used on `WorkerAlarmRpcDispatcher` and on `MxAccessGatewayService.AcknowledgeAlarm` / `QueryActiveAlarms`: dropped the stale "PR A.6 / A.7" prefix from the summary, and replaced the "this PR ships a not-yet-wired default that returns a clear worker-pending diagnostic" clause with the correct statement that DI binds the production `WorkerAlarmRpcDispatcher` by default and `NotWiredAlarmRpcDispatcher` is only the null fallback for DI omission / standalone tests. Pure documentation change; no test.
|
||||
**Resolution:** 2026-05-20 — Rewrote `IAlarmRpcDispatcher`'s `<summary>` and `<remarks>` (`src/MxGateway.Server/Sessions/IAlarmRpcDispatcher.cs`) to match the language now used on `WorkerAlarmRpcDispatcher` and on `MxAccessGatewayService.AcknowledgeAlarm` / `QueryActiveAlarms`: dropped the stale "PR A.6 / A.7" prefix from the summary, and replaced the "this PR ships a not-yet-wired default that returns a clear worker-pending diagnostic" clause with the correct statement that DI binds the production `WorkerAlarmRpcDispatcher` by default and `NotWiredAlarmRpcDispatcher` is only the null fallback for DI omission / standalone tests. Pure documentation change; no test. Re-verified 2026-08-18 (feat/followups-tickets): not regressed but no longer verifiable in place — `IAlarmRpcDispatcher.cs` was deleted with the rest of the dispatcher trio in `dc9c0c9`. No stale "PR A.6/A.7" or "not-yet-wired" prose survives anywhere in Server source. Nothing to re-fix.
|
||||
|
||||
### Server-023
|
||||
|
||||
@@ -574,7 +574,7 @@ closed. New findings filed against this pass: Server-051..053.
|
||||
|
||||
**Recommendation:** Replace the `<summary>` and `<remarks>` on `NotWiredAlarmRpcDispatcher` with text that matches the language now used on the interface and `WorkerAlarmRpcDispatcher` — "null fallback `IAlarmRpcDispatcher` used when no dispatcher is registered (DI omission / standalone tests); production wires `WorkerAlarmRpcDispatcher`." Either drop the `AcknowledgeAsync` diagnostic string's dev-rig framing entirely or shorten it to "alarm dispatcher is not registered." `#pragma warning disable CS1998` on `QueryActiveAlarmsAsync` is correct here (empty stream is intentional for the null fallback) and should stay.
|
||||
|
||||
**Resolution:** 2026-05-20 — Rewrote `NotWiredAlarmRpcDispatcher` summary/remarks as the null-fallback dispatcher and shortened the `AcknowledgeAsync` diagnostic to "Alarm dispatcher is not registered."; updated the two tests that asserted the old "worker"-prefixed diagnostic.
|
||||
**Resolution:** 2026-05-20 — Rewrote `NotWiredAlarmRpcDispatcher` summary/remarks as the null-fallback dispatcher and shortened the `AcknowledgeAsync` diagnostic to "Alarm dispatcher is not registered."; updated the two tests that asserted the old "worker"-prefixed diagnostic. Re-verified 2026-08-18 (feat/followups-tickets): not regressed but no longer verifiable in place — `NotWiredAlarmRpcDispatcher.cs` and its tests were deleted with the dispatcher trio in `dc9c0c9`. Nothing to re-fix.
|
||||
|
||||
### Server-024
|
||||
|
||||
@@ -637,7 +637,7 @@ closed. New findings filed against this pass: Server-051..053.
|
||||
|
||||
**Recommendation:** Update the `ResolveCommandScope` snippet to include the four bulk-write arms. Update the Constraint Enforcement prose to enumerate the bulk read/write commands that are actually filtered, and reference the per-entry index-ordered merge that `BulkConstraintPlan.MergeDeniedInto` performs. Adding `ReadBulk` to the `InvokeRead` row of the Scope Catalog would also be useful — the table currently lists `Register`/`AddItem`/`Advise` against `InvokeRead` but not `ReadBulk`.
|
||||
|
||||
**Resolution:** 2026-05-20 — Updated the `ResolveCommandScope` snippet in `docs/Authorization.md` to enumerate the four bulk-write arms (`WriteBulk`/`Write2Bulk` against `InvokeWrite`, `WriteSecuredBulk`/`WriteSecured2Bulk` against `InvokeSecure`); expanded the Constraint Enforcement prose to list `ReadBulk` and all four bulk-write commands and to call out `BulkConstraintPlan.MergeDeniedInto`'s index-ordered merge; added `ReadBulk` to the `InvokeRead` row of the Scope Catalog.
|
||||
**Resolution:** 2026-05-20 — Updated the `ResolveCommandScope` snippet in `docs/Authorization.md` to enumerate the four bulk-write arms (`WriteBulk`/`Write2Bulk` against `InvokeWrite`, `WriteSecuredBulk`/`WriteSecured2Bulk` against `InvokeSecure`); expanded the Constraint Enforcement prose to list `ReadBulk` and all four bulk-write commands and to call out `BulkConstraintPlan.MergeDeniedInto`'s index-ordered merge; added `ReadBulk` to the `InvokeRead` row of the Scope Catalog. Re-verified present 2026-08-18 (feat/followups-tickets) — all three corrections still stand in `docs/Authorization.md`: the snippet carries the four bulk-write arms, the Constraint Enforcement prose names `ReadBulk` plus the four bulk-write commands and `BulkConstraintPlan.MergeDeniedInto`, and `ReadBulk` is in the `InvokeRead` catalog row.
|
||||
|
||||
### Server-028
|
||||
|
||||
@@ -670,7 +670,7 @@ closed. New findings filed against this pass: Server-051..053.
|
||||
|
||||
**Recommendation:** Either (a) extend the advertised list with `bulk-read-command` and `bulk-write-commands` (`WriteBulk` / `Write2Bulk` / `WriteSecuredBulk` / `WriteSecured2Bulk` collectively), or (b) document in `gateway.md` and `docs/Contracts.md` that `Capabilities` is informational only and not the contract version. Option (a) is the simplest forward-compatible fix and keeps the capability token shape clients are already familiar with.
|
||||
|
||||
**Resolution:** 2026-05-20 — Extended the `OpenSession` capabilities list with `bulk-read-commands` and `bulk-write-commands` alongside the existing `bulk-subscribe-commands` token, so clients that gate on capability strings have an explicit signal for the bulk-read and bulk-write families.
|
||||
**Resolution:** 2026-05-20 — Extended the `OpenSession` capabilities list with `bulk-read-commands` and `bulk-write-commands` alongside the existing `bulk-subscribe-commands` token, so clients that gate on capability strings have an explicit signal for the bulk-read and bulk-write families. Re-verified present 2026-08-18 (feat/followups-tickets) — `MxAccessGatewayService.OpenSession` still adds all three tokens.
|
||||
|
||||
### Server-030
|
||||
|
||||
@@ -839,7 +839,7 @@ Add a regression test that advises N items without an active `StreamEvents` cons
|
||||
|
||||
**Recommendation:** Before the EventsHub is exercised by Admin-only sessions or session-scoped Viewer roles, gate `SubscribeSession` on a session-access check — either via a per-session role check in the hub method itself, or by storing a per-user allowed-session-id set in the connection's `Context.Items` at connect time and rejecting subscribes outside that set. The current dashboard surfaces only a per-page Session Details view that the page can prove it's authorized for, but as soon as a Viewer role exists the gap matters.
|
||||
|
||||
**Resolution:** 2026-05-24 — Documented the v1 acceptance per the prompt's "practical fix for v1" direction. Added a detailed `<remarks>` block to `EventsHub.SubscribeSession` (`src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHub.cs`) stating that (a) in v1 the hub-level `HubClientsPolicy` only requires one of the dashboard roles (Admin or Viewer) and both may subscribe to any session id, (b) this is acceptable today because the dashboard's per-session views show non-secret session metadata any authenticated user can already see and value logging is gated by the same redaction policy, and (c) the per-session ACL that gates the gRPC `StreamEvents` RPC is intentionally not yet mirrored here. Added an explicit `TODO(per-session-acl)` describing the future enforcement seam — once a role/scope is introduced that scopes a Viewer to a specific session or tenant, add a session-access check at this method (inline on `Context.User` claims/`Context.Items`, or via a dedicated authorization policy applied to the hub method). No code-behavior change in this pass; the per-session ACL data model design is out of scope for the resolution window. No new regression test (the change is documentation-only).
|
||||
**Resolution:** 2026-05-24 — Documented the v1 acceptance per the prompt's "practical fix for v1" direction. Added a detailed `<remarks>` block to `EventsHub.SubscribeSession` (`src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHub.cs`) stating that (a) in v1 the hub-level `HubClientsPolicy` only requires one of the dashboard roles (Admin or Viewer) and both may subscribe to any session id, (b) this is acceptable today because the dashboard's per-session views show non-secret session metadata any authenticated user can already see and value logging is gated by the same redaction policy, and (c) the per-session ACL that gates the gRPC `StreamEvents` RPC is intentionally not yet mirrored here. Added an explicit `TODO(per-session-acl)` describing the future enforcement seam — once a role/scope is introduced that scopes a Viewer to a specific session or tenant, add a session-access check at this method (inline on `Context.User` claims/`Context.Items`, or via a dedicated authorization policy applied to the hub method). No code-behavior change in this pass; the per-session ACL data model design is out of scope for the resolution window. No new regression test (the change is documentation-only). Re-verified 2026-08-18 (feat/followups-tickets): the documented v1 gap was subsequently **closed**, not regressed — `EventsHub` now takes `IDashboardSessionAcl` and `SubscribeSession` throws `HubException("Not authorized for this session.")` when `CanViewSession` denies (SEC-25 / TST-15). The `TODO(per-session-acl)` is correctly gone and the `<remarks>` block now describes the shipped tag-scoped gate (admin bypass first, tag intersection, `Dashboard:UntaggedSessionVisibility`, phantom-id denial for non-Admins). Nothing to re-fix; the finding's own recommendation is what shipped.
|
||||
|
||||
### Server-039
|
||||
|
||||
@@ -869,7 +869,7 @@ Add a regression test that advises N items without an active `StreamEvents` cons
|
||||
|
||||
**Recommendation:** Add a one-line comment above the loop explaining the precedence: full DN/CN literal first, leading-RDN fallback second. Mention the case-insensitive map comparer (`OrdinalIgnoreCase`) so the next reader doesn't ask why `"GwAdmin"` matches `"gwadmin"`.
|
||||
|
||||
**Resolution:** 2026-05-24 — Added a precedence comment block above the lookup in `MapGroupsToRoles` (`src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardAuthenticator.cs:156-163`) explaining that the full literal group string is tried first and the leading-RDN value (e.g. `GwAdmin` extracted from `ou=GwAdmin,ou=groups,...`) is the fallback, and back-referencing `DashboardOptions.GroupToRole` as the source of the `OrdinalIgnoreCase` comparer so a maintainer sees why `"GwAdmin"` matches `"gwadmin"`. No code change — existing `DashboardAuthenticatorTests.MapGroupsToRoles_ResolvesByShortNameAndDistinguishedName` already pins both the full-match and RDN-fallback paths and the case-insensitive lookup; pure documentation-only resolution, no new test.
|
||||
**Resolution:** 2026-05-24 — Added a precedence comment block above the lookup in `MapGroupsToRoles` (`src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardAuthenticator.cs:156-163`) explaining that the full literal group string is tried first and the leading-RDN value (e.g. `GwAdmin` extracted from `ou=GwAdmin,ou=groups,...`) is the fallback, and back-referencing `DashboardOptions.GroupToRole` as the source of the `OrdinalIgnoreCase` comparer so a maintainer sees why `"GwAdmin"` matches `"gwadmin"`. No code change — existing `DashboardAuthenticatorTests.MapGroupsToRoles_ResolvesByShortNameAndDistinguishedName` already pins both the full-match and RDN-fallback paths and the case-insensitive lookup; pure documentation-only resolution, no new test. Regressed or never applied; re-fixed 2026-08-18 in `src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardGroupRoleMapping.cs` (feat/followups-tickets). Chain of events: the lookup moved out of `DashboardAuthenticator` into the shared `DashboardGroupRoleMapping` helper in `792e3f9` (the `IGroupRoleMapper<string>` seam) **with the comment intact**, then `fca978d` ("docs(src): add missing XML docs and strip tracking-ID comments") deleted the entire block rather than just the `(Server-040)` marker inside it. That also took out a second, substantive paragraph added later — the note that the shared `ZB.MOM.WW.Auth.Ldap` provider already strips groups to short RDN names, making the RDN fallback a no-op on the live login path and making a full-DN `GroupToRole` **key** unsupported. Both paragraphs are restored, minus the tracking IDs (that part of the sweep's intent is respected).
|
||||
|
||||
### Server-041
|
||||
|
||||
@@ -914,7 +914,7 @@ Add a regression test that advises N items without an active `StreamEvents` cons
|
||||
|
||||
**Recommendation:** Add a `<remarks>` block to `HubTokenService` noting "Registered as a singleton in `AddGatewayDashboard`; the underlying `ITimeLimitedDataProtector` is thread-safe and shared across hub-token issuance and validation." Optionally add a comment near the DI registration explaining the lifetime contract.
|
||||
|
||||
**Resolution:** 2026-05-24 — Added a `<remarks>` block to `HubTokenService` (`src/ZB.MOM.WW.MxGateway.Server/Dashboard/HubTokenService.cs`) documenting that the service is registered as a singleton in `DashboardServiceCollectionExtensions.AddGatewayDashboard` and is shared by two consumer scopes — `DashboardHubConnectionFactory` (scoped, per-circuit; calls `Issue` from the cookie-authenticated dashboard) and `HubTokenAuthenticationHandler` (transient, per-request; calls `Validate` from the SignalR negotiate / connection path). Notes that the underlying `ITimeLimitedDataProtector` is thread-safe so concurrent mint/validate from any number of callers is safe, and explicitly asks future maintainers to preserve the singleton lifetime to keep the protector instance stable. Pure documentation change; no test.
|
||||
**Resolution:** 2026-05-24 — Added a `<remarks>` block to `HubTokenService` (`src/ZB.MOM.WW.MxGateway.Server/Dashboard/HubTokenService.cs`) documenting that the service is registered as a singleton in `DashboardServiceCollectionExtensions.AddGatewayDashboard` and is shared by two consumer scopes — `DashboardHubConnectionFactory` (scoped, per-circuit; calls `Issue` from the cookie-authenticated dashboard) and `HubTokenAuthenticationHandler` (transient, per-request; calls `Validate` from the SignalR negotiate / connection path). Notes that the underlying `ITimeLimitedDataProtector` is thread-safe so concurrent mint/validate from any number of callers is safe, and explicitly asks future maintainers to preserve the singleton lifetime to keep the protector instance stable. Pure documentation change; no test. Re-verified present 2026-08-18 (feat/followups-tickets) — the `<remarks>` block still names the `AddGatewayDashboard` singleton registration, the two consumer scopes, the thread-safe `ITimeLimitedDataProtector`, and the preserve-the-lifetime request. One consumer was renamed since (the issuing side is now the `/hubs/token` endpoint rather than `DashboardHubConnectionFactory`) and the doc tracked that change correctly.
|
||||
|
||||
## Re-review 2026-05-24 (commit 42b0037)
|
||||
|
||||
@@ -983,7 +983,7 @@ The user-visible difference: rotating/revoking/deleting a key vs closing/killing
|
||||
|
||||
**Recommendation:** Align `ApiKeysPage.ConfirmPendingAsync` with the sessions pages: hold `PendingAction`, set `IsBusy = true`, run the action, then clear `PendingAction` in the `finally`. The current ApiKeysPage shape was inherited from before the dialog existed (when the confirmation was a `confirm()` JS call); the dialog component change can flatten the difference now. As a smaller alternative, document the divergence on the component's XML doc — but the shared component should ideally be used consistently.
|
||||
|
||||
**Resolution:** 2026-05-24 — Took the recommended alignment. `ApiKeysPage.ConfirmPendingAsync` (`src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Pages/ApiKeysPage.razor`) now holds `PendingAction` for the duration of the awaited action (so the shared `ConfirmDialog` renders its `IsBusy` in-flight state on the dialog itself, matching the sessions pages) and clears it in `finally` regardless of outcome. The action is captured up front so a clear in `finally` works even when the action throws. `RunManagementActionAsync` continues to drive `IsBusy = true` inside its own `try/finally`, so the dialog now correctly disables Confirm/Cancel while the awaited service call runs. Pure UX-consistency change; no new automated test (no bUnit harness in the test project — same precedent as Server-010).
|
||||
**Resolution:** 2026-05-24 — Took the recommended alignment. `ApiKeysPage.ConfirmPendingAsync` (`src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Pages/ApiKeysPage.razor`) now holds `PendingAction` for the duration of the awaited action (so the shared `ConfirmDialog` renders its `IsBusy` in-flight state on the dialog itself, matching the sessions pages) and clears it in `finally` regardless of outcome. The action is captured up front so a clear in `finally` works even when the action throws. `RunManagementActionAsync` continues to drive `IsBusy = true` inside its own `try/finally`, so the dialog now correctly disables Confirm/Cancel while the awaited service call runs. Pure UX-consistency change; no new automated test (no bUnit harness in the test project — same precedent as Server-010). Re-verified present 2026-08-18 (feat/followups-tickets) — `ApiKeysPage.ConfirmPendingAsync` still captures the action up front, awaits inside `try`, and clears `PendingAction` in `finally`; the explanatory comment survives.
|
||||
|
||||
### Server-048
|
||||
|
||||
@@ -1017,7 +1017,7 @@ The user-visible difference: rotating/revoking/deleting a key vs closing/killing
|
||||
|
||||
**Recommendation:** Add `<summary>` blocks to `IDashboardSessionAdminService.CanManage` (states the Admin-role gate), `CloseSessionAsync` and `KillWorkerAsync` (state that missing sessions return `DashboardSessionAdminResult.Fail(...)` rather than throwing, and that the audit log captures actor + remote IP). Add `<param>` and `<returns>` for the request/response shape. The same sweep can pick up the longstanding gap on `IDashboardApiKeyManagementService` if the team wants — but the new file is the load-bearing one.
|
||||
|
||||
**Resolution:** 2026-05-24 — Added `<summary>` + `<remarks>` blocks to every member of `IDashboardSessionAdminService` (`src/ZB.MOM.WW.MxGateway.Server/Dashboard/IDashboardSessionAdminService.cs`): an interface-level `<remarks>` describing the Admin-role gate, audit log shape, and `DashboardSessionAdminResult.Fail` semantics; per-member docs on `CanManage`, `CloseSessionAsync`, and `KillWorkerAsync` calling out the missing-session-returns-Fail contract and the `dashboard-admin-kill` reason constant that reaches the worker-kill audit log and `mxgateway.workers.killed` counter tag. `DashboardSessionAdminService` (`src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSessionAdminService.cs`) picked up a class-level `<summary>` + `<remarks>` describing the per-page audit-log seam, plus `<inheritdoc />` on each public method. Pure documentation change; no test (the behavioral contracts the docs describe are already exercised by the existing `DashboardSessionAdminServiceTests` cases).
|
||||
**Resolution:** 2026-05-24 — Added `<summary>` + `<remarks>` blocks to every member of `IDashboardSessionAdminService` (`src/ZB.MOM.WW.MxGateway.Server/Dashboard/IDashboardSessionAdminService.cs`): an interface-level `<remarks>` describing the Admin-role gate, audit log shape, and `DashboardSessionAdminResult.Fail` semantics; per-member docs on `CanManage`, `CloseSessionAsync`, and `KillWorkerAsync` calling out the missing-session-returns-Fail contract and the `dashboard-admin-kill` reason constant that reaches the worker-kill audit log and `mxgateway.workers.killed` counter tag. `DashboardSessionAdminService` (`src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSessionAdminService.cs`) picked up a class-level `<summary>` + `<remarks>` describing the per-page audit-log seam, plus `<inheritdoc />` on each public method. Pure documentation change; no test (the behavioral contracts the docs describe are already exercised by the existing `DashboardSessionAdminServiceTests` cases). Re-verified present 2026-08-18 (feat/followups-tickets) — the interface-level `<remarks>` and all three per-member blocks are intact, including the `dashboard-admin-kill` reason constant and the missing-session-returns-`Fail` contract.
|
||||
|
||||
### Server-050
|
||||
|
||||
@@ -1078,7 +1078,7 @@ catch (Exception ex) { _logger.LogWarning(ex, "...continuing with configuration-
|
||||
|
||||
**Recommendation:** For (1), align the `IAlarmWatchListResolver` doc with whatever Server-051 settles on. For (2), either restrict the exclude to GR-discovered rows (apply `RemoveAll` before appending the `IncludeAttributes` entries) or update the option XML doc and `GatewayConfiguration.md` to say excludes are applied to the merged GR-plus-include list and therefore also suppress matching explicit includes.
|
||||
|
||||
**Resolution:** Resolved 2026-06-15. (1) No longer over-promises: the Server-051 fix makes the implementation propagate `OperationCanceledException`, so the `IAlarmWatchListResolver.ResolveAsync` `<returns>` doc is now accurate and was left unchanged. (2) Kept the "excludes win" code behaviour (excludes applied to the merged GR-plus-include list) and corrected the prose to match: `AlarmDiscoveryOptions.ExcludeAttributes` XML doc and `docs/GatewayConfiguration.md:247` now state the exclude runs after the GR rows and explicit `IncludeAttributes` are combined, so an exclude matching an explicit include suppresses it too. The "excludes win" precedence is pinned by `AlarmWatchListResolverTests.ResolveAsync_ExcludeAlsoSuppressesMatchingExplicitInclude`.
|
||||
**Resolution:** Resolved 2026-06-15. (1) No longer over-promises: the Server-051 fix makes the implementation propagate `OperationCanceledException`, so the `IAlarmWatchListResolver.ResolveAsync` `<returns>` doc is now accurate and was left unchanged. (2) Kept the "excludes win" code behaviour (excludes applied to the merged GR-plus-include list) and corrected the prose to match: `AlarmDiscoveryOptions.ExcludeAttributes` XML doc and `docs/GatewayConfiguration.md:247` now state the exclude runs after the GR rows and explicit `IncludeAttributes` are combined, so an exclude matching an explicit include suppresses it too. The "excludes win" precedence is pinned by `AlarmWatchListResolverTests.ResolveAsync_ExcludeAlsoSuppressesMatchingExplicitInclude`. Re-verified present 2026-08-18 (feat/followups-tickets) — both doc corrections stand: `IAlarmWatchListResolver.ResolveAsync`'s `<returns>` still documents cancellation propagation (and the implementation still honors it), and the "excludes win over explicit includes" wording is present on `AlarmDiscoveryOptions.ExcludeAttributes` (now in `Configuration/AlarmFallbackOptions.cs`) and in the `docs/GatewayConfiguration.md` option table.
|
||||
|
||||
### Server-053
|
||||
|
||||
@@ -1113,7 +1113,7 @@ Additionally, `GatewayAlarmMonitor.ApplyProviderModeChangeAsync` increments the
|
||||
|
||||
**Recommendation:** Update both `DesignDecisions.md` sections and the revisit list to describe the shipped behavior (gated by `AllowMultipleEventSubscribers`, `DetachGraceSeconds`, replay options), and amend the CLAUDE.md convention bullet.
|
||||
|
||||
**Resolution:** 2026-06-16: updated `docs/DesignDecisions.md` (Session Reconnect section rewritten to describe the shipped detach-grace + replay-on-reconnect behavior with config references; Event Subscribers section rewritten to describe the config-gated multi-subscriber fan-out, mode-dependent `FailFast` semantics, and internal vs external subscriber distinction; Later Revisit Items list removes the two shipped items and records them as shipped with config cross-references) and the `CLAUDE.md` conventions bullet to describe the shipped config-gated multi-subscriber + reconnect-replay behavior while preserving the one-worker-per-session invariant.
|
||||
**Resolution:** 2026-06-16: updated `docs/DesignDecisions.md` (Session Reconnect section rewritten to describe the shipped detach-grace + replay-on-reconnect behavior with config references; Event Subscribers section rewritten to describe the config-gated multi-subscriber fan-out, mode-dependent `FailFast` semantics, and internal vs external subscriber distinction; Later Revisit Items list removes the two shipped items and records them as shipped with config cross-references) and the `CLAUDE.md` conventions bullet to describe the shipped config-gated multi-subscriber + reconnect-replay behavior while preserving the one-worker-per-session invariant. Re-verified present 2026-08-18 (feat/followups-tickets) — `docs/DesignDecisions.md` still records the "no reconnectable sessions" constraint as superseded and documents `DetachGraceSeconds` / `AllowMultipleEventSubscribers` / `ReplayBufferCapacity`, the Later Revisit Items list still cross-references them as shipped, and the `CLAUDE.md` bullet still carries the config-gated wording.
|
||||
|
||||
### Server-055
|
||||
|
||||
@@ -1158,7 +1158,7 @@ Additionally, `GatewayAlarmMonitor.ApplyProviderModeChangeAsync` increments the
|
||||
|
||||
**Recommendation:** Either (a) extend `NormalizeOutboundCommand` and the `MapCommand` tracking path to normalize each `AddItemBulk.TagAddresses` entry (and `AddBufferedItem.ItemDefinition`) the same `IsArray`-gated way, keeping the constraint check, the worker bind, and the stored `SessionItemRegistration.TagAddress` consistent; or (b) if bulk-add normalization is intentionally out of scope for this feature, state that explicitly in `gateway.md` and the client READMEs (alongside the existing `ReadBulk` carve-out) so clients know bulk-added array handles must carry the `[]` suffix themselves to be writable.
|
||||
|
||||
**Resolution:** 2026-06-18 — Took option (a). Root cause confirmed: `NormalizeOutboundCommand` had no `AddItemBulk`/`AddBufferedItem` case, so the worker bound bare (non-write-capable) array handles for those paths while single-add was already fixed. Added `AddItemBulk` (normalizes each `TagAddresses` entry in place) and `AddBufferedItem` (normalizes `ItemDefinition`) cases to `NormalizeOutboundCommand`; added the matching `AddBufferedItem` normalization to the `TrackCommandReply`/`MapCommand` tracking path (its registration keys off the command's `ItemDefinition`). `AddItemBulk` tracking needs no change — the worker echoes the already-suffixed address back in each `SubscribeResult.TagAddress`, which `TrackBulkItems` stores. Authz is unchanged and consistent: `FilterTagBulkAsync` checks the bare address through `ConstraintEnforcer.ResolveTarget`'s `[]` fallback, mirroring single-add. Updated `gateway.md` and all five client READMEs (dotnet/go/python/rust/java) so the add-family normalization no longer reads as AddItem-only; the `ReadBulk` carve-out stays. Regression tests: `GatewayArrayWriteWiringTests.AddItemBulk_BareArrayAddress_NormalizedOnWireAndInRegistration`, `.AddBufferedItem_BareArrayAddress_NormalizedOnWireAndInRegistration`.
|
||||
**Resolution:** 2026-06-18 — Took option (a). Root cause confirmed: `NormalizeOutboundCommand` had no `AddItemBulk`/`AddBufferedItem` case, so the worker bound bare (non-write-capable) array handles for those paths while single-add was already fixed. Added `AddItemBulk` (normalizes each `TagAddresses` entry in place) and `AddBufferedItem` (normalizes `ItemDefinition`) cases to `NormalizeOutboundCommand`; added the matching `AddBufferedItem` normalization to the `TrackCommandReply`/`MapCommand` tracking path (its registration keys off the command's `ItemDefinition`). `AddItemBulk` tracking needs no change — the worker echoes the already-suffixed address back in each `SubscribeResult.TagAddress`, which `TrackBulkItems` stores. Authz is unchanged and consistent: `FilterTagBulkAsync` checks the bare address through `ConstraintEnforcer.ResolveTarget`'s `[]` fallback, mirroring single-add. Updated `gateway.md` and all five client READMEs (dotnet/go/python/rust/java) so the add-family normalization no longer reads as AddItem-only; the `ReadBulk` carve-out stays. Regression tests: `GatewayArrayWriteWiringTests.AddItemBulk_BareArrayAddress_NormalizedOnWireAndInRegistration`, `.AddBufferedItem_BareArrayAddress_NormalizedOnWireAndInRegistration`. Re-verified present 2026-08-18 (feat/followups-tickets) — doc sub-claim only; `gateway.md` still describes normalization across the whole add family (`AddItem`/`AddItem2`/`AddItemBulk`/`AddBufferedItem`) and all five client READMEs still name the bulk and buffered members (Python in its snake_case form: `add_item_bulk` / `add_buffered_item`).
|
||||
|
||||
### Server-058
|
||||
|
||||
|
||||
@@ -835,7 +835,7 @@ carries `degraded = true`.
|
||||
### Watch-list discovery
|
||||
|
||||
`GatewayAlarmMonitor` resolves the subtag watch-list at subscribe time by
|
||||
calling `IAlarmWatchListResolver.GetAlarmAttributesAsync`. The resolver merges:
|
||||
calling `IAlarmWatchListResolver.ResolveAsync`. The resolver merges:
|
||||
|
||||
1. Galaxy Repository SQL (`GetAlarmAttributesAsync`) — objects that have alarm
|
||||
extensions in the configured area.
|
||||
@@ -847,8 +847,47 @@ The resolved list is a set of `AlarmSubtagTarget` messages sent to the worker
|
||||
inside `SubscribeAlarmsCommand.watch_list`. Each target carries the composed
|
||||
MXAccess item addresses for the `InAlarm`, `Acked`, `AckMsg`, and `Priority`
|
||||
subtags (confirmed AVEVA `AlarmExtension` field names, verified against the live
|
||||
ZB Galaxy `attribute_definition` rows). The gateway re-runs discovery on its
|
||||
reconcile cadence and pushes an updated watch-list when the model changes.
|
||||
ZB Galaxy `attribute_definition` rows).
|
||||
|
||||
The gateway refreshes this list on the existing alarm reconciliation cadence
|
||||
and immediately before publishing a failover-to-subtag status. It compares a
|
||||
canonical fingerprint of all six target fields and sends
|
||||
`UpdateAlarmWatchListCommand` only for a material change. The worker constructs,
|
||||
subscribes, and attaches the replacement standby on its STA before swapping it
|
||||
in; a setup failure leaves the previous standby and the current primary/subtag
|
||||
mode unchanged. The gateway advances its acknowledged fingerprint only after an
|
||||
OK reply, so a rejected update remains degraded and is retried.
|
||||
|
||||
`SubscribeAlarmsCommand.fallback_enabled` distinguishes an initially empty but
|
||||
discovery-enabled fallback from an intentionally alarm-manager-only
|
||||
subscription. In the former case the worker creates an empty standby that can be
|
||||
hot-swapped after discovery recovers. `ForceAlarmManager` leaves the field false
|
||||
and does not create a standby.
|
||||
|
||||
Because the list is fixed, the resolution is **graded** rather than merely
|
||||
logged (GWC-37). `ResolveAsync` returns an `AlarmWatchListResolution` carrying
|
||||
whether Galaxy Repository discovery was attempted and whether it failed, and the
|
||||
monitor folds the verdict into the alarm feed's `AlarmProviderStatus`:
|
||||
|
||||
- Zero targets while the effective mode can need them (`Auto` or `ForceSubtag`)
|
||||
sets `degraded = true` with reason `AlarmProviderReasons.SubtagWatchListEmpty`
|
||||
— the subtag provider covers no alarms.
|
||||
- Discovery enabled but failing adds
|
||||
`AlarmProviderReasons.SubtagDiscoveryFailed` and the failure message, so a
|
||||
Galaxy Repository outage at startup is a standing status rather than one log
|
||||
line emitted possibly weeks before the failover that needs the list.
|
||||
- If the worker rejects a material update, the monitor retains the previous
|
||||
acknowledged fingerprint and reports
|
||||
`AlarmProviderReasons.SubtagWatchListStale` with the rejection diagnostic.
|
||||
The next reconciliation or failover edge retries the same resolved targets.
|
||||
|
||||
`degraded` on the status is therefore the OR of "the subtag provider is the one
|
||||
serving" and "the subtag watch-list cannot cover anything". In `Auto` the second
|
||||
can be true while the mode is still `ALARMMGR`: the live feed is healthy and the
|
||||
standby behind it is empty. The dashboard renders that combination as the amber
|
||||
`Alarm Manager (no subtag standby)` badge rather than the failover label, so the
|
||||
badge never names the wrong live provider. `ForceAlarmManager` never builds a
|
||||
subtag consumer, so its watch-list is not graded at all.
|
||||
|
||||
Each target's canonical `AlarmFullReference` is composed as
|
||||
`Galaxy!{area}.{reference}` (literal `Galaxy` provider). The `{area}` is the
|
||||
|
||||
+153
-4
@@ -24,13 +24,13 @@ the blocker rather than rediscovering it.
|
||||
|---|---|
|
||||
| GUID stable across polls and `ALM → RTN` | Answered — yes (2026-05-01 capture in `AlarmClientDiscovery.md`, reconfirmed 2026-08-18) |
|
||||
| GUID stable across clear-then-re-raise | Answered 2026-08-18 — **no**, a re-raise mints a new GUID |
|
||||
| GUID stable across `UNACK → ACK` | **Open** — the rig cannot be driven into an acknowledged state at all |
|
||||
| GUID stable across `UNACK → ACK` | **Open** — the rig cannot be driven into an acknowledged state; 2026-08-18 (third attempt) identifies the reason as the test attribute's `MxSecurityOperate` classification, which no non-interactive ack surface on the rig can satisfy |
|
||||
| `COUNT` = total active vs records-in-reply under a capped fetch | Answered 2026-08-18 — **records in the reply** |
|
||||
|
||||
The 2026-08-17 run below is kept because it is the record of the wrong-verb blocker. The
|
||||
2026-08-18 run cleared that blocker with `AuthenticateUser` + `WriteSecured` and answered
|
||||
two of the three questions; the acknowledge leg is now blocked on something narrower and
|
||||
different, described in "Second attempt".
|
||||
different, described in "Second attempt" and diagnosed in "Third attempt".
|
||||
|
||||
## First attempt (2026-08-17): plain `Write`
|
||||
|
||||
@@ -258,15 +258,164 @@ MXAccess-side entry point and the wnwrap-side entry point is inert.
|
||||
requirement that the wnwrap consumer, which passes an operator *name* string and no
|
||||
authenticated identity, cannot meet. If so, ack over wnwrap is not merely untested here
|
||||
but unavailable by configuration, and the gateway's `AcknowledgeByName` path needs the
|
||||
same treatment on any customer galaxy configured that way.
|
||||
same treatment on any customer galaxy configured that way. (Answered by the Third attempt
|
||||
below — enforced by the alarm attribute's `MxSecurityOperate` security classification, an
|
||||
engine-level write-security setting, not a separate `alarmmgr`-side ack policy.)
|
||||
- If neither lands, the acknowledge leg stays assumed. It is worth restating that this is a
|
||||
documentation gap, not a correctness one: a re-minted GUID on acknowledge would produce a
|
||||
spurious Clear plus a spurious Raise, which is the same shape the now-observed re-raise
|
||||
behaviour produces and which `ComputeTransitions` already handles as two instances.
|
||||
|
||||
## Third attempt (2026-08-18): why the acknowledge is refused
|
||||
|
||||
The second attempt left two candidate explanations for `AlarmAckByName` returning `rc=0`
|
||||
and changing nothing: the rig enforces an acknowledgement security requirement the wnwrap
|
||||
consumer cannot meet, or wnwrap's ack is simply broken here. This attempt was read-only —
|
||||
no writes, no alarms raised, no configuration touched — and settles the first question:
|
||||
the requirement is **enforced**, and the alarm attribute's security classification is what
|
||||
enforces it.
|
||||
|
||||
### Method
|
||||
|
||||
Read-only inspection of the `ZB` Galaxy Repository over `sqlcmd -S localhost -d ZB -E`,
|
||||
plus the already-built `mxa` CLI (`C:\Users\dohertj2\Desktop\wwtools\mxaccesscli\src\MxAccess.Cli\bin\x86\Release\net48\mxa.exe`)
|
||||
for runtime reads. Nothing was written and nothing was installed. The `lmxopcua\gr` schema
|
||||
notes referenced elsewhere in this repo are **not present on this box** (`Test-Path` is
|
||||
`False`), so the schema was located by querying `sys.tables` / `sys.columns` directly.
|
||||
|
||||
### The test attribute is classified `MxSecurityOperate`
|
||||
|
||||
UDA security classification lives in `dynamic_attribute.security_classification`, keyed by
|
||||
`gobject_id`. For the `$TestMachine` template that is `1055`:
|
||||
|
||||
```
|
||||
SELECT gobject_id, tag_name, hierarchical_name FROM gobject WHERE gobject_id=1055;
|
||||
1055|$TestMachine|$TestMachine
|
||||
```
|
||||
|
||||
```
|
||||
attribute_name | security_classification | mx_attribute_category
|
||||
ProtectedValue | 2 | 10
|
||||
ProtectedValue1 | 3 | 10
|
||||
TestAlarm001 | 1 | 10
|
||||
TestAlarm002 | 1 | 10
|
||||
TestAlarm003 | 1 | 10
|
||||
TestChangingInt | 1 | 10
|
||||
```
|
||||
|
||||
The enum is pinned by two independent sources rather than assumed. `ProtectedValue` and
|
||||
`ProtectedValue1` are the mxaccess analysis project's documented *secured-write* and
|
||||
*verified-write* fixtures (`C:\Users\dohertj2\Desktop\mxaccess\docs\galaxy-test-fixtures.md`),
|
||||
and `docs/NMX-COM-Contracts.md` in the same project records "Galaxy security classification
|
||||
(`2` for `SecuredWrite`, `3` for VerifiedWrite)" — so `2` and `3` land on exactly the two
|
||||
attributes that are supposed to carry them. `galaxy-test-fixtures.md` also records the
|
||||
provisioning verb used for every UDA in that inventory: `--security MxSecurityOperate`.
|
||||
`TestAlarm001` therefore reads `1` = **`MxSecurityOperate`**.
|
||||
|
||||
That is the missing piece from 2026-08-17. `Operate` is not free access: it requires an
|
||||
authenticated galaxy identity holding Operate permission on the object's security group.
|
||||
An unauthenticated `Write` is refused with `SecurityError` `1008`, and the same write after
|
||||
`AuthenticateUser` succeeds — which is precisely the pair of results both prior attempts
|
||||
recorded. Galaxy security is live on this rig, not disabled.
|
||||
|
||||
`wwAlarmConsumerClass.AlarmAckByName` carries an operator *name* string and a comment. It
|
||||
carries no authenticated user id, no credential, and no token — there is no parameter on
|
||||
either the 6-arg or the 8-arg overload that could convey one. A consumer calling it cannot
|
||||
satisfy an `Operate` classification, and `rc=0` followed by no state change is what an ack
|
||||
dropped downstream of an accepted call looks like.
|
||||
|
||||
The one step this stops short of is a direct experiment: the rig's security configuration
|
||||
is out of scope for a read-only probe, so "an `Operate` alarm cannot be acknowledged
|
||||
without an authenticated identity" is inferred from the classification plus the observed
|
||||
`1008`/`rc=0` pattern rather than observed by relaxing the classification and watching the
|
||||
ack start working. The human step below is exactly that experiment.
|
||||
|
||||
### There is no writeable `.Ack` attribute — the second attempt targeted the only one there is
|
||||
|
||||
A plausible reading of the second attempt was that it wrote the wrong sub-attribute:
|
||||
ArchestrA alarm extensions are commonly described as exposing a writeable `.Ack` alongside
|
||||
the read-only `.Acked`. On this galaxy they do not. Every `Ack`-named attribute the alarm
|
||||
primitives define:
|
||||
|
||||
```
|
||||
attribute_name | security_classification | mx_attribute_category
|
||||
Acked | -1 | 2
|
||||
AckMsg | 0 | 6
|
||||
Bad.Acked | -1 | 2
|
||||
Bad.AckMsg | 0 | 6
|
||||
TimeAlarmAcked | -1 | 2
|
||||
AlarmAckCnt | -1 | 2
|
||||
AlarmAckErrorsCnt | -1 | 2
|
||||
AlarmMostUrgentAcked | -1 | 2
|
||||
AlarmUnAckedCnt | -1 | 2
|
||||
```
|
||||
|
||||
No `Ack`. `Acked` carries `security_classification = -1` — no classification at all, which
|
||||
is what a non-writeable attribute carries, as distinct from the `1` on the writeable
|
||||
`TestAlarm001`. That is the configuration-side counterpart of the `detail=1007`
|
||||
operational refusal the second attempt got when it wrote `.Acked`: not a permission
|
||||
refusal, but an attribute that has no write path to refuse. The second attempt had already
|
||||
found the only MXAccess-side candidate, and it is read-only by definition.
|
||||
|
||||
### No non-interactive acknowledge surface is installed
|
||||
|
||||
The `wwtools` collection on the box (`aalogcli`, `aot`, `graccesscli`, `grdb`, `histdb`,
|
||||
`mbproxy`, `mxaccesscli`, `secrets`) is the most likely home for a scriptable ack. There
|
||||
isn't one. `mxa --help` lists `diag`, `info`, `read`, `read-batch`, `subscribe`,
|
||||
`subscribe-batch`, `write`, `write-batch` — a tag data-plane only, with no alarm surface.
|
||||
`graccesscli` is a Galaxy Repository configuration tool (`object uda add`, `instance
|
||||
deploy`), which acts at configure/deploy time and not on live alarm state. Reading the
|
||||
galaxy's own authentication mode at runtime is also unavailable: `Galaxy.AuthenticationMode`
|
||||
does not resolve over MXAccess (`Category=4 Detail=6`), and the value is not in
|
||||
`dynamic_attribute` — only the attribute *definition* names `AuthenticationMode` and
|
||||
`_AuthenticationModeEnum` exist in `attribute_definition`.
|
||||
|
||||
So every remaining acknowledge surface on this rig is interactive: the IDE's alarm client,
|
||||
InTouch, or an ArchestrA graphic bound to the alarm. Driving those is out of scope.
|
||||
|
||||
### Status of the acknowledge leg
|
||||
|
||||
**Unavailable by configuration, and the GUID question stays assumed.** The two are separate
|
||||
statements and both matter:
|
||||
|
||||
- The wnwrap ack path is unavailable on this rig as configured, for an identified reason
|
||||
rather than an unknown one. This is a real finding for the gateway: `AcknowledgeByName`
|
||||
will behave the same way — accepted, inert — on any customer galaxy whose alarmed
|
||||
attributes carry a non-free-access security classification (inferred from the mechanism —
|
||||
no `AlarmAckByName` overload can carry a credential — not confirmed by relaxing the
|
||||
classification and re-testing; see "What a human would need to do"). It is worth noting in
|
||||
the alarm client's documentation that a silent `rc=0` is not proof of acknowledgement.
|
||||
- Whether wnwrap re-mints the record GUID on `UNACK_ALM → ACK_ALM` is still unobserved, and
|
||||
after three attempts it stays assumed. As the second attempt already noted, this remains
|
||||
a documentation gap rather than a correctness one: a re-minted GUID produces a spurious
|
||||
Clear plus a spurious Raise, the same shape the observed re-raise behaviour produces, and
|
||||
`ComputeTransitions` already handles that correctly as two instances.
|
||||
|
||||
#### What a human would need to do
|
||||
|
||||
Either of these answers it; the second is cheaper and also confirms or refutes the
|
||||
classification hypothesis above, which the read-only probe could only infer.
|
||||
|
||||
1. Acknowledge `TestMachine_001.TestAlarm001` from an interactive System Platform client
|
||||
(IDE alarm client, InTouch, or an ArchestrA graphic) while a wnwrap probe polls
|
||||
`GetXmlCurrentAlarms2` against `\\DESKTOP-6JL3KKO\Galaxy!TestArea`, and record whether
|
||||
`STATE` reaches `ACK_ALM` and whether `GUID` survives the transition. Raise the alarm
|
||||
first with `AuthenticateUser` + `WriteSecured` as the second attempt did.
|
||||
2. Reclassify `TestAlarm001` on the `$TestMachine` template to free access and redeploy —
|
||||
`graccesscli object uda ... --security MxSecurityFreeAccess` against `$TestMachine`,
|
||||
then `instance deploy TestMachine_001` — and re-run the second attempt's ack probe
|
||||
unchanged. If `AlarmAckByName` then moves `STATE` to `ACK_ALM`, the classification is
|
||||
confirmed as the blocker and the GUID question is answered in the same run. Restore the
|
||||
classification to `MxSecurityOperate` afterwards, since the secured-write fixtures in
|
||||
`WorkerLiveMxAccessSmokeTests` depend on the alarm UDAs being secured.
|
||||
|
||||
### Rig state left behind
|
||||
|
||||
The three `TestMachine_00{1,2,3}.TestAlarm001` UDAs are back to `false` and their
|
||||
The third attempt changed nothing. `TestMachine_001.TestAlarm001` and its `.Acked` subtag
|
||||
both read `false` at the end of the session, matching the state the second attempt left.
|
||||
|
||||
The second attempt's inventory, unchanged since: the three
|
||||
`TestMachine_00{1,2,3}.TestAlarm001` UDAs are back to `false` and their
|
||||
`.InAlarm` subtags read `false`, but each leaves a `UNACK_RTN` record in the wnwrap
|
||||
snapshot, since nothing can acknowledge them away. `SnapshotActiveAlarms` counts only
|
||||
`UNACK_ALM` and `ACK_ALM` as active, so these are inert for the gateway; they will clear on
|
||||
|
||||
+17
-4
@@ -138,7 +138,9 @@ would otherwise let any peer throttle a key it does not hold — and it joins th
|
||||
partition only when the presented token is validly shaped
|
||||
(`mxgw_<keyId>_<secret>`, key id at most 64 characters), with at most 32 key-id
|
||||
partitions per address before the overflow collapses onto that address's fallback
|
||||
partition. An over-limit state admits one probe per
|
||||
partition. That peer-layer collapse does not exempt a validly shaped key id from
|
||||
the cross-peer aggregate: its failures still feed and consult the key-id aggregate,
|
||||
whose own cardinality is bounded by the configured LRU cap. An over-limit state admits one probe per
|
||||
`ApiKeyFailureProbeIntervalSeconds` through to the real verifier and refuses
|
||||
everything else with `ResourceExhausted` before the store read, so a legitimate
|
||||
holder presenting the correct secret always reaches the constant-time compare and
|
||||
@@ -175,6 +177,17 @@ optional serialized constraints, and the `created_utc`, `last_used_utc`,
|
||||
belong to `ZB.MOM.WW.Auth.ApiKeys`, this document does not restate their column
|
||||
readers or SQL; consult the library for that detail.
|
||||
|
||||
One connection-level behavior is worth stating here because it is what keeps the
|
||||
store usable under concurrency rather than an implementation detail of the schema:
|
||||
the library's `AuthSqliteConnectionFactory.OpenConnectionAsync` opens pooled
|
||||
connections with a non-zero command timeout and applies `PRAGMA journal_mode=WAL`
|
||||
and `PRAGMA busy_timeout` (5 s). Last-used stamping runs on every authenticated
|
||||
request and the audit store appends on every denial, so without WAL and a busy
|
||||
timeout those concurrent writers would surface `SQLITE_BUSY` as a hard failure on
|
||||
the request path instead of degrading gracefully. WAL is a persistent
|
||||
database-level setting, so re-applying it per connection is a cheap no-op;
|
||||
`busy_timeout` is per-connection state and must be set each time.
|
||||
|
||||
### Audit trail
|
||||
|
||||
The library emits its own API-key audit entries (from the admin verbs — create,
|
||||
@@ -260,9 +273,9 @@ Examples:
|
||||
|
||||
```bash
|
||||
mxgateway apikey init-db
|
||||
mxgateway apikey create-key --key-id ops.alice --display-name "Alice (ops)" --scopes invoke:read,invoke:write
|
||||
mxgateway apikey create-key --key-id area1.reader --display-name "Area 1 reader" --scopes invoke:read,metadata:read --read-subtree "Area1/*" --browse-subtree "Area1/*"
|
||||
mxgateway apikey create-key --key-id ops.temp --display-name "Temp contractor" --scopes invoke:read --expires 90d
|
||||
mxgateway apikey create-key --key-id ops.alice --display-name "Alice (ops)" --scopes session:open,invoke:read,invoke:write
|
||||
mxgateway apikey create-key --key-id area1.reader --display-name "Area 1 reader" --scopes session:open,invoke:read,metadata:read --read-subtree "Area1/*" --browse-subtree "Area1/*"
|
||||
mxgateway apikey create-key --key-id ops.temp --display-name "Temp contractor" --scopes session:open,invoke:read --expires 90d
|
||||
mxgateway apikey create-key --key-id team-a.svc --display-name "Team A service" --scopes session:open,invoke:read --dashboard-tags team-a
|
||||
mxgateway apikey create-key --key-id ops.audit --display-name "Audit window" --scopes metadata:read --expires 2027-01-01T00:00:00Z
|
||||
mxgateway apikey list-keys --json
|
||||
|
||||
@@ -180,6 +180,21 @@ Supported constraints are:
|
||||
| `read_historized_only` | Read/subscription commands must target objects with historized attributes. |
|
||||
| `dashboard_tags` | Dashboard event-visibility tags. **Not a data-access constraint** — see below. |
|
||||
|
||||
`browse_subtrees` is applied by `GatewayBrowseScopeProvider`, the host seam the
|
||||
shared `ZB.MOM.WW.GalaxyRepository` package calls through `IGalaxyBrowseScopeProvider`;
|
||||
it reads the constraint off the ambient per-call identity that
|
||||
`GatewayGrpcAuthorizationInterceptor` pushed earlier in the same call. An empty
|
||||
`browse_subtrees` list is the library's "no scoping" signal, so the provider has to
|
||||
distinguish two states that look alike (GWC-38): an authenticated key with no browse
|
||||
constraint is legitimately unscoped and returns the empty list, whereas **no ambient
|
||||
identity at all** means the interceptor did not run before the Galaxy service and the
|
||||
request's scope cannot be established. In that second case, with
|
||||
`MxGateway:Authentication:Mode = ApiKey`, the provider logs a warning naming the broken
|
||||
ordering invariant and fails the call with `PermissionDenied` rather than serving the full
|
||||
hierarchy — a narrowing control's silent absence is invisible in normal operation, so it is
|
||||
made loud instead. Under `Mode = Disabled` an absent identity is the expected steady state
|
||||
(nothing authenticates, so no key and no constraints exist) and browse stays unscoped.
|
||||
|
||||
`dashboard_tags` is the one member of the blob that constrains nothing on the
|
||||
gRPC data path. No read, write, browse, or subscribe check consults it, and
|
||||
`HasReadConstraints` / `HasWriteConstraints` deliberately ignore it: adding a tag
|
||||
|
||||
@@ -72,6 +72,23 @@ carries the equivalent guard for the Go module: it refuses to create a
|
||||
`ClientVersion` already equals `X.Y.Z` (CLI-21/CLI-39), so a forgotten
|
||||
version bump fails the tag instead of shipping a mismatched module.
|
||||
|
||||
Both of those guards only fire when someone actually runs a publish.
|
||||
CLI-46 found the same hazard recurring silently *between* publishes: a
|
||||
real bug fix (the pypi 0.2.0 `connect()` fix above, commit `a8f86b5`) plus
|
||||
later wire-surface changes shipped into the tree while every version
|
||||
constant still read the already-published number, and nothing recorded
|
||||
that the registry artifact and the in-repo source had diverged.
|
||||
`scripts/check-client-versions.ps1` is the PR-time half of that guard —
|
||||
wired into CI (`.gitea/workflows/ci.yml`, the `portable` job, next to the
|
||||
codegen freshness checks): for each client it finds the latest publish tag
|
||||
(`clients/go/vX.Y.Z` for Go — the only client with a git tag at all; .NET,
|
||||
Python, Rust, and Java publish straight to a package registry via
|
||||
`pack-clients.ps1 -Publish` with no corresponding git tag, so the check
|
||||
reports "no tags found" and passes for those four today) and fails if
|
||||
`clients/<lang>` differs from that tag while the version constant still
|
||||
equals the tag's version. A client with no publish tag is not a
|
||||
violation — there is no "since the last publish" window to check yet.
|
||||
|
||||
As of 2026-08-07 (CLI-39) all five clients — plus `ZB.MOM.WW.MxGateway.Contracts`,
|
||||
which releases in lockstep with the .NET Client — moved to **0.2.0**,
|
||||
converging on one number after four of the five had drifted onto the
|
||||
@@ -111,6 +128,26 @@ confirmed present through the Gitea package API after the push, and
|
||||
`go list -m` resolves the Go tag. These are the numbers a future release
|
||||
bumps off.
|
||||
|
||||
**The published pypi `0.2.0` artifact has a broken `connect()` (CLI-46).**
|
||||
Its `connect()` built the `grpc.aio` channel inside `asyncio.to_thread`, and
|
||||
a `grpc.aio` channel binds to the event loop current on the *constructing*
|
||||
thread, so every non-stub connection raised `RuntimeError: There is no
|
||||
current event loop in thread` — TST-24's first real-wire run caught it.
|
||||
Fixed in-tree by commit `a8f86b5` (2026-08-10): `resolve_channel_security`
|
||||
now runs off-loop, `create_channel` runs on-loop. That fix, plus later
|
||||
wire-surface additions (`d9ea8a8`, `c748361`), shipped after the 2026-08-07
|
||||
publish while the version constants still read the already-published
|
||||
0.2.0/0.2.1 numbers above — see `clients/python/README.md` for the
|
||||
user-facing warning. **0.2.0 is superseded by 0.2.1; do not install it.**
|
||||
|
||||
As of this remediation (CLI-46), the in-source version constants for the
|
||||
.NET, Go, Python, and Rust clients (plus Contracts) read **0.2.1**, and the
|
||||
Java client reads **0.2.2**. These are not yet published — the numbers
|
||||
above (0.2.0 four clients + Contracts, 0.2.1 Java) remain the live registry
|
||||
state until the next `pack-clients.ps1` / `tag-go-module.ps1` run. Once that
|
||||
publish lands, update this section with the new coordinates and commit(s),
|
||||
the same way the 2026-08-07 record above was written after the fact.
|
||||
|
||||
## .NET
|
||||
|
||||
The .NET client uses .NET 10 and references
|
||||
|
||||
@@ -40,6 +40,15 @@ the corresponding MXAccess `AddItem`, `Advise`, `UnAdvise`, and `RemoveItem`
|
||||
calls sequentially on the session STA and preserves input order in the result
|
||||
list.
|
||||
|
||||
The shared alarm command model includes `SubscribeAlarms` and
|
||||
`UpdateAlarmWatchList`. `SubscribeAlarmsCommand.fallback_enabled` makes an empty
|
||||
initial watch list explicitly fallback-capable, allowing discovery recovery to
|
||||
populate a standby without restarting the primary alarm-manager subscription.
|
||||
`UpdateAlarmWatchListCommand.watch_list` is the complete replacement list. The
|
||||
worker prepares the replacement on its STA before swapping it into the existing
|
||||
failover consumer and returns a normal `MxCommandReply`; callers must treat only
|
||||
`protocol_status = OK` as acknowledgement of the new list.
|
||||
|
||||
The command model also includes bulk write/read command kinds:
|
||||
`WriteBulk`, `Write2Bulk`, `WriteSecuredBulk`, `WriteSecured2Bulk`, and
|
||||
`ReadBulk`. They are unary `Invoke` payloads on the same `MxAccessGateway`
|
||||
@@ -94,6 +103,15 @@ only and does not share types with `mxaccess_gateway.proto`. See
|
||||
[Galaxy Repository Browse](./GalaxyRepository.md) for the RPC catalog and
|
||||
behavior.
|
||||
|
||||
The retained proto remains the client-generation source of truth while the
|
||||
gateway serves the wire-identical descriptor from the
|
||||
`ZB.MOM.WW.GalaxyRepository` package. `GalaxyContractParityTests` converts both
|
||||
loaded descriptors to `FileDescriptorProto`, removes source locations, and
|
||||
normalizes the C# namespace and file name before comparing deterministic bytes.
|
||||
Its mutation test also requires drift failures to identify the first differing
|
||||
protobuf message. Any parity failure must be resolved before publishing either
|
||||
Contracts or clients.
|
||||
|
||||
Generated C# output is written to `src/ZB.MOM.WW.MxGateway.Contracts/Generated/`. Do not
|
||||
hand-edit generated files.
|
||||
|
||||
@@ -140,6 +158,13 @@ Run the focused contract tests after changing either `.proto` file:
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter ProtobufContractRoundTripTests
|
||||
```
|
||||
|
||||
When changing `galaxy_repository.proto` or the served Galaxy Repository
|
||||
package, also run the retained-vs-served parity guard:
|
||||
|
||||
```bash
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter GalaxyContractParityTests
|
||||
```
|
||||
|
||||
The full solution build also regenerates the C# contracts before compiling
|
||||
gateway and test projects:
|
||||
|
||||
|
||||
@@ -66,6 +66,23 @@ declaration-ordered JSON; and the row sits alone on its own line for Go and for
|
||||
Rust's `--jsonl`, but inside an `events` array for Python and for Rust's
|
||||
aggregate `--json`.
|
||||
|
||||
Alarm-feed status rows deliberately preserve each CLI's established JSON
|
||||
mapping while making false values explicit in every text formatter:
|
||||
|
||||
| CLI | Healthy provider text | Clearing snapshot text | JSON false-value shape |
|
||||
|-----|-----------------------|------------------------|------------------------|
|
||||
| `mxgw-dotnet` | `provider-status mode=Unspecified degraded=false reason=""` | `snapshot-status truncated=false` | `{"providerStatus": {}}` / `{"snapshotStatus": {}}` |
|
||||
| `mxgw-go` | `provider-status mode=ALARM_PROVIDER_MODE_UNSPECIFIED degraded=false reason=""` | `snapshot-status truncated=false` | `{"providerStatus":{}}` / `{"snapshotStatus":{}}` |
|
||||
| `mxgw-py` | same JSON row as `--json` | same JSON row as `--json` | `{"providerStatus": {}}` / `{"snapshotStatus": {}}`, emitted one row at a time |
|
||||
| `mxgw-java` | `provider-status mode=ALARM_PROVIDER_MODE_UNSPECIFIED degraded=false reason=` | `snapshot-status truncated=false` | `{"providerStatus":{}}` / `{"snapshotStatus":{}}` |
|
||||
| `mxgw-rs` | explicit `degraded=false` summary | explicit `truncated=false` summary | always includes `"degraded":false` / `"truncated":false` |
|
||||
|
||||
For the protobuf-JSON rows, an absent proto3 boolean means `false`; Rust's
|
||||
always-explicit booleans are semantically equivalent. Python `stream-alarms`
|
||||
defaults to an unbounded feed and emits each row immediately; a positive
|
||||
`--max-messages` value provides a bounded sample and includes both status
|
||||
preamble rows in the count.
|
||||
|
||||
## Integration Gate
|
||||
|
||||
Cross-language smoke execution is opt-in. Runners should skip the matrix unless
|
||||
|
||||
@@ -90,9 +90,9 @@ detach-grace and replay-retention windows are on by default, the reconnect
|
||||
surface is a trust boundary — a retained session outlives the stream that opened
|
||||
it. The session records the opening key id (`GatewaySession.OwnerKeyId`) and
|
||||
every `StreamEvents` attach/reattach is rejected with `PermissionDenied` unless
|
||||
the caller's key id matches the owner. Gating on the `event` scope alone was
|
||||
rejected: it would let any `event`-scoped key that learned a session id attach to
|
||||
another key's retained session and receive its replayed and live data. The check
|
||||
the caller's key id matches the owner. Gating on the `events:read` scope alone was
|
||||
rejected: it would let any `events:read`-scoped key that learned a session id
|
||||
attach to another key's retained session and receive its replayed and live data. The check
|
||||
runs before any subscriber is attached, so a foreign key never touches the
|
||||
replay ring. Admin-scope override is deferred.
|
||||
|
||||
|
||||
@@ -123,7 +123,7 @@ launch CWD (SEC-01, SEC-33).
|
||||
| `MxGateway:Worker:ShutdownTimeoutSeconds` | `10` | Grace period for worker shutdown before the gateway treats shutdown as failed and may kill the worker process tree. |
|
||||
| `MxGateway:Worker:HeartbeatIntervalSeconds` | `5` | Worker heartbeat send interval and gateway heartbeat check cadence input. |
|
||||
| `MxGateway:Worker:HeartbeatGraceSeconds` | `15` | Maximum age of the last worker heartbeat before the gateway faults the worker. This must be greater than or equal to `HeartbeatIntervalSeconds`. |
|
||||
| `MxGateway:Worker:MaxMessageBytes` | `16842752` | Maximum worker IPC frame payload size in bytes. The validator allows values from `1024` through `268435456`, and additionally requires this to be at least `MxGateway:Protocol:MaxGrpcMessageBytes` plus a 65536-byte (64 KiB) envelope-overhead reserve. The default is the 16 MiB public gRPC cap plus that reserve. The gateway conveys the negotiated value to the worker in the handshake (`GatewayHello.max_frame_bytes`), so the worker frames to the same limit rather than a hard-coded default. |
|
||||
| `MxGateway:Worker:MaxMessageBytes` | `16842752` | Maximum worker IPC frame payload size in bytes. The validator allows values from `1024` through `268435456`, and additionally requires this to be at least `MxGateway:Protocol:MaxGrpcMessageBytes` plus a 65536-byte (64 KiB) envelope-overhead reserve **and at most that cap plus twice the reserve**. The default is the 16 MiB public gRPC cap plus one reserve. The gateway conveys the negotiated value to the worker in the handshake (`GatewayHello.max_frame_bytes`), so the worker frames to the same limit rather than a hard-coded default. The band is narrow by design, so changing either message-size knob alone no longer validates — move `MxGateway:Protocol:MaxGrpcMessageBytes` and this one together, keeping this one one to two reserves above it. |
|
||||
|
||||
`StartupProbeRetryAttempts`, `StartupProbeRetryDelayMilliseconds`,
|
||||
`PipeConnectAttemptTimeoutMilliseconds`, timeout values, heartbeat values, and
|
||||
@@ -132,7 +132,15 @@ to avoid accidental large allocations from malformed or oversized frames. It
|
||||
must also stay above `MaxGrpcMessageBytes` by the envelope-overhead reserve so a
|
||||
maximally-sized accepted gRPC payload always fits one worker frame once wrapped
|
||||
in a `WorkerEnvelope`; otherwise the oversized outbound write would fault the
|
||||
whole session. Startup validation fails fast when the headroom is violated.
|
||||
whole session. The coupling is two-sided: `MaxMessageBytes` may exceed
|
||||
`MaxGrpcMessageBytes` by at most *twice* the reserve, because the worker budgets
|
||||
a `DrainEvents` reply against the frame maximum less two reserves and the
|
||||
gateway then returns that same `MxCommandReply` through `Invoke`, where the gRPC
|
||||
send cap applies. Without the upper bound a legal configuration (large pipe
|
||||
maximum, stock gRPC cap) lets the worker pack a reply the pipe carries and the
|
||||
gRPC layer refuses to send — with the drained events already removed from the
|
||||
worker queue and therefore lost to both the caller and the event stream. Startup
|
||||
validation fails fast when either half of the headroom rule is violated.
|
||||
|
||||
## Session Options
|
||||
|
||||
@@ -186,15 +194,15 @@ events (a "gap") and must re-snapshot; whatever is still retained is replayed.
|
||||
| Option | Default | Description |
|
||||
|--------|---------|-------------|
|
||||
| `MxGateway:Dashboard:Enabled` | `true` | Enables Blazor Server dashboard route mapping. The dashboard mounts at the host root (`/`); there is no separate path-base prefix. |
|
||||
| `MxGateway:Dashboard:AllowAnonymousLocalhost` | `true` | Allows loopback dashboard requests to bypass the dashboard cookie requirement for local development. Remote requests still require dashboard authentication. |
|
||||
| `MxGateway:Dashboard:AllowAnonymousLocalhost` | `true` | Allows loopback dashboard requests to bypass the dashboard cookie requirement for local development. Remote requests still require dashboard authentication. The bypass is **read-only** — it satisfies a requirement including the `Viewer` role but never `AdminOnly`, so API-key CRUD and session Close/Kill stay closed — and since SEC-40 it does **not** reach the alarm feed: `/alarms` and `/hubs/alarms` carry `MxGateway.Dashboard.Alarms`, a policy built from a requirement type the bypass handler does not handle, so an anonymous loopback caller is denied the page, denied the hub, and not offered the Alarms link. The alarm monitor is gateway-central and its messages carry no session provenance, so alarm metadata cannot be tag-scoped the way session events are — requiring a real role claim is the narrowing that exists. Sign in (or use `DisableLogin` in dev, which mints a genuine principal) to see alarms; no setting re-opens the feed to the bypass. The same exclusion applies to `Authentication:Mode=Disabled`. |
|
||||
| `MxGateway:Dashboard:RequireHttpsCookie` | `true` | Sets the dashboard auth cookie's secure policy. `true` keeps `CookieSecurePolicy.Always` — the cookie is only sent over HTTPS, which matches a production HTTPS deployment. Set to `false` for plain-HTTP dev deployments to use `CookieSecurePolicy.SameAsRequest`; the cookie is still flagged Secure on HTTPS requests, but it can round-trip over HTTP. Browsers drop Secure cookies set over HTTP from non-localhost hosts, so leaving this `true` while serving the dashboard over plain HTTP will break login from any remote browser. |
|
||||
| `MxGateway:Dashboard:CookieName` | _(unset)_ | Dashboard auth cookie name. Leave unset (null/blank) to use the resolved default: `__Host-MxGatewayDashboard` when `RequireHttpsCookie` is `true` (Secure cookie, so the `__Host-` prefix's Secure/no-Domain/`Path=/` guarantees apply), else the plain `MxGatewayDashboard` (a `__Host-` cookie without Secure is silently dropped by browsers). Override it to give a distinct name to a gateway that shares a hostname with another gateway instance: browser cookies are scoped by host+path but **not** by port, so two instances on the same host would otherwise clobber each other's dashboard session under a shared cookie name. An explicit override always wins over the `__Host-` default. Changing it signs out existing dashboard sessions on next deploy. |
|
||||
| `MxGateway:Dashboard:SnapshotIntervalMilliseconds` | `1000` | Dashboard snapshot refresh interval used by the snapshot SignalR hub and the pages that subscribe to it. |
|
||||
| `MxGateway:Dashboard:RecentFaultLimit` | `100` | Maximum number of fault summaries projected into each dashboard snapshot. |
|
||||
| `MxGateway:Dashboard:RecentSessionLimit` | `200` | Maximum number of session summaries projected into each dashboard snapshot. |
|
||||
| `MxGateway:Dashboard:ShowTagValues` | `false` | Controls whether tag values reach the dashboard at all. `false` (the default) suppresses them on all three seams that carry one: (1) the **events hub mirror** — `DashboardEventBroadcaster` blanks `MxEvent.value` plus the alarm body's `current_value`/`limit_value` from a deep-cloned copy before it reaches any hub subscriber (see `docs/GatewayDashboardDesign.md`'s `EventsHub` row); (2) the **alarms hub** — `AlarmsHubPublisher` clears `current_value`/`limit_value` from a deep-cloned copy of each `AlarmFeedMessage`, on both value-bearing payload arms (`transition` and `active_alarm`), before broadcasting to `/hubs/alarms`; (3) the **`/browse` live-value panel** — `DashboardLiveDataService` substitutes the literal `[redacted]` for the value text of each **successfully read** tag; a failed read keeps its `-` placeholder (there was no value to suppress, and the row's error is left untouched, so the two cannot contradict each other). Both hub redactions clone: the source `MxEvent` is shared with the gRPC event stream and the replay ring, and the source `AlarmFeedMessage` fans out to gRPC `StreamAlarms` subscribers, so neither is mutated in place and **no gRPC client is affected by this flag** — it is a dashboard-display control only. Everything that is not the value survives on every seam: tag reference, alarm reference/severity/state/operator, data type, quality, status, and timestamps still render, so the dashboard stays diagnostic without disclosing process values. This is one of two independent layers: `IDashboardSessionAcl` decides *which* sessions a caller may subscribe to at all (see `GroupToTag` / `UntaggedSessionVisibility` below), while this flag decides what a permitted subscriber sees. Setting it `true` exposes tag values to everyone the ACL admits — every Administrator, plus each Viewer holding a matching tag — and, because the alarms hub is session-less, exposes alarm values to every dashboard client that can reach `/hubs/alarms`. |
|
||||
| `MxGateway:Dashboard:GroupToRole` | _(empty)_ | LDAP group → dashboard role mapping. Keys are LDAP group names (short CN or full DN — leading-RDN match). Values must be `Administrator` (read/write, API-key CRUD) or `Viewer` (read-only) — matched ordinally by the startup validator, so the spelling is exact and `Admin` is rejected. A user whose LDAP groups don't intersect this map cannot sign in; with no mapping at all, only the loopback bypass admits anyone. |
|
||||
| `MxGateway:Dashboard:GroupToTag` | _(empty)_ | LDAP group → dashboard visibility tags. Keys follow the same convention as `GroupToRole` (short CN or full DN — leading-RDN match, case-insensitive); values are tag lists. A dashboard user's granted tag set is the union over the groups they belong to; an unmapped group contributes nothing. **Visibility only:** tags scope which sessions' event streams a Viewer may observe on the dashboard — they never grant or deny data access, which stays with the API key's scopes and constraints. Independent of `GroupToRole`: a group may appear in either map, both, or neither. Empty (the default) means Viewers hold no tags, so under the default `UntaggedSessionVisibility` they observe no session's events. |
|
||||
| `MxGateway:Dashboard:GroupToRole` | _(empty)_ | LDAP group → dashboard role mapping. Keys must be **short** LDAP group names — the CN / first-RDN value, e.g. `GwAdmin`, not `ou=GwAdmin,ou=groups,dc=zb,dc=local` — matched case-insensitively. The shared `ZB.MOM.WW.Auth.Ldap` provider delivers a user's groups already stripped to short names, so a full-DN key can never match and the group silently maps to nothing. (The mapper does try the full literal string before falling back to the leading-RDN value, but on the live login path both branches see the same short name; the fallback only matters to non-library callers of the `IGroupRoleMapper<string>` seam.) Values must be `Administrator` (read/write, API-key CRUD) or `Viewer` (read-only) — matched ordinally by the startup validator, so the spelling is exact and `Admin` is rejected. A user whose LDAP groups don't intersect this map cannot sign in; with no mapping at all, only the loopback bypass admits anyone. |
|
||||
| `MxGateway:Dashboard:GroupToTag` | _(empty)_ | LDAP group → dashboard visibility tags. Keys follow the same convention as `GroupToRole` — short LDAP group names (CN / first-RDN value), matched case-insensitively; a full-DN key never matches the pre-stripped groups the LDAP provider returns. Values are tag lists. A dashboard user's granted tag set is the union over the groups they belong to; an unmapped group contributes nothing. **Visibility only:** tags scope which sessions' event streams a Viewer may observe on the dashboard — they never grant or deny data access, which stays with the API key's scopes and constraints. Independent of `GroupToRole`: a group may appear in either map, both, or neither. Empty (the default) means Viewers hold no tags, so under the default `UntaggedSessionVisibility` they observe no session's events. **Applies to session event streams only:** the alarm feed (`/alarms`, `/hubs/alarms`) is deliberately *not* tag-scoped — the alarm monitor is gateway-central and session-less, so an `AlarmFeedMessage` carries no session or owning key to intersect a tag grant against. Alarm metadata (reference, state, severity, category, area, acknowledging operator) is therefore galaxy-wide for every authenticated dashboard role, regardless of tag grant; alarm *values* still follow `ShowTagValues`. What SEC-40 narrowed there is admission rather than scope: the environment bypasses (`AllowAnonymousLocalhost`, `Authentication:Mode=Disabled`) no longer receive the feed at all. **Revocation:** the ACL re-resolves a caller's grant from their LDAP group claims against this map on every subscribe, so removing an entry takes effect at the next subscribe after the gateway restart that rebinds options (options bind once at startup); a change to the user's LDAP group membership takes effect when their principal is rebuilt — at re-login for cookie sessions, within one 5-minute hub-token lifetime for token clients. An already-open subscription is never re-checked and lives until it drops. |
|
||||
| `MxGateway:Dashboard:UntaggedSessionVisibility` | `AdminOnly` | Who may observe a session that carries no tags (its owning API key declared none). `AdminOnly` (default, fail-closed) restricts untagged sessions to dashboard Administrators. `AllViewers` shows them to every Viewer — opt-in for a single-tenant deployment that wants the pre-tag behaviour. Administrators always see every session regardless of tags. |
|
||||
| `MxGateway:Dashboard:DisableLogin` | `false` | Dev/test only. When `true`, replaces the cookie authentication handler with `DashboardAutoLoginAuthenticationHandler`, which auto-authenticates every dashboard request — including requests from remote browsers, not just loopback — as `AutoLoginUser` holding both `Administrator` and `Viewer` roles. No login form, LDAP bind, or cookie is involved. A loud one-time startup warning is logged. Differs from `AllowAnonymousLocalhost`: `DisableLogin` mints a real authenticated principal (so role-gated write affordances appear), whereas `AllowAnonymousLocalhost` satisfies the authorization requirement on loopback only without minting a principal (write affordances stay hidden). Never enable in production. **Production hard-stop (SEC-04):** when the host runs in the `Production` environment and `DisableLogin` is `true`, startup validation fails and the process aborts — the flag is only accepted outside Production, where the one-time startup warning still fires. |
|
||||
| `MxGateway:Dashboard:AutoLoginUser` | `(null)` | Username stamped on the synthetic principal when `DisableLogin` is `true`. Default `(null)` — a null or blank value falls back to `multi-role`. Has no effect when `DisableLogin` is `false`. |
|
||||
@@ -385,7 +393,7 @@ is **not yet available**.
|
||||
| Option | Default | Description |
|
||||
|--------|---------|-------------|
|
||||
| `MxGateway:Protocol:WorkerProtocolVersion` | `1` | Worker IPC protocol version expected by the gateway and worker. This must match `GatewayContractInfo.WorkerProtocolVersion`. |
|
||||
| `MxGateway:Protocol:MaxGrpcMessageBytes` | `16777216` | Public gRPC max send and receive message size in bytes. The same default is used by official clients. The validator allows values from `1024` through `268435456`, and requires `MxGateway:Worker:MaxMessageBytes` to exceed this by at least the 64 KiB worker-frame envelope reserve (see the Worker options). |
|
||||
| `MxGateway:Protocol:MaxGrpcMessageBytes` | `16777216` | Public gRPC max send and receive message size in bytes. The same default is used by official clients. The validator allows values from `1024` through `268435456`, and requires `MxGateway:Worker:MaxMessageBytes` to exceed this by at least the 64 KiB worker-frame envelope reserve and by no more than twice that reserve, so a `DrainEvents` reply budgeted at the pipe maximum still fits this send cap (see the Worker options). |
|
||||
|
||||
The protocol option is exposed for diagnostics and explicit deployment
|
||||
configuration, not for compatibility negotiation. A mismatch fails validation
|
||||
@@ -431,7 +439,7 @@ behavior.
|
||||
| `MxGateway:Alarms:Enabled` | `false` | Gates the gateway's always-on central alarm monitor. When `true`, the gateway opens one gateway-owned worker session dedicated to alarms, caches the active-alarm set, and fans it out to every client through the `StreamAlarms` RPC — no client opens its own session to see alarms. |
|
||||
| `MxGateway:Alarms:SubscriptionExpression` | _(empty)_ | AVEVA alarm-subscription expression the monitor subscribes on startup, in canonical `\\<machine>\Galaxy!<area>` form. The literal `Galaxy` provider is correct regardless of the Galaxy database name. When empty and `Enabled` is `true`, the gateway falls back to `\\<MachineName>\Galaxy!<DefaultArea>` if `DefaultArea` is set. |
|
||||
| `MxGateway:Alarms:DefaultArea` | _(empty)_ | Area name used to compose a default subscription when `SubscriptionExpression` is empty. If both are empty while `Enabled` is `true`, the monitor faults with a configuration diagnostic. |
|
||||
| `MxGateway:Alarms:ReconcileIntervalSeconds` | `30` | How often the monitor reconciles its in-process alarm cache against the worker's authoritative active-alarm snapshot, catching transitions the live poll-and-diff feed missed. Floored at 5 seconds. |
|
||||
| `MxGateway:Alarms:ReconcileIntervalSeconds` | `30` | How often the monitor re-resolves and, on material change, hot-swaps the subtag fallback watch-list, then reconciles its in-process alarm cache against the worker's authoritative active-alarm snapshot. The acknowledged watch-list fingerprint advances only after an OK worker reply. Floored at 5 seconds. |
|
||||
| `MxGateway:Alarms:PollIntervalMilliseconds` | `500` | Cadence at which the worker's STA polls the AVEVA alarm consumer (`GetXmlCurrentAlarms2`) for the active-alarm snapshot the live feed diffs. Must be between `100` and `3600000` (one hour): every poll is a COM call plus an XML parse on the same STA that serves reads and writes, so a tighter cadence starves the command path, while a value above an hour stops being a cadence and silently disables alarm polling. The gateway conveys the value to the worker via the `MXGATEWAY_ALARM_POLL_INTERVAL_MS` environment variable; a missing or out-of-range value leaves the worker on the 500 ms default rather than failing the session. |
|
||||
| `MxGateway:Alarms:MaxAlarmsPerFetch` | `1024` | Cap the worker passes to `GetXmlCurrentAlarms2`'s `maxAlmCnt`. Must be between `64` and `65536` — the worker is a 32-bit process that materializes each reply as one BSTR plus a full `XmlDocument`, so an unbounded cap faults the STA with an out-of-memory rather than merely slowing it. It doubles as the **truncation threshold**: a fetch returning exactly this many records is treated as truncated, because the COM API caps its reply with no "more available" flag. On a truncated poll the worker retains the alarms the capped reply could not mention instead of letting their absence read as a clear, and logs a rate-limited `AlarmSnapshotTruncated` warning to its stderr (identifiers and counts only). **Remediation when you see that warning: raise this value** so the steady-state active-alarm count fits inside one fetch. A galaxy permanently above the cap holds stale entries in the snapshot until a sub-cap poll, and loses clear→re-raise cycles that happen entirely out of window (see `docs/DesignDecisions.md`). Conveyed to the worker via the `MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH` environment variable; a missing or out-of-range value leaves the worker on the 1024 default. |
|
||||
|
||||
@@ -445,11 +453,11 @@ native wnwrap alarm-manager provider and the subtag-monitoring fallback.
|
||||
|
||||
| Option | Default | Description |
|
||||
|--------|---------|-------------|
|
||||
| `MxGateway:Alarms:Fallback:Mode` | `Auto` | Provider selection mode. `Auto` uses the alarm manager as primary and fails over to subtag monitoring after consecutive COM failures, then fails back automatically. `ForceAlarmManager` disables failover. `ForceSubtag` forces subtag monitoring on from startup. Values are case-insensitive. |
|
||||
| `MxGateway:Alarms:Fallback:Mode` | `Auto` | Provider selection mode. `Auto` uses the alarm manager as primary and fails over to subtag monitoring after consecutive COM failures, then fails back automatically. `ForceAlarmManager` disables failover (and is the one mode whose subtag watch-list is never graded, because no subtag consumer is built). `ForceSubtag` forces subtag monitoring on from startup. Values are case-insensitive. |
|
||||
| `MxGateway:Alarms:Fallback:ConsecutiveFailureThreshold` | `3` | Number of consecutive wnwrap COM failures (`COMException` or failure HRESULT from `Subscribe` / `GetXmlCurrentAlarms2`) before the monitor switches to subtag mode. Floored at 1. |
|
||||
| `MxGateway:Alarms:Fallback:FailbackProbeIntervalSeconds` | `30` | While in subtag mode, how often (in seconds) the monitor probes the wnwrap provider to detect recovery. Floored at 1. |
|
||||
| `MxGateway:Alarms:Fallback:FailbackStableProbes` | `3` | Number of consecutive clean wnwrap probes required before the monitor switches back to the alarm manager. Floored at 1. |
|
||||
| `MxGateway:Alarms:Fallback:Discovery:UseGalaxyRepository` | `true` | When `true`, the monitor queries the Galaxy Repository SQL database to build the subtag watch-list for the configured area. |
|
||||
| `MxGateway:Alarms:Fallback:Discovery:UseGalaxyRepository` | `true` | When `true`, the monitor queries the Galaxy Repository SQL database to build the subtag watch-list for the configured area. A failed query is not fatal — the monitor continues with whatever configuration contributed — but with `Mode` `Auto` or `ForceSubtag` the resulting coverage gap is reported on the alarm feed's `AlarmProviderStatus` (`degraded = true` with a `Subtag watch-list is empty` / `Galaxy Repository alarm discovery failed` reason) and shown on the dashboard as the amber `Alarm Manager (no subtag standby)` badge, instead of only a startup log line. |
|
||||
| `MxGateway:Alarms:Fallback:Discovery:Area` | _(empty)_ | Galaxy area to scope the Repository query to. Falls back to `MxGateway:Alarms:DefaultArea` when empty. Ignored when `UseGalaxyRepository` is `false`. This area is **not** used to compose a Repository-derived alarm's canonical `Galaxy!{area}.{reference}`: each discovered alarm uses its object's real Galaxy area (discovered via `gobject.area_gobject_id`), so the reference's group matches what the native alarmmgr emits. `Discovery:Area` / `DefaultArea` is used as the composition area only for explicit `IncludeAttributes` entries, which carry no discovered area. |
|
||||
| `MxGateway:Alarms:Fallback:Discovery:IncludeAttributes` | _(empty)_ | Explicit MXAccess attribute paths to add to the subtag watch-list, supplementing (or replacing, when `UseGalaxyRepository` is `false`) the Repository-derived list. |
|
||||
| `MxGateway:Alarms:Fallback:Discovery:ExcludeAttributes` | _(empty)_ | Attribute paths to remove from the merged watch-list (case-insensitive). The exclude is applied after the Repository-derived rows and the explicit `IncludeAttributes` entries are combined, so an exclude that matches an explicit include suppresses it too — excludes win. Ignored when `UseGalaxyRepository` is `false`. |
|
||||
|
||||
+122
-30
@@ -186,7 +186,7 @@ hubs stay for the audience that genuinely needs a wire.
|
||||
|---|---|---|
|
||||
| every page deriving from `DashboardPageBase` | `IDashboardSnapshotFeed.WatchAsync` | `DashboardSnapshotFeed` (singleton) multicasting one `IDashboardSnapshotService.WatchSnapshotsAsync` enumeration |
|
||||
| `SessionDetailsPage` | `IDashboardSessionEventSubscriber.Subscribe(sessionId)` | `DashboardEventBroadcaster` — the same singleton the session mirror publishes to, registered behind both interfaces |
|
||||
| `AlarmsPage` | `IGatewayAlarmService.StreamAsync` | the central alarm monitor, **provider status only**; the alarm rows still come from the 3 s `QueryAlarmsAsync` poll |
|
||||
| `AlarmsPage` | `IGatewayAlarmService.StreamAsync` | the central alarm monitor, **gateway-status frames only** — `provider_status` for the badge and `snapshot_status` for the truncated-snapshot banner; the alarm rows still come from the 3 s `QueryAlarmsAsync` poll |
|
||||
|
||||
The snapshot feed multicasts rather than handing each page its own enumeration:
|
||||
`WatchSnapshotsAsync` is not multicast on its own — each enumeration owns a timer
|
||||
@@ -245,8 +245,9 @@ cancelling, a detach-driven exit leaves the pill to the incoming subscription; w
|
||||
the pill therefore reports is the case it exists for — the channel completing under
|
||||
a page that is still watching.
|
||||
|
||||
`AlarmsPage` owns two loops of its own (the 3 s alarm poll and the provider-status
|
||||
badge) and bounds their drain at 5 seconds on dispose, for the same reason
|
||||
`AlarmsPage` owns two loops of its own (the 3 s alarm poll and the status feed that
|
||||
drives the provider badge and the truncated-snapshot banner) and bounds their drain
|
||||
at 5 seconds on dispose, for the same reason
|
||||
`DashboardPageBase` bounds its watch drain: both loops render through the renderer's
|
||||
dispatcher, and disposal can run on it. The two are drained concurrently, so the
|
||||
bound on disposal is 5 seconds in total rather than per loop — a wedged dispatcher
|
||||
@@ -273,7 +274,7 @@ bearer). Each hub class is `[Authorize(Policy = HubClientsPolicy)]`.
|
||||
| Hub | Path | Producer | Payload | Routing |
|
||||
|---|---|---|---|---|
|
||||
| `DashboardSnapshotHub` | `/hubs/snapshot` | `DashboardSnapshotPublisher` (BackgroundService consuming `IDashboardSnapshotService.WatchSnapshotsAsync`) | `DashboardSnapshot` | Sent to all connected clients on every snapshot tick, but only while at least one client is connected (see "Idle gating" below); new connections receive the current snapshot synchronously in `OnConnectedAsync`. |
|
||||
| `AlarmsHub` | `/hubs/alarms` | `AlarmsHubPublisher` (BackgroundService consuming `IGatewayAlarmService.StreamAsync(filter: null)`) | `AlarmFeedMessage` (`active_alarm` / `snapshot_complete` / `transition` / `provider_status` / `snapshot_status`) | Connected clients auto-join `__alarms__`; all clients receive every message. Publisher auto-reconnects every 5s on stream faults. The alarm value fields (`current_value` / `limit_value`) are stripped from a deep-cloned copy of the message when `Dashboard:ShowTagValues` is false (the default) — on both value-bearing payload arms, so neither a live transition nor a snapshot record leaks a process value; every other field still renders. The source message is never mutated: it fans out from the same feed to gRPC `StreamAlarms` subscribers, which this dashboard-display flag does not govern. Arms carrying no value are forwarded as-is, uncloned. |
|
||||
| `AlarmsHub` | `/hubs/alarms` | `AlarmsHubPublisher` (BackgroundService consuming `IGatewayAlarmService.StreamAsync(filter: null)`) | `AlarmFeedMessage` (`active_alarm` / `snapshot_complete` / `transition` / `provider_status` / `snapshot_status`) | Guarded by `MxGateway.Dashboard.Alarms`, not `HubClients`: a real authenticated dashboard role is required, so the `Authentication:Mode=Disabled` and anonymous-localhost bypasses are excluded (SEC-40). Connected clients auto-join `__alarms__`; all admitted clients receive every message — the feed is galaxy-wide and carries no session provenance, so no per-caller scoping applies (see "Alarm-feed visibility"). Publisher auto-reconnects every 5s on stream faults. The alarm value fields (`current_value` / `limit_value`) are stripped from a deep-cloned copy of the message when `Dashboard:ShowTagValues` is false (the default) — on both value-bearing payload arms, so neither a live transition nor a snapshot record leaks a process value; every other field still renders. The source message is never mutated: it fans out from the same feed to gRPC `StreamAlarms` subscribers, which this dashboard-display flag does not govern. Arms carrying no value are forwarded as-is, uncloned. |
|
||||
| `EventsHub` | `/hubs/events` | `DashboardEventBroadcaster` invoked by each session's internal dashboard-mirror subscriber on its `SessionEventDistributor` (registered when the session becomes Ready) | `MxEvent` | Clients call `SubscribeSession(sessionId)` to join `session:{id}`, which also registers them in `EventsHubViewerRegistry` — the mirror is gated on that registry, which counts hub and in-process viewers alike (see "Mirror gating" below). The dashboard is a first-class distributor subscriber, so it receives the session's events whether or not a gRPC client is streaming. It sees RAW session events — not the per-gRPC-subscriber `AfterWorkerSequence` filtering that `EventStreamService` applies at its own boundary — because the dashboard is a separate LDAP-authenticated monitoring view meant to show the session's full event activity. Tag values are stripped from the mirrored `MxEvent` copy by `DashboardEventBroadcaster` when `Dashboard:ShowTagValues` is false (the default) — event metadata (tag reference, quality, status, timestamps) still renders, but the value fields are blanked, so no value leaks through this seam. `SubscribeSession` is gated by `IDashboardSessionAcl` (SEC-25 / TST-15): a denied caller gets a `HubException`, is not joined to the group, and is not registered as a viewer, so the mirror stays off for a session nobody is legitimately watching. The same ACL gates the in-process seam the session-details page uses, so neither path is the weaker one. Value redaction remains an independent layer — it bounds what a *permitted* subscriber sees. |
|
||||
|
||||
### Default cadences
|
||||
@@ -285,11 +286,18 @@ Both seams consume the same producing services, so they share these cadences:
|
||||
- alarm publisher emits on each transition observed by the central monitor;
|
||||
- event publisher emits per event fanned by the session's `SessionEventDistributor`
|
||||
to its internal dashboard-mirror subscriber (independent of any gRPC `StreamEvents`);
|
||||
- the alarms page's provider-status badge resubscribes one second after its
|
||||
- the alarms page's status feed resubscribes one second after its
|
||||
`IGatewayAlarmService.StreamAsync` enumeration ends — the monitor completes a
|
||||
subscriber's stream when it falls behind and again when it restarts, both
|
||||
recoverable by resubscribing — and holds its last value in between. The page's
|
||||
alarm rows are independent of that stream and refresh on the 3 s poll.
|
||||
subscriber's stream only when that subscriber has fallen behind, which resubscribing
|
||||
recovers (a monitor restart keeps the channel and pushes cleared status frames through
|
||||
it) — and the badge and banner hold their last values in between. That feed carries both gateway-status frames: `provider_status` drives the
|
||||
badge, and `snapshot_status` drives the truncated-snapshot banner, so the caveat
|
||||
appears on the monitor's verdict change rather than up to three seconds later. Every
|
||||
subscriber is primed with a `snapshot_status` frame at open, so a page attaching
|
||||
mid-truncation needs no priming logic of its own. The page's alarm rows are
|
||||
independent of that stream and refresh on the 3 s poll, which also re-asserts the
|
||||
truncation verdict as its reconcile baseline — both sources read the same monitor
|
||||
verdict, and neither is synthesized page-side.
|
||||
|
||||
### Idle gating and snapshot cost
|
||||
|
||||
@@ -540,11 +548,22 @@ alarm-history store, so the page reflects only the live active set. The page is
|
||||
read-only; it does not acknowledge alarms. A provider-status badge tracks the
|
||||
central monitor's health from `IGatewayAlarmService.StreamAsync` in process — the
|
||||
alarm service is already a multi-subscriber fan-out, so the badge needs no SignalR
|
||||
client, no loopback socket, and no hub token — while the alarm rows themselves
|
||||
still come from the three-second poll. If `MxGateway:Alarms:Enabled` is
|
||||
client, no loopback socket, and no hub token — and the same subscription carries the
|
||||
`snapshot_status` frame behind the truncated-snapshot banner ("Alarm snapshot may be
|
||||
incomplete"), so a capped provider fetch is caveated the moment the monitor decides
|
||||
it. The alarm rows themselves still come from the three-second poll, which also
|
||||
re-asserts the truncation verdict as the reconcile baseline. If `MxGateway:Alarms:Enabled` is
|
||||
false the central monitor never starts, and the page says so instead of showing
|
||||
an empty list with no explanation.
|
||||
|
||||
The page carries `[Authorize(Policy = MxGateway.Dashboard.Alarms)]` rather than relying on the
|
||||
`Viewer` policy `MapRazorComponents<App>()` applies to every component route (SEC-40), so it
|
||||
needs a real authenticated dashboard role: the `Authentication:Mode=Disabled` and
|
||||
anonymous-localhost bypasses reach every other read-only page but not this one. It is the same
|
||||
policy `AlarmsHub` carries — the two read one central monitor and must not disagree about who
|
||||
may see it — and the side rail hides the Alarms link for callers the policy denies. What a
|
||||
permitted caller sees is galaxy-wide and not tag-scoped; see "Alarm-feed visibility".
|
||||
|
||||
### Live data source
|
||||
|
||||
Both the Browse subscription panel and the Alarms page read live MXAccess data
|
||||
@@ -679,8 +698,10 @@ Implemented behavior:
|
||||
- a static `/login` HTML form posts username/password to the gateway;
|
||||
- `DashboardAuthenticator` binds against `MxGateway:Ldap` (service-account bind,
|
||||
user search, candidate bind) using `Novell.Directory.Ldap.NETStandard`;
|
||||
- the user's `memberOf` (or short CN) is matched against
|
||||
`MxGateway:Dashboard:GroupToRole`; the resolved role(s) are emitted as
|
||||
- the user's groups arrive from the LDAP provider already stripped to short
|
||||
names (the CN / first-RDN value of each `memberOf` entry) and are matched
|
||||
against the short-name keys of `MxGateway:Dashboard:GroupToRole` — a full-DN
|
||||
key there never matches; the resolved role(s) are emitted as
|
||||
`ClaimTypes.Role` claims, alongside the per-group `mxgateway:ldap_group`
|
||||
claims;
|
||||
- a successful login signs in the `MxGateway.Dashboard` cookie scheme
|
||||
@@ -693,14 +714,25 @@ Implemented behavior:
|
||||
generic credential-rejected message;
|
||||
- antiforgery tokens guard the login and logout POSTs.
|
||||
|
||||
Three authorization policies are registered:
|
||||
Four authorization policies are registered:
|
||||
|
||||
- `MxGateway.Dashboard.Viewer` — Razor component routes. Satisfied by Admin or
|
||||
Viewer.
|
||||
- `MxGateway.Dashboard.Admin` — Admin-only write surfaces (API-key CRUD).
|
||||
- `MxGateway.Dashboard.HubClients` — SignalR hubs. Accepts the dashboard
|
||||
cookie OR a `MxGateway.Dashboard.HubToken` bearer (used by WebSocket upgrades
|
||||
where the cookie can't be forwarded).
|
||||
- `MxGateway.Dashboard.HubClients` — the snapshot and events SignalR hubs. Accepts
|
||||
the dashboard cookie OR a `MxGateway.Dashboard.HubToken` bearer (used by WebSocket
|
||||
upgrades where the cookie can't be forwarded).
|
||||
- `MxGateway.Dashboard.Alarms` — the alarm feed: `AlarmsHub` and the `/alarms`
|
||||
page. Same two schemes as `HubClients`, but built from
|
||||
`AuthenticatedDashboardRoleRequirement` instead of
|
||||
`DashboardAuthorizationRequirement` (SEC-40). Because `DashboardAuthorizationHandler`
|
||||
handles only the latter type, the read-only environment bypasses below cannot
|
||||
satisfy this policy — see "Alarm-feed visibility".
|
||||
|
||||
The first three requirement types are satisfiable by the environment bypasses;
|
||||
`MxGateway.Dashboard.Alarms` and the mounted Secrets UI's `secrets:manage` are not,
|
||||
both for the same structural reason — the bypass handler is typed to a requirement
|
||||
they do not use.
|
||||
|
||||
The in-process page feeds carry no authentication of their own, and need none:
|
||||
`MapRazorComponents<App>()` applies `RequireAuthorization(ViewerPolicy)` to the
|
||||
@@ -720,12 +752,30 @@ caller's granted tags, ordinal-ignore-case. A session's tags are inherited from
|
||||
owning API key's `--dashboard-tags` constraint and are immutable for the session's
|
||||
life, so a subscribe-time decision cannot go stale while the subscription lives and
|
||||
no per-event re-check is needed. A Viewer's grant comes from
|
||||
`Dashboard:GroupToTag` applied to their LDAP groups, stamped as
|
||||
`zb:dashboardtag` claims at cookie login and re-resolved (not copied) at hub-token
|
||||
mint, so the token's 5-minute lifetime bounds how long a revoked grant survives. A
|
||||
principal carrying no tag claims — anonymous localhost included — is an
|
||||
empty-grant Viewer. The mirror's value redaction is an independent layer: it bounds
|
||||
what a permitted subscriber sees, not who may subscribe.
|
||||
`Dashboard:GroupToTag` applied to their LDAP groups. Since SEC-41 the ACL **re-resolves**
|
||||
that grant on every subscribe, from the caller's `mxgateway:ldap_group` claims against the
|
||||
currently bound `GroupToTag` map — the same one `DashboardGroupTagMapping.MapGroupsToTags`
|
||||
call the hub-token mint makes — rather than trusting the `zb:dashboardtag` claims stamped
|
||||
into the cookie at login. Those stamped claims were resolved under whatever map was bound
|
||||
when the user signed in, and the auth cookie outlives the gateway restart that applies a
|
||||
`GroupToTag` edit, so trusting them let a revoked grant survive the restart meant to remove
|
||||
it. The stamped claims remain the **fallback** for a principal carrying no group claims,
|
||||
which in practice means the hub-token principal: the token payload carries roles and tags
|
||||
but no LDAP groups, and the mint already re-resolved them. That fallback cannot re-open the
|
||||
cookie hole, because `DashboardAuthenticator` stamps group and tag claims from one group
|
||||
list — a cookie principal holding a tag claim always holds the groups it came from, and a
|
||||
login yielding no groups yields no tags. A principal carrying neither kind of claim —
|
||||
anonymous localhost included — is an empty-grant Viewer. The mirror's value redaction is an
|
||||
independent layer: it bounds what a permitted subscriber sees, not who may subscribe.
|
||||
|
||||
**Revocation bounds.** Three different revocations propagate on three different clocks, and
|
||||
the difference is intrinsic to where each fact lives:
|
||||
|
||||
| Revocation | Takes effect |
|
||||
|---|---|
|
||||
| A `GroupToTag` entry removed from config | At the next subscribe after the gateway restart that rebinds options — identically on both seams, since both re-resolve against current options. `GatewayOptions` binds once at startup, so the restart is what applies the edit; nothing further is needed. |
|
||||
| An LDAP group-membership revocation | When the principal is next rebuilt: at re-login for the cookie seam, bounded by the cookie's sliding 8-hour idle life (renewable by activity); within one 5-minute token lifetime for the hub-token seam. Neither seam re-queries LDAP per subscribe. |
|
||||
| Either, for an **already-open** subscription | Never — the subscription lives until it drops. The ACL is a subscribe-time gate by design (session tags are immutable, so no per-event re-check is warranted); closing an in-flight stream on a grant change would need a revocation channel the dashboard does not have. Force it by closing the session, or by restarting the gateway, which tears down every circuit and hub connection. |
|
||||
|
||||
Two environmental bypasses still apply, both scoped to **read-only** access:
|
||||
`MxGateway:Authentication:Mode = Disabled` and `MxGateway:Dashboard:AllowAnonymousLocalhost`
|
||||
@@ -733,11 +783,50 @@ Two environmental bypasses still apply, both scoped to **read-only** access:
|
||||
role (`AnyDashboardRole`) but **never** the `AdminOnly` requirement. Destructive/admin
|
||||
surfaces (API-key CRUD, session Close, worker Kill) still demand a real `Administrator`
|
||||
role claim, so the "anonymous localhost is read-only" contract holds at the policy layer
|
||||
rather than only in downstream service re-checks. Remote requests (outside the `Disabled`
|
||||
rather than only in downstream service re-checks. Since SEC-40 the bypasses also do not
|
||||
reach the **alarm feed** (`MxGateway.Dashboard.Alarms`) — read-only is not the same as
|
||||
"may see the whole plant's alarm metadata"; see "Alarm-feed visibility" below. Remote requests (outside the `Disabled`
|
||||
bypass) always require an authenticated principal carrying at least the Viewer role. The
|
||||
loopback test trusts `Connection.RemoteIpAddress`; if forwarded-headers middleware is ever
|
||||
added upstream, `DashboardAuthorizationHandler.IsLoopbackRequest()` must be revisited.
|
||||
|
||||
### Alarm-feed visibility
|
||||
|
||||
The alarm feed is **not** tag-scoped, and that is a deliberate, bounded decision rather than
|
||||
an oversight in the SEC-25 ACL.
|
||||
|
||||
The gateway's alarm monitor is central and session-less: one subscription expression, one
|
||||
`IGatewayAlarmService` feed, consumed by `AlarmsHubPublisher` (which re-broadcasts every
|
||||
message to the `__alarms__` group) and by the `/alarms` page in process. An `AlarmFeedMessage`
|
||||
carries no session id and no owning API key, so there is nothing for the per-session tag
|
||||
intersection to intersect — a tag ACL cannot be applied to it without inventing provenance the
|
||||
feed does not have. Deriving a filter from the caller's tags would also mean equating a
|
||||
visibility tag with an alarm-reference prefix, which `GroupToTag` deliberately does not do
|
||||
(tags are visibility labels, never data-access constraints).
|
||||
|
||||
So **alarm metadata stays galaxy-wide for every authenticated dashboard Viewer**: alarm
|
||||
reference (a full tag path), state, severity, priority, category, area, and acknowledging
|
||||
operator are all visible to any real dashboard role, regardless of which sessions their tag
|
||||
grant covers. Alarm *values* (`current_value` / `limit_value`) are still stripped when
|
||||
`Dashboard:ShowTagValues` is false (the default), on the hub seam as on the events seam.
|
||||
|
||||
What SEC-40 tightened is the entry gate rather than the scope. Both readers of the monitor —
|
||||
`AlarmsHub` and the `/alarms` page — carry `MxGateway.Dashboard.Alarms`, whose requirement type
|
||||
the environment-bypass handler does not handle. Consequences:
|
||||
|
||||
- `MxGateway:Authentication:Mode = Disabled` and anonymous localhost (`AllowAnonymousLocalhost`,
|
||||
default `true`) **no longer receive the alarm feed at all**. They keep read-only access to every
|
||||
other dashboard page; `/alarms` and `/hubs/alarms` deny, and the side rail hides the Alarms link
|
||||
rather than offering a dead end.
|
||||
- `Dashboard:DisableLogin` auto-login is unaffected: it mints a genuinely authenticated principal
|
||||
holding both roles, so it satisfies the policy like any signed-in user.
|
||||
- Gating only the hub was rejected: the page reads the same monitor in process, so the disclosure
|
||||
would have been one navigation away.
|
||||
|
||||
Operators who *want* the alarm feed on an unauthenticated local console must sign in (or, in a dev
|
||||
environment, set `Dashboard:DisableLogin=true`); there is deliberately no config flag that re-opens
|
||||
the feed to the bypasses.
|
||||
|
||||
### DisableLogin dev bypass
|
||||
|
||||
`MxGateway:Dashboard:DisableLogin` (default `false`) is a third bypass for
|
||||
@@ -811,13 +900,16 @@ the deliberate mitigation: it bounds that exposure window without the cost of a
|
||||
revocation store. It now bounds a stale *tag* grant the same way: the token carries
|
||||
the tags resolved from the caller's LDAP groups at mint time, so removing a
|
||||
`GroupToTag` entry takes effect for token-authenticated hub connections within one
|
||||
lifetime. That 5-minute staleness bound covers token-authenticated connections only:
|
||||
a cookie principal carries the `zb:dashboardtag` claims stamped at login for the
|
||||
cookie's whole life, so for cookie-authenticated (in-process page) subscriptions a
|
||||
revoked `GroupToTag` grant takes effect at the user's next login, not within five
|
||||
minutes. That is where the per-session ACL's revocation need landed — a jti
|
||||
denylist stays deferred, since the short lifetime already bounds every grant the
|
||||
token carries.
|
||||
lifetime. Since SEC-41 the cookie seam is no longer the weaker one: the ACL
|
||||
re-resolves a cookie principal's grant from its group claims against current
|
||||
options on every subscribe, so a `GroupToTag` edit takes effect on both seams at
|
||||
the next subscribe after the restart that rebinds options — the stamped
|
||||
`zb:dashboardtag` claims are consulted only for a principal with no group claims
|
||||
to re-resolve from, i.e. the hub-token principal itself. What the token's lifetime
|
||||
still bounds uniquely is a stale **role** set and a stale LDAP **group** list, since
|
||||
both are copied into the payload at mint. See the revocation-bounds table under
|
||||
"Authentication And Authorization". A jti denylist stays deferred — the short
|
||||
lifetime already bounds every claim the token carries.
|
||||
|
||||
## Configuration
|
||||
|
||||
|
||||
@@ -593,9 +593,19 @@ Pending command handling:
|
||||
Timeouts should not assume the COM call stopped. A timed-out command may still
|
||||
finish inside the worker.
|
||||
|
||||
On timeout the client also forwards a `WorkerCancel` carrying the abandoned
|
||||
correlation id, best-effort: the gateway has stopped waiting, but the worker has
|
||||
not stopped working, and the worker owns a single STA. `WorkerPipeSession` routes
|
||||
On timeout **and on caller cancellation** the client also forwards a
|
||||
`WorkerCancel` carrying the abandoned correlation id, best-effort: the gateway
|
||||
has stopped waiting, but the worker has not stopped working, and the worker owns
|
||||
a single STA. Caller cancellation — a per-call deadline shorter than the
|
||||
gateway's command timeout, or a client disconnect cancelling the RPC's token — is
|
||||
the more common of the two paths, and under a retry storm it is exactly the
|
||||
backlog the cancel exists to shorten, so both arms send it; the pending command
|
||||
is failed as `CallerCancelled` rather than `GatewayShutdown` so metrics and logs
|
||||
name the real cause (both still map to gRPC `Cancelled`). The send is gated on
|
||||
the same ownership check as the timeout arm — only the arm that removed the
|
||||
pending entry sends, so a reply that won the race is never chased — and is
|
||||
skipped when the client's own stop token has fired, since the outbound channel is
|
||||
being completed and every pending command is failed anyway. `WorkerPipeSession` routes
|
||||
the cancel to `CancelCommand`, which drops the correlation from the STA queue if
|
||||
it has not started and replies `Canceled` for it. A cancel that arrives after the
|
||||
command reached MXAccess is a no-op — there is no way to abort an in-flight COM
|
||||
@@ -606,9 +616,10 @@ the worker reads the command and then its cancel and drops it before execution.
|
||||
|
||||
Cancels ride the same outbound channel as commands, whose capacity is
|
||||
`MaxPendingCommands + 4`: the reserve above the pending-command limit is what
|
||||
absorbs them, so a burst of timeouts stays bounded and cannot deadlock the
|
||||
enqueue path. Failing to send the cancel is logged at debug and never replaces
|
||||
the `CommandTimeout` the caller is owed.
|
||||
absorbs them, so a burst of timeouts or cancellations stays bounded and cannot
|
||||
deadlock the enqueue path. Failing to send the cancel is logged at debug and
|
||||
never replaces the `CommandTimeout` — or the `OperationCanceledException` — the
|
||||
caller is owed.
|
||||
|
||||
Cancellation outranks the deadline. When a caller's token is canceled around the
|
||||
same time the timeout fires, the command is reported as canceled
|
||||
|
||||
@@ -116,7 +116,7 @@ Optional live smoke variables:
|
||||
| `MXGATEWAY_LIVE_MXACCESS_CLIENT_NAME` | `ZB.MOM.WW.MxGateway.IntegrationTests` | Client name passed to `Register`. |
|
||||
| `MXGATEWAY_LIVE_MXACCESS_EVENT_TIMEOUT_SECONDS` | `15` | Maximum wait for the first `OnDataChange` (also used for the `OnWriteComplete` round-trip and the abnormal-exit fault transition). |
|
||||
| `MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_USER` | `admin` | ArchestrA user name passed to `AuthenticateUser` before the `WriteSecured` parity step. |
|
||||
| `MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD` | `admin123` | Password paired with the user above. Never logged; the test asserts the value does not appear in the WriteSecured diagnostic message. |
|
||||
| `MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD` | `password` | Password paired with the user above (matches the shared GLAuth `admin` user's dev password — see `glauth.md`). Never logged; the test asserts the value does not appear in the WriteSecured diagnostic message. |
|
||||
|
||||
The test output includes session id, worker process id, command status,
|
||||
HRESULT/status diagnostics, event sequence and handles, close status, and worker
|
||||
@@ -650,11 +650,13 @@ runtime because the x86 Worker cannot build on Linux:
|
||||
the tier is down, not the change**: an unreachable windev fails the job (never skips), so
|
||||
if it goes red without a related code change, check the tier — `ssh windev`, read
|
||||
`C:\build\mxaccessgw-ci\ci.log` — before assuming a real test failure.
|
||||
- **`nightly-windev`** (Linux runner, scheduled `0 6 * * *`) — re-runs the full x86 build +
|
||||
Worker.Tests on windev, then the live-MXAccess smoke (`MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`)
|
||||
and a full `slnx` build (which surfaces Windows-only analyzer diagnostics such as
|
||||
`xUnit1030`). Gated `if: github.event_name == 'schedule'`, so it never gates a push. On
|
||||
failure it opens a Gitea issue via the Actions token, since nobody watches the Actions page.
|
||||
- **`nightly-windev`** (Linux runner, scheduled `0 6 * * *`) — runs, in order, the x86 Worker
|
||||
build, `Worker.Tests`, a full `slnx` build (which surfaces Windows-only analyzer diagnostics
|
||||
such as `xUnit1030`), the net10 gateway suite, and the live-MXAccess smoke
|
||||
(`MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`). The checkout and all five tiers share the windev
|
||||
host lock, and the first nonzero native exit stops the sequence and propagates to the CI job.
|
||||
Gated `if: github.event_name == 'schedule'`, so it never gates a push. On failure it opens a
|
||||
Gitea issue via the Actions token, since nobody watches the Actions page.
|
||||
|
||||
### Runner capacity is shared and finite
|
||||
|
||||
|
||||
+10
-2
@@ -84,6 +84,12 @@ Carrying the enqueue timestamp into the worker layer is what lets queue-wait tim
|
||||
|
||||
An accepted gRPC command payload can still be too large for the worker pipe: the envelope built around it must fit `MxGateway:Worker:MaxMessageBytes`, which is validated at startup to sit at least a fixed envelope-overhead reserve above `MaxGrpcMessageBytes` (see the headroom rule in [Gateway Configuration](./GatewayConfiguration.md)) so this should not occur for a conformant payload, but if it does, `WorkerClient` raises `WorkerClientException(CommandTooLarge)` and `Invoke` reports `ResourceExhausted` for that command — the session is not faulted, so a client can retry with a smaller payload without reopening the session.
|
||||
|
||||
Selected control commands have additional public behavior beyond their protobuf shape:
|
||||
|
||||
| Command | Public behavior |
|
||||
|---|---|
|
||||
| `DrainEvents` | Returns a destructive diagnostic snapshot of the worker's outbound event queue. `max_events = 0` selects the default cap; positive values may not exceed `GatewayContractInfo.MaxDrainEventsPerCommand` (10,000). The worker also applies a serialized-byte budget derived from the negotiated frame limit, so a reply may contain fewer events than the requested count. When either cap leaves queued events, `MxCommandReply.diagnostic_message` reports the truncation and callers repeat `DrainEvents` until an empty reply. Events included in a drain are removed from the worker queue and therefore do not later appear on the normal session `StreamEvents` path; the background stream drain may concurrently consume other events first. |
|
||||
|
||||
### `StreamEvents`
|
||||
|
||||
`StreamEvents` is a server-streaming RPC. The handler delegates the full pipeline to `IEventStreamService` and just forwards each `MxEvent` onto the response stream. Keeping the channel and producer/consumer machinery out of the handler means cancellation, exception mapping, and metric bookkeeping live in one place.
|
||||
@@ -92,13 +98,15 @@ An accepted gRPC command payload can still be too large for the worker pipe: the
|
||||
|
||||
`AcknowledgeAlarm` is a unary, **session-less** RPC that acknowledges a single alarm. The handler validates `alarm_full_reference` inline (it does not run through `MxAccessGrpcRequestValidator`) and delegates to `IGatewayAlarmService.AcknowledgeAsync`. The always-on `GatewayAlarmMonitor` routes the ack over its own gateway-managed worker session — clients no longer open a session to acknowledge an alarm. A reference that parses as a canonical GUID forwards to `AcknowledgeAlarmCommand`; a `Provider!Group.Tag` reference forwards to `AcknowledgeAlarmByNameCommand`.
|
||||
|
||||
An `OK` response means the alarm provider accepted the acknowledgement, not that it applied it: the gateway forwards the ack and reports what the provider returned, in keeping with the parity rule. On galaxies whose alarmed attributes carry a non-free-access security classification the by-name ack is accepted and inert — the underlying `AlarmAckByName` conveys an operator name rather than an authenticated identity, observed on the probe rig with the mechanism inferred rather than confirmed — so a client that needs to confirm an acknowledgement should watch for the resulting transition on `StreamAlarms` instead of treating `OK` as proof. See [Alarm Probe Findings](./AlarmProbeFindings.md).
|
||||
|
||||
### `StreamAlarms`
|
||||
|
||||
`StreamAlarms` is a server-streaming, **session-less** RPC that attaches to the gateway's central alarm feed. The handler delegates to `IGatewayAlarmService.StreamAsync`. The stream opens with a `provider_status` and a `snapshot_status` `AlarmFeedMessage` (the current provider mode and snapshot-completeness verdict), then one `AlarmFeedMessage` carrying an `active_alarm` per currently-active alarm (the ConditionRefresh snapshot), then a single `snapshot_complete`, then a `transition` for every subsequent raise / acknowledge / clear — interleaved with a further `provider_status` on each failover/failback and a further `snapshot_status` on each change of the truncation verdict. It is served by the always-on `GatewayAlarmMonitor`, which owns a single gateway-managed worker session and fans out to every attached client — clients no longer open a session of their own. `alarm_filter_prefix`, when set, scopes the stream to a sub-tree.
|
||||
|
||||
### `QueryActiveAlarms`
|
||||
|
||||
`QueryActiveAlarms` is a server-streaming, **session-less** RPC that returns a point-in-time snapshot of the currently-active alarm set. Unlike `StreamAlarms`, it does not stay attached for live transitions — it writes the snapshot and completes, which clients use after a reconnect to reseed alarm state or to reconcile alarms that may have been missed during a transport blip. It is served from the always-on `GatewayAlarmMonitor` cache (`IGatewayAlarmService.CurrentAlarms`), so no worker session is opened. The handler does not run through `MxAccessGrpcRequestValidator`; it only null-checks the request inline, then streams one `ActiveAlarmSnapshot` per cached alarm, honouring cancellation between writes. When `alarm_filter_prefix` is non-empty, only alarms whose `alarm_full_reference` starts with that prefix (ordinal `StartsWith`) are emitted; an empty prefix returns the full set. Exceptions are translated by the same `MapException` path as the other handlers. The RPC requires the `event` scope (`GatewayScopes.EventsRead`), the same scope as `StreamEvents` and `StreamAlarms`.
|
||||
`QueryActiveAlarms` is a server-streaming, **session-less** RPC that returns a point-in-time snapshot of the currently-active alarm set. Unlike `StreamAlarms`, it does not stay attached for live transitions — it writes the snapshot and completes, which clients use after a reconnect to reseed alarm state or to reconcile alarms that may have been missed during a transport blip. It is served from the always-on `GatewayAlarmMonitor` cache (`IGatewayAlarmService.CurrentAlarms`), so no worker session is opened. The handler does not run through `MxAccessGrpcRequestValidator`; it only null-checks the request inline, then streams one `ActiveAlarmSnapshot` per cached alarm, honouring cancellation between writes. When `alarm_filter_prefix` is non-empty, only alarms whose `alarm_full_reference` starts with that prefix (ordinal `StartsWith`) are emitted; an empty prefix returns the full set. Exceptions are translated by the same `MapException` path as the other handlers. The RPC requires the `events:read` scope (`GatewayScopes.EventsRead`), the same scope as `StreamEvents` and `StreamAlarms`.
|
||||
|
||||
#### Provider status on the alarm feed
|
||||
|
||||
@@ -280,7 +288,7 @@ StatusCode statusCode = exception.ErrorCode switch
|
||||
};
|
||||
```
|
||||
|
||||
`WorkerClientException` follows the same pattern: `CommandTimeout` becomes `DeadlineExceeded`, `GatewayShutdown` becomes `Cancelled`, `InvalidState` becomes `FailedPrecondition`, `ProtocolViolation` becomes `Internal`, `CommandTooLarge` becomes `ResourceExhausted`, and unmapped codes fall through to `Unavailable`.
|
||||
`WorkerClientException` follows the same pattern: `CommandTimeout` becomes `DeadlineExceeded`, `GatewayShutdown` and `CallerCancelled` both become `Cancelled`, `InvalidState` becomes `FailedPrecondition`, `ProtocolViolation` becomes `Internal`, `CommandTooLarge` becomes `ResourceExhausted`, and unmapped codes fall through to `Unavailable`. `CallerCancelled` is the caller's own token firing — a per-call deadline shorter than the gateway command timeout, or a disconnect — and is separate from `GatewayShutdown` only so metrics and logs stop attributing those calls to a shutdown that did not happen; the status the client observes is the same.
|
||||
|
||||
## Event Streaming Model
|
||||
|
||||
|
||||
+2
-2
@@ -72,7 +72,7 @@ Observable gauges are pull-based; the `Meter` invokes the supplied callback when
|
||||
|------------|--------------|-------------|
|
||||
| `mxgateway.sessions.open` | `_openSessions` | Currently open sessions tracked by `SessionManager`. |
|
||||
| `mxgateway.workers.running` | `_workersRunning` | Worker clients in a running state. |
|
||||
| `mxgateway.events.worker_queue.depth` | `_workerEventQueueDepthSources` (summed on demand) | Undelivered worker events held by `WorkerClient` — staged *and* queued (GWC-24) — summed across every live client at collection time (GWC-30). Each client owns an interlocked counter incremented when the read loop stages an event and decremented when the consumer reads it, and registers it as a gauge source for its lifetime, so a backlog stuck in a staging channel is visible and concurrent sessions add up instead of overwriting one another. |
|
||||
| `mxgateway.events.worker_queue.depth` | `_workerEventQueueDepthSources` (summed on demand) | Undelivered worker events held by `WorkerClient` — staged *and* queued (GWC-24) — summed across every live client at collection time (PERF-02). Each client owns an interlocked counter incremented when the read loop stages an event and decremented when the consumer reads it, and registers it as a gauge source for its lifetime, so a backlog stuck in a staging channel is visible and concurrent sessions add up instead of overwriting one another. |
|
||||
| `mxgateway.events.grpc_stream_queue.depth` | `_eventStreamBacklogSources` (summed on demand) | Live backlog buffered across every active `EventStreamService` subscriber, summed from the subscribers' channel `Count` at collection time. |
|
||||
|
||||
## Snapshot Shape
|
||||
@@ -111,7 +111,7 @@ The scalar fields mirror the counters and gauges. The four dictionaries provide
|
||||
- `EventsBySession` keys by `sessionId`; entries are removed via `RemoveSessionEvents` when a session closes so the map does not grow without bound.
|
||||
- `RetryAttemptsByArea` keys by the resilience `area` tag, e.g. `worker_startup`.
|
||||
|
||||
`EventsReceived` is read with `Interlocked.Read(ref _eventsReceived)` because `EventReceived` increments it via `Interlocked.Increment` outside the lock to keep the event-ingestion path non-blocking. `CommandsStarted`, `CommandsSucceeded`, `CommandsFailed`, and `CommandFailuresByMethod` are read the same way: the command counters run two-to-three times per gRPC call, so they are recorded with `Interlocked` and a `ConcurrentDictionary` rather than under `_syncRoot` (GWC-30). The two queue depths are pulled from their registered sources before the lock is taken, since those delegates reach into subscriber channels and worker clients.
|
||||
`EventsReceived` is read with `Interlocked.Read(ref _eventsReceived)` because `EventReceived` increments it via `Interlocked.Increment` outside the lock to keep the event-ingestion path non-blocking. `CommandsStarted`, `CommandsSucceeded`, `CommandsFailed`, and `CommandFailuresByMethod` are read the same way: the command counters run two-to-three times per gRPC call, so they are recorded with `Interlocked` and a `ConcurrentDictionary` rather than under `_syncRoot` (PERF-02). The two queue depths are pulled from their registered sources before the lock is taken, since those delegates reach into subscriber channels and worker clients.
|
||||
|
||||
## Recording Sites
|
||||
|
||||
|
||||
@@ -90,13 +90,22 @@ Expected command-line arguments:
|
||||
--protocol-version <version>
|
||||
```
|
||||
|
||||
Expected protected environment values:
|
||||
Expected launch environment values (the shared names are defined by
|
||||
`WorkerLaunchEnvironment` in Contracts):
|
||||
|
||||
```text
|
||||
MXGATEWAY_WORKER_NONCE=<random nonce>
|
||||
MXGATEWAY_WORKER_LOG_CONTEXT=<optional context>
|
||||
MXGATEWAY_WORKER_PIPE_CONNECT_ATTEMPT_TIMEOUT_MS=<milliseconds>
|
||||
MXGATEWAY_WORKER_WRITE_COMPLETION_WAIT_MS=<milliseconds>
|
||||
MXGATEWAY_EVENT_QUEUE_CAPACITY=<events>
|
||||
MXGATEWAY_ALARM_POLL_INTERVAL_MS=<milliseconds>
|
||||
MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH=<alarms>
|
||||
```
|
||||
|
||||
The nonce is protected bootstrap material. The remaining values are derived
|
||||
from the public `MxGateway:Worker:*` and `MxGateway:Alarms:*` configuration
|
||||
keys documented in [Worker Process Launcher](./WorkerProcessLauncher.md).
|
||||
|
||||
Startup sequence:
|
||||
|
||||
1. Parse command-line arguments.
|
||||
@@ -312,11 +321,48 @@ write wait still yields the empty-`statuses` unconfirmed reply, and an expired
|
||||
per-tag `ReadBulk` wait still reports its own timeout.
|
||||
|
||||
The write wait's budget is `MxGateway:Worker:WriteCompletionWaitMilliseconds`
|
||||
(default 1500). It is a bounded hold on the STA per unary write, so deployments
|
||||
(default 1500). It applies to `Write`, `Write2`, `WriteSecured`, and
|
||||
`WriteSecured2` and is a bounded hold on the STA per single-item write, so deployments
|
||||
whose write workload is effectively fire-and-forget — no consumer reads the
|
||||
reply's `statuses` — can lower it, or set `0` to skip the wait entirely and
|
||||
reply on acceptance alone.
|
||||
|
||||
**Write-completion correlation is recency-based, not per-call.** MXAccess's
|
||||
`OnWriteComplete` callback carries no correlation token — only the server/item
|
||||
handle pair — so `MxAccessCommandExecutor` correlates a completion to a write
|
||||
by *version*: it snapshots `MxAccessWriteCompletionCache.CurrentVersion` for
|
||||
the handle as a baseline before the COM call, then reports the first
|
||||
completion whose version exceeds that baseline as the write's outcome
|
||||
(`AwaitWriteCompletion`). This is exact for the common case, but if an earlier
|
||||
write to the same handle times out unconfirmed — its `OnWriteComplete` still
|
||||
undelivered when the wait gives up — and the caller immediately issues another
|
||||
write to the same tag, the stale callback can dispatch during the second
|
||||
write's pump-wait and be reported as *its* outcome. The executor mitigates the
|
||||
most common instance of this (`CaptureCompletionBaseline`) by running one
|
||||
`pumpStep()` immediately before capturing the baseline, so a completion already
|
||||
queued for the handle folds into the baseline instead of being available to
|
||||
ride the next write's reply; this closes the gap for a callback that arrived
|
||||
before the new write started, but a callback that MXAccess queues after that
|
||||
pump step and before the COM call returns (or during the wait itself) is still
|
||||
indistinguishable from the new write's own completion. Full closure is not
|
||||
possible without an MXAccess-side correlation token, which does not exist —
|
||||
this is a property of the COM surface, not a gap this worker introduced (a
|
||||
native client correlating by handle has the identical ambiguity). The exposure
|
||||
is bounded: `statuses` is an enrichment field, and `Hresult`/`ProtocolStatus`
|
||||
stay acceptance-only regardless of which completion (if any) was attributed.
|
||||
|
||||
## Live alarm watch-list replacement
|
||||
|
||||
An auto-mode `SubscribeAlarms` with `fallback_enabled = true` always builds a
|
||||
`FailoverAlarmConsumer`, even when initial discovery supplied no targets. The
|
||||
handler retains the subscription expression so `UpdateAlarmWatchList` can build
|
||||
and subscribe a new subtag consumer entirely on the session STA. The failover
|
||||
consumer attaches the replacement's transition callback before swapping the
|
||||
standby reference, then detaches and disposes the old standby. If construction,
|
||||
subscription, or attachment fails, it disposes the replacement and preserves
|
||||
the old standby. The active-provider selector is never reset, so primary mode
|
||||
stays primary and subtag mode immediately delegates to the new standby.
|
||||
|
||||
## COM Creation
|
||||
|
||||
The MXAccess analysis source at `C:\Users\dohertj2\Desktop\mxaccess` identifies
|
||||
|
||||
+3
-1
@@ -213,6 +213,8 @@ The repair transitions the monitor's reconcile broadcasts on the alarm feed (Rai
|
||||
|
||||
`AttachInternalEventSubscriber` enforces the same readiness gate as `AttachEventSubscriber` — a session (or worker) that is not `Ready` throws `SessionNotReady` *before* the distributor is constructed. A premature attach would otherwise start the pump against a source that throws, completing every subscriber with that error and latching the distributor for the session's whole lifetime.
|
||||
|
||||
The same one-distributor-per-session rule is enforced at the other end of the lifecycle. Session disposal disposes the distributor and latches the fact, so an attach that passed its readiness gate and then lost the CPU until the session was torn down throws `SessionNotReady` rather than constructing a replacement distributor. A replacement can never be pumped — the pump-start flag is a one-way latch — so its subscriber would block on an empty channel that is never completed until the client's own deadline expired, with no error, log or metric to explain it. Every attach entry point (`AttachEventSubscriber`, `AttachEventSubscriberWithReplay`, `AttachInternalEventSubscriber`) fails this way, and each rolls back the subscriber-count slot it took, so a caller that loses the race to teardown sees the same failure it would have seen a moment later from the readiness gate.
|
||||
|
||||
Sessions open with `MxGateway:Sessions:DefaultLeaseSeconds` (default 1800) added to the open timestamp. Unary client activity refreshes the lease by the same duration. `ExtendLease` and `IsLeaseExpired` cooperate with `SessionManager.CloseExpiredLeasesAsync`, which iterates a registry snapshot and closes any session whose lease has expired with `LeaseExpiredReason`. `SessionLeaseMonitorHostedService` runs that sweep every `MxGateway:Sessions:LeaseSweepIntervalSeconds` seconds (default 30).
|
||||
|
||||
#### Teardown parallelism
|
||||
@@ -245,7 +247,7 @@ A session that faults (for example, a slow single-subscriber client that overflo
|
||||
|
||||
A client that drops mid-stream reconnects by re-issuing `StreamEvents` with `StreamEventsRequest.after_worker_sequence` set to the last `worker_sequence` it observed. A non-zero `after_worker_sequence` means *resume*; `0` means *fresh stream* and behaves exactly as a first-time subscribe — no replay, no sentinel.
|
||||
|
||||
**Owner-scoped attach (security control).** `OpenSession` records the caller's API key id on the session as `GatewaySession.OwnerKeyId`. Every `StreamEvents` attach and reattach is owner-checked: `EventStreamService.StreamEventsAsync` compares the caller's key id (threaded from `GatewayGrpcAuthorizationInterceptor` via `MxAccessGatewayService.StreamEvents`) to the session owner and throws `SessionManagerException(PermissionDenied)` — surfaced as gRPC `PermissionDenied` — when they differ, before any subscriber is attached and before any replayed or live event is delivered. Possessing the `event` scope and knowing a session id is not enough. This is what makes the default-on detach-grace and replay-retention windows safe: without the check, any `event`-scoped key that learned a session id could attach to another key's retained session and receive its replayed and live data. A `null` owner (session opened with authentication disabled) matches only a `null` caller key.
|
||||
**Owner-scoped attach (security control).** `OpenSession` records the caller's API key id on the session as `GatewaySession.OwnerKeyId`. Every `StreamEvents` attach and reattach is owner-checked: `EventStreamService.StreamEventsAsync` compares the caller's key id (threaded from `GatewayGrpcAuthorizationInterceptor` via `MxAccessGatewayService.StreamEvents`) to the session owner and throws `SessionManagerException(PermissionDenied)` — surfaced as gRPC `PermissionDenied` — when they differ, before any subscriber is attached and before any replayed or live event is delivered. Possessing the `events:read` scope and knowing a session id is not enough. This is what makes the default-on detach-grace and replay-retention windows safe: without the check, any `events:read`-scoped key that learned a session id could attach to another key's retained session and receive its replayed and live data. A `null` owner (session opened with authentication disabled) matches only a `null` caller key.
|
||||
|
||||
On a resume, `EventStreamService.StreamEventsAsync` attaches through `GatewaySession.AttachEventSubscriberWithReplay`, which calls `SessionEventDistributor.RegisterWithReplay`. That method snapshots the session's replay ring for events newer than `after_worker_sequence` **and** registers the live subscriber inside a single `_replayLock` critical section. This atomicity is what makes the replay→live handoff free of gaps and duplicates: the pump appends each event to the replay ring (under `_replayLock`) before fanning it to subscriber channels, so relative to that one critical section every event is either in the replay snapshot or fanned into the freshly-registered live channel — never both observably, never neither.
|
||||
|
||||
|
||||
@@ -69,8 +69,8 @@ a .NET Framework or COM interop build needs classic Visual Studio MSBuild.
|
||||
| Tool | Version | Path |
|
||||
| --- | --- | --- |
|
||||
| Go | 1.26.2 windows/amd64 | `C:\Program Files\Go\bin\go.exe` |
|
||||
| protoc-gen-go | latest installed by `go install` | `C:\Users\dohertj2\go\bin\protoc-gen-go.exe` |
|
||||
| protoc-gen-go-grpc | latest installed by `go install` | `C:\Users\dohertj2\go\bin\protoc-gen-go-grpc.exe` |
|
||||
| protoc-gen-go | v1.36.11 (pinned) | `C:\Users\dohertj2\go\bin\protoc-gen-go.exe` |
|
||||
| protoc-gen-go-grpc | 1.6.2 (pinned) | `C:\Users\dohertj2\go\bin\protoc-gen-go-grpc.exe` |
|
||||
|
||||
Environment:
|
||||
|
||||
@@ -80,11 +80,14 @@ GOPATH=C:\Users\dohertj2\go
|
||||
Go plugin bin=C:\Users\dohertj2\go\bin
|
||||
```
|
||||
|
||||
Installed plugin commands:
|
||||
Installed plugin commands. Install the pinned versions, not `@latest`: the committed Go
|
||||
bindings stamp the plugin version in their headers, so `clients/go/generate-proto.ps1`
|
||||
throws when a plugin reports anything else. That script is the source of truth for the
|
||||
pin — change it there first, then update this table.
|
||||
|
||||
```powershell
|
||||
go install google.golang.org/protobuf/cmd/protoc-gen-go@latest
|
||||
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest
|
||||
go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.11
|
||||
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.6.2
|
||||
```
|
||||
|
||||
## Rust
|
||||
|
||||
@@ -43,9 +43,23 @@ guarantees hellos, heartbeats, acks, and faults fit.
|
||||
Every worker-to-gateway frame must serialize within this limit, control replies
|
||||
included, so reply builders truncate to fit rather than emit a frame the writer
|
||||
will reject. `WorkerPipeSession` pre-sizes a `DrainEvents` reply below the
|
||||
negotiated maximum (less a fixed envelope/reply-wrapper reserve) and reports the
|
||||
truncation in the reply's `DiagnosticMessage`; the caller contract is to repeat
|
||||
`DrainEvents` until it returns an empty reply. Should a reply still overshoot,
|
||||
negotiated maximum and reports the truncation in the reply's
|
||||
`DiagnosticMessage`; the caller contract is to repeat `DrainEvents` until it
|
||||
returns an empty reply. The event budget is `max(MaxMessageBytes - 2 x 64 KiB,
|
||||
MaxMessageBytes / 2)`: the reserve is subtracted **twice** because the reply has
|
||||
two boundaries to clear, not one. The first reserve covers the
|
||||
`WorkerEnvelope`/`WorkerCommandReply`/`MxCommandReply` wrapper on the pipe; the
|
||||
second exists because the gateway returns that same `MxCommandReply` verbatim
|
||||
through `Invoke`, where the public gRPC send cap applies. `DrainEvents` removes
|
||||
the events it packs from the worker queue, so a reply the pipe carries and the
|
||||
gRPC layer then refuses would lose them for good — the gateway's validator
|
||||
therefore also caps `MxGateway:Worker:MaxMessageBytes` at
|
||||
`MxGateway:Protocol:MaxGrpcMessageBytes` plus **two** reserves, which is what
|
||||
makes the doubled subtraction sufficient rather than merely generous. The
|
||||
`MaxMessageBytes / 2` floor keeps the budget usable at frame maxima small enough
|
||||
that two reserves would consume them (below 256 KiB), where a collapsed budget
|
||||
would truncate every drain and never let the drain-until-empty caller finish.
|
||||
Should a reply still overshoot,
|
||||
`MessageTooLarge` at the reply-write seam is answered with a small
|
||||
`InvalidRequest` reply for that correlation, not with session teardown — no
|
||||
diagnostics command may kill a session.
|
||||
|
||||
@@ -26,6 +26,23 @@ The launcher sets the nonce through the `MXGATEWAY_WORKER_NONCE` environment
|
||||
variable. The nonce is not included in `WorkerProcessCommandLine` so logs and
|
||||
diagnostics can report the launch command without exposing the secret.
|
||||
|
||||
All launch-variable names live in the shared Contracts
|
||||
`WorkerLaunchEnvironment` class and are consumed by both the launcher and the
|
||||
worker. The launcher always exports all six values:
|
||||
|
||||
| Environment variable | Gateway source | Worker default when missing/invalid |
|
||||
|---|---|---|
|
||||
| `MXGATEWAY_WORKER_NONCE` | `WorkerProcessLaunchRequest.Nonce` | None; bootstrap fails because the nonce is required |
|
||||
| `MXGATEWAY_WORKER_PIPE_CONNECT_ATTEMPT_TIMEOUT_MS` | `MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds` | `2000` ms |
|
||||
| `MXGATEWAY_WORKER_WRITE_COMPLETION_WAIT_MS` | `MxGateway:Worker:WriteCompletionWaitMilliseconds` | `1500` ms |
|
||||
| `MXGATEWAY_EVENT_QUEUE_CAPACITY` | `MxGateway:Worker:EventQueueCapacity` | `10000` events |
|
||||
| `MXGATEWAY_ALARM_POLL_INTERVAL_MS` | `MxGateway:Alarms:PollIntervalMilliseconds` | `500` ms |
|
||||
| `MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH` | `MxGateway:Alarms:MaxAlarmsPerFetch` | `1024` alarms |
|
||||
|
||||
The write-completion wait applies to the four single-item write variants:
|
||||
`Write`, `Write2`, `WriteSecured`, and `WriteSecured2`. Bulk writes report COM
|
||||
acceptance per entry and do not wait for `OnWriteComplete` rows.
|
||||
|
||||
## Validation And Cleanup
|
||||
|
||||
Before starting the process, the launcher validates that the configured worker
|
||||
@@ -44,11 +61,8 @@ process remains alive. The policy is configured by
|
||||
`WorkerOptions.StartupProbeRetryDelayMilliseconds`; the retry counter is
|
||||
recorded as `mxgateway.retries.attempted` with `area=worker_startup`.
|
||||
|
||||
The launcher also passes
|
||||
`MXGATEWAY_WORKER_PIPE_CONNECT_ATTEMPT_TIMEOUT_MS` to the worker process from
|
||||
`WorkerOptions.PipeConnectAttemptTimeoutMilliseconds`. The worker uses that
|
||||
value as the per-attempt named-pipe connect timeout inside its own bounded
|
||||
Polly retry loop.
|
||||
The worker uses the exported pipe-attempt value inside its own bounded Polly
|
||||
retry loop.
|
||||
|
||||
If startup fails or exceeds `WorkerOptions.StartupTimeoutSeconds`, the launcher
|
||||
kills the worker process tree, disposes the process handle, disposes the
|
||||
|
||||
@@ -241,7 +241,7 @@ Run `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx` (expect 0 warnings) a
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs`
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameReader.cs` (comment only)
|
||||
|
||||
**Why:** On net48, `NamedPipeClientStream.ReadAsync` ignores its `CancellationToken` (`WorkerFrameReader.cs:109-111`). Fault-path exits (event-drain fault, oversized event, watchdog, heartbeat write failure) leave `readTask` pending; it is unblocked only when `WorkerPipeClient`'s `using` disposes the pipe, at which point it faults with `ObjectDisposedException`/`IOException` on a Task nobody observes (the finally at `:303-309` awaits only heartbeat and drain). The frame-pooling change (GWC-30) makes this sharper: the abandoned read owns the per-instance `_lengthPrefix` buffer and possibly a rented ArrayPool payload — the reader's single-consumer invariant holds today only because nothing ever reads again after abandonment.
|
||||
**Why:** On net48, `NamedPipeClientStream.ReadAsync` ignores its `CancellationToken` (`WorkerFrameReader.cs:109-111`). Fault-path exits (event-drain fault, oversized event, watchdog, heartbeat write failure) leave `readTask` pending; it is unblocked only when `WorkerPipeClient`'s `using` disposes the pipe, at which point it faults with `ObjectDisposedException`/`IOException` on a Task nobody observes (the finally at `:303-309` awaits only heartbeat and drain). The frame-pooling change (PERF-22) makes this sharper: the abandoned read owns the per-instance `_lengthPrefix` buffer and possibly a rented ArrayPool payload — the reader's single-consumer invariant holds today only because nothing ever reads again after abandonment.
|
||||
|
||||
**Spec — constraints, implementer designs within them:**
|
||||
1. **No unobserved faulted Task.** After the stream is disposed, `readTask`'s fault must be awaited/observed (reuse `ObserveBackgroundTaskStopAsync`'s timeout-and-log shape, `:312-348`) before `WorkerPipeClient.RunAsync` returns.
|
||||
|
||||
@@ -608,6 +608,24 @@ Commit anything found: `docs: remediation plan doc sweep`
|
||||
|
||||
---
|
||||
|
||||
## PERF-ID register
|
||||
|
||||
This plan's tasks were cited in code comments and test names as `GWC-30`/`GWC-31` before
|
||||
those labels were recognized as colliding with the `archreview` finding register — see
|
||||
cycle-3 finding `GWC-31` (`archreview/2026-08-19/10-gateway-core.md`). The comments were
|
||||
swept to the `PERF-nn` prefix below (`nn` = this doc's task number); `GWC-30`/`GWC-31` no
|
||||
longer appear in perf-pass comments. Citations of the *genuine* archreview `GWC-30`
|
||||
("Frame reader still allocates a fresh 4-byte prefix array per frame",
|
||||
`archreview/2026-07-12/10-gateway-core.md`) — the gateway-side
|
||||
`Server/Workers/WorkerFrameReader.cs` fix and its test — were left untouched; they refer
|
||||
to a real register entry, not this plan.
|
||||
|
||||
| PERF-ID | Task | Files |
|
||||
|---|---|---|
|
||||
| PERF-02 | Task 2 — Metrics: pull-gauge for worker queue depth, lock-free command counters | `Server/Workers/WorkerClient.cs` (`:44`, `:355`), `Server/Metrics/GatewayMetrics.cs` (`:46`, `:213`, `:284`, `:463`, `:524`), `Tests/Metrics/GatewayMetricsTests.cs` (`:22`, `:61`), `docs/Metrics.md` |
|
||||
| PERF-07 | Task 7 — WorkerClient: pooled-timer timeout, single sizing pass, `WorkerCancel` on timeout | `Server/Workers/WorkerClient.cs` (`:236`, `:253`), `Tests/Gateway/Workers/WorkerClientTests.cs` (`:163`, `:179`) |
|
||||
| PERF-22 | Task 22 — Worker frame writer/reader: pooled buffers | `Worker/Ipc/WorkerFrameReader.cs` (`:23`), `Worker/Ipc/WorkerFrameWriter.cs` (`:671`), `docs/plans/2026-08-15-deferred-remediation.md` (Task cross-reference) |
|
||||
|
||||
## Explicitly deferred (decided, not forgotten)
|
||||
|
||||
| Finding | Why deferred |
|
||||
|
||||
@@ -480,22 +480,40 @@ design; they need only a cheap tip re-build there.
|
||||
rig shows the new GUID replaces the record. The prose that implied coexistence was
|
||||
corrected rather than the code.
|
||||
|
||||
Follow-ups recorded, not started:
|
||||
Follow-ups recorded, not started — **all closed 2026-08-18 on `feat/followups-tickets`**
|
||||
(plan `docs/plans/2026-08-18-followups-and-tickets.md`; per-bullet closing commits below):
|
||||
|
||||
- `WorkerPipeSessionTests.RunAsync_LongInFlightCommandThatKeepsPumping_DoesNotFaultAndDeliversReply`
|
||||
fails deterministically on windev **and on `main`** — pre-existing, needs its own
|
||||
investigation.
|
||||
investigation. *Closed (`7da52b6`, `462850a`, `aaeb86b`): test-harness defect — the fake
|
||||
runtime session stamped STA activity only at construction, so the watchdog correctly
|
||||
faulted `StaHung` pre-dispatch. Test-only fix; windev worker suite fully green (524/11
|
||||
skipped) for the first time.*
|
||||
- `check-codegen.ps1` Check 4 is unrunnable on Windows: the `protoc-gen-go` version banner
|
||||
carries a `.exe` suffix that the exact-string compare in
|
||||
`clients/go/generate-proto.ps1:10,55` does not tolerate.
|
||||
- Windev has `protoc-gen-go-grpc` 1.6.1 against the repo's pinned 1.6.2.
|
||||
`clients/go/generate-proto.ps1:10,55` does not tolerate. *Closed (`c94c4d4`, plus
|
||||
`8ae0c2f` for a second Windows-only blocker found during verification: PS 5.1 strips
|
||||
embedded double quotes from the Python probe's `-c` argument). Check 4 verified 4/4 on
|
||||
windev under both PowerShell 5.1 and pwsh 7.*
|
||||
- Windev has `protoc-gen-go-grpc` 1.6.1 against the repo's pinned 1.6.2. *Closed: 1.6.2
|
||||
installed on windev; `docs/ToolchainLinks.md` corrected from `@latest` to the pinned
|
||||
install commands (`6b5c737`).*
|
||||
- `clients/java`'s `checkGeneratedClean` is dead under Gradle 9 (`Project.exec` was
|
||||
removed); it needs `ExecOperations` injection to work again.
|
||||
removed); it needs `ExecOperations` injection to work again. *Closed (`df45cb4`) via
|
||||
`ProviderFactory.exec`; verified on Gradle 9.5.1 (macOS) and 9.4.1 (windev), including
|
||||
a configuration-cache ordering proof.*
|
||||
- `SettingsPage` renders every `EffectiveDashboardConfiguration` member except
|
||||
`RecentFaultLimit` / `RecentSessionLimit` (pre-existing, predates this branch).
|
||||
*Closed (`a390fe1`, test tightened in `fb68bdb`).*
|
||||
- The ack-leg probe stays blocked; unblock paths are in `docs/AlarmProbeFindings.md`.
|
||||
*Closed as answered-why (`d1ae43d`, `bc22792`, `1605f54`): the ack is unavailable by
|
||||
configuration — the test attributes carry `MxSecurityOperate` and no `AlarmAckByName`
|
||||
overload can carry a credential (inferred, caveated). The GUID-across-ack question
|
||||
itself stays assumed; the remaining paths need a human at an interactive client.*
|
||||
- The dashboard `AlarmsPage` truncation banner is still poll-driven — it could consume
|
||||
the new `snapshot_status` feed frame instead.
|
||||
the new `snapshot_status` feed frame instead. *Closed (`7b6dfba`, `f57a6ae`): the
|
||||
page's status feed loop now consumes `snapshot_status`; the poll stays as reconcile
|
||||
baseline.*
|
||||
- **A closed code-review finding regressed, or was never applied.** Server-012
|
||||
(`code-reviews/Server/findings.md:405-412`) is recorded *Resolved 2026-05-18* and claims
|
||||
it corrected two scope lists to the canonical `*:*` strings — the `CLAUDE.md`
|
||||
@@ -504,7 +522,9 @@ Follow-ups recorded, not started:
|
||||
with a third instance the finding never covered (`docs/Authentication.md`'s `ops.alice`
|
||||
example). The bookkeeping is the follow-up: other `Server-0xx` entries marked Resolved
|
||||
with documentation-only fixes should be spot-checked for the same pattern, since a
|
||||
finding that reads Resolved is not otherwise re-examined.
|
||||
finding that reads Resolved is not otherwise re-examined. *Closed (`d3ac527`): 20
|
||||
doc-only resolutions audited; two more regressions found and re-fixed (Server-040,
|
||||
Server-009); four moot (target files deleted); annotations recorded in findings.md.*
|
||||
|
||||
Explicitly decided, not an omission: **`../scadaproj/CLAUDE.md` needs no update.** The
|
||||
umbrella index records the *set* of `.proto` files this repo owns, and that set is
|
||||
|
||||
@@ -459,3 +459,82 @@ accepted-but-inert (`rc=0`, state stays `UNACK_ALM`), and `.Acked` is write-reje
|
||||
Wave 4 = Task 10, final integration review, then hold for the user's merge decision.
|
||||
- **Push cadence:** controller pushes the branch before any windev task needs its code
|
||||
there (Task 7 investigates on-branch; Tasks 8–9 need the wave-1/2 tips).
|
||||
|
||||
---
|
||||
|
||||
## As-built record (2026-08-18)
|
||||
|
||||
All 10 tasks completed on `feat/followups-tickets`; every classification-driven review
|
||||
chain resolved **Approved** (Tasks 5, 7, and 8 after fix rounds; Task 7's high-risk chain
|
||||
took two rounds, ending with the mid-window auto-release race made structurally
|
||||
unreachable — 30 s `BlockedDispatchSafetyNet` > every test's 20 s cancellation). The
|
||||
final integration review returned *Ready with reservations*; every reservation was fixed
|
||||
on-branch: I-2 (`6b5c737` — `ToolchainLinks.md` recorded `@latest` for plugins the
|
||||
regeneration script hard-pins), I-3 (`db7b1db` — three doc sites claimed full-DN
|
||||
`GroupToRole`/`GroupToTag` keys work; the Server-040 restoration proves they never match
|
||||
the pre-stripped groups; rows were pre-existing on `main`), M-2/M-5 (`1605f54`), M-3
|
||||
(`fb68bdb`), M-4 (`753d070`), and M-1 was discharged by evidence (a
|
||||
`--configuration-cache --rerun-tasks` discriminator proved `providers.exec` evaluates at
|
||||
execution time after `generateProto`; no change needed).
|
||||
|
||||
Verification (Task 9, windev CI clone at `5417222`, post-fix commits re-verified
|
||||
individually by their owners at their pushed tips): full `slnx` 0W/0E; worker x86 **524
|
||||
passed / 11 skipped / 0 failed** — the suite's first fully green windev run; gateway
|
||||
1155/1155 with no reruns; `check-codegen.ps1` **4/4 on Windows** under pwsh 7, and after
|
||||
`8ae0c2f` also end-to-end under Windows PowerShell 5.1; Gradle 9.4.1
|
||||
`checkGeneratedClean` green on windev (and 9.5.1 on macOS); live MXAccess smoke 8/8;
|
||||
toolchain pins now real on windev (`protoc-gen-go v1.36.11`, `protoc-gen-go-grpc 1.6.2`).
|
||||
|
||||
Notable findings made along the way:
|
||||
|
||||
- **Check 4 had a second Windows-only blocker** behind the `.exe` banner: Windows
|
||||
PowerShell 5.1 strips embedded double quotes when marshaling native-exe args, breaking
|
||||
the Python `grpcio-tools` version probe (`clients/python/generate-proto.ps1:39`).
|
||||
Fixed (`8ae0c2f`) by swapping quote nesting; no other native-exe invocation in the
|
||||
codegen scripts carries the pattern.
|
||||
- **The ack leg is blocked by galaxy security configuration**, not a missing verb: the
|
||||
probe fixtures carry `MxSecurityOperate`, and no `AlarmAckByName` overload can convey
|
||||
an authenticated identity (inferred from mechanism, honestly caveated). `gateway.md`
|
||||
and `docs/Grpc.md` now carry the "acceptance is not application" caveat on the
|
||||
acknowledge RPC.
|
||||
- **The `fca978d` tracking-marker sweep deleted substantive prose** along with markers in
|
||||
at least one place (Server-040's precedence comment plus an operator-facing RDN
|
||||
paragraph that was never part of the finding). A targeted re-read of that commit's
|
||||
larger comment deletions is ticket-worthy (203 files swept; only the audited file was
|
||||
examined).
|
||||
- **`generateProto`'s up-to-date check does not notice an out-of-band deletion** of a
|
||||
single generated file (protobuf-gradle-plugin behavior, orthogonal to this branch) — a
|
||||
manually deleted generated file stays missing until a `.proto` change or
|
||||
`--rerun-tasks` invalidates the task.
|
||||
- Windev process notes: the CI clone sat on a detached HEAD, so bare `git pull` silently
|
||||
no-ops (use `git pull origin <branch>`); concurrent MSBuilds on the clone can kill each
|
||||
other's child nodes (MSB4166) — `-m:1` avoids it; ssh can transiently refuse with "Too
|
||||
many authentication failures" under agent concurrency (back off and retry).
|
||||
|
||||
Follow-ups recorded, not started (deliberately small) — *first two closed 2026-08-18 on
|
||||
`feat/sweep-reread-tag-summary`*:
|
||||
|
||||
- The `fca978d` sweep re-read (above) — the one genuinely ticket-worthy item.
|
||||
*Closed in `f82dac1`: a mechanical pre-pass narrowed the sweep's 1,383 deletions to
|
||||
68 files / 816 residual prose lines; a judged review of every one found 17 collateral
|
||||
deletions across 10 files (rationale prose with no surviving equivalent — SessionManager
|
||||
metrics invariants, HubTokenService hollow-token guard, AcknowledgeAlarm routing
|
||||
remarks, LmxSubtagAlarmSource unsecured-write/idempotency notes, WnWrapAlarmConsumer's
|
||||
WIN-911 precedent, DashboardSessionAdminService error-boundary catches,
|
||||
WorkerPipeSession factory-null throw, three test-design rationales) — all restored,
|
||||
markers left stripped. Everything else flagged verified benign (`<inheritdoc/>`
|
||||
resolves to equal-or-richer interface docs, or the substance survives relocated).
|
||||
Verified: NonWindows slnx 0W/0E, touched gateway test classes 65/65; windev worker
|
||||
x86 build 0W/0E, worker suite 524/11/0.*
|
||||
- `DashboardGroupTagMapping`'s class-level `<summary>` still describes the lookup as
|
||||
"full DN first, leading-RDN fallback" — accurate mechanics, but it could point at the
|
||||
short-name-keys consequence the inline comment now records.
|
||||
*Closed in `4b6f907`: the summary now states full-DN keys can never match and points
|
||||
at the lookup comment in `MapGroupsToTags`.*
|
||||
- wwtools `mxa read` human-readable formatter throws `RuntimeBinderException` on a failed
|
||||
read (`ReadCommand.cs:137`); `--llm-json` works. Different repo, noted here so it isn't
|
||||
lost.
|
||||
|
||||
Explicitly decided, not an omission: **`../scadaproj/CLAUDE.md` needs no update** — no
|
||||
`.proto`, contract, command, or architecture fact the umbrella index records changed on
|
||||
this branch.
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
{
|
||||
"planPath": "docs/plans/2026-08-18-followups-and-tickets.md",
|
||||
"tasks": [
|
||||
{"id": 1, "subject": "Task 1: check-codegen Check 4 — Windows .exe banner normalization", "status": "pending"},
|
||||
{"id": 2, "subject": "Task 2: Gradle 9 checkGeneratedClean via ProviderFactory.exec", "status": "pending"},
|
||||
{"id": 3, "subject": "Task 3: SettingsPage RecentFaultLimit/RecentSessionLimit rows", "status": "pending"},
|
||||
{"id": 4, "subject": "Task 4: Authentication.md runnable-as-written examples pass", "status": "pending"},
|
||||
{"id": 5, "subject": "Task 5: AlarmsPage push-driven truncation banner (snapshot_status)", "status": "pending"},
|
||||
{"id": 6, "subject": "Task 6: Server-0xx doc-only resolution audit", "status": "pending", "blockedBy": [4]},
|
||||
{"id": 7, "subject": "Task 7: WorkerPipeSessionTests deterministic failure — investigate + fix (windev)", "status": "pending"},
|
||||
{"id": 8, "subject": "Task 8: Ack-leg probe bounded unblock attempt (windev rig)", "status": "pending", "blockedBy": [7]},
|
||||
{"id": 9, "subject": "Task 9: Windev toolchain pin + full verification", "status": "pending", "blockedBy": [1, 2, 3, 4, 5, 6, 7, 8]},
|
||||
{"id": 10, "subject": "Task 10: Bookkeeping — close the follow-ups record", "status": "pending", "blockedBy": [9]}
|
||||
{"id": 1, "subject": "Task 1: check-codegen Check 4 — Windows .exe banner normalization", "status": "completed", "commit": "c94c4d4", "review": "Approved (Haiku code review, no issues); macOS check-codegen 4/4; Windows-side proof deferred to Task 9"},
|
||||
{"id": 2, "subject": "Task 2: Gradle 9 checkGeneratedClean via ProviderFactory.exec", "status": "completed", "commit": "df45cb4", "review": "Approved; verified on Gradle 9.5.1 (local toolchain already Gradle 9 — task was dead here), clean-pass and forced-stale-fail both proven"},
|
||||
{"id": 3, "subject": "Task 3: SettingsPage RecentFaultLimit/RecentSessionLimit rows", "status": "completed", "commit": "a390fe1", "review": "Approved; fail-first render test with non-default values, 3/3 filtered"},
|
||||
{"id": 4, "subject": "Task 4: Authentication.md runnable-as-written examples pass", "status": "completed", "commit": "2b1efb5", "review": "Approved; three examples gained session:open, ops.audit correctly left session-less (metadata:read gates session-less Galaxy RPCs)"},
|
||||
{"id": 5, "subject": "Task 5: AlarmsPage push-driven truncation banner (snapshot_status)", "status": "completed", "commits": ["7b6dfba", "f57a6ae"], "review": "Spec compliant (rename judged in-scope); code review Approved after fixes: stale doc table row, monitor-contract comment (restart does NOT complete subscribers), mid-truncation attach priming test; 5/5 + 370/370 filtered"},
|
||||
{"id": 6, "subject": "Task 6: Server-0xx doc-only resolution audit", "status": "completed", "commit": "d3ac527", "review": "Spec compliant; code review Approved (controller verified DashboardGroupRoleMapping change comment-only). 20 entries audited; 2 regressions re-fixed (Server-040 swept comment block incl. RDN pre-strip paragraph; Server-009 WAL/busy-timeout prose); 4 moot (target deleted); Server-038 closed-not-regressed. Systemic finding: fca978d tracking-marker sweep deleted substantive prose in at least one place"},
|
||||
{"id": 7, "subject": "Task 7: WorkerPipeSessionTests deterministic failure — investigate + fix (windev)", "status": "completed", "commits": ["7da52b6", "462850a", "aaeb86b"], "review": "High-risk chain: spec compliant; code review Approved after two fix rounds (doc-comment attachment; watchdog-window headroom 200ms/1s + 2s window/30-frame floor; 30s BlockedDispatchSafetyNet > 20s CTS makes the mid-window auto-release race structurally unreachable). Root cause: test-harness defect — FakeRuntimeSession stamped LastStaActivityUtc only at construction, watchdog correctly faulted StaHung pre-dispatch. Test-only fix; windev worker suite 524 passed/11 skipped (was 523+1 fail)"},
|
||||
{"id": 8, "subject": "Task 8: Ack-leg probe bounded unblock attempt (windev rig)", "status": "completed", "commits": ["d1ae43d", "bc22792"], "review": "Spec compliant; code review Approved after fixes (superseded bullet closed, summary hedged, gobject_id evidenced, ack-caveat added to gateway.md + Grpc.md as authorized scope extension). Outcome: ack unavailable-by-configuration (MxSecurityOperate classification, no credential-bearing AlarmAckByName overload — inferred, honestly caveated); no .Ack attribute exists; no non-interactive platform-side surface; GUID-across-ack row stays Open with reason updated"},
|
||||
{"id": 9, "subject": "Task 9: Windev toolchain pin + full verification", "status": "completed", "verifiedAt": "5417222", "commit": "6b5c737", "result": "protoc-gen-go-grpc 1.6.2 installed; slnx 0W/0E; worker x86 524/11/0; gateway 1155/1155; check-codegen 4/4 (pwsh 7; and under PS 5.1 after 8ae0c2f fixed the quote-stripping probe bug it surfaced); Gradle 9.4.1 checkGeneratedClean green; live smoke 8/8; ToolchainLinks @latest rows corrected to pins (integration-review I-2)"},
|
||||
{"id": 10, "subject": "Task 10: Bookkeeping — close the follow-ups record", "status": "completed", "review": "None (trivial); closes the 2026-08-17 plan's follow-ups block with per-bullet commits, appends the as-built record (integration-review resolution: I-2 6b5c737, I-3 db7b1db, M-2/M-5 1605f54, M-3 fb68bdb, M-4 753d070, M-1 discharged by evidence; plus 8ae0c2f PS5.1 fix), final statuses here"}
|
||||
],
|
||||
"lastUpdated": "2026-08-18"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
# Cycle 3 P2 And Follow-Up Design
|
||||
|
||||
**Date:** 2026-08-25
|
||||
|
||||
**Source:** `archreview/2026-08-19/remediation/00-tracking.md`
|
||||
|
||||
**Scope:** The 17 findings recorded as deferred P2 plus the three follow-ups recorded by the cycle-3 integration gate. `SEC-37` and `TST-32` remain external actions and are not part of this repository change.
|
||||
|
||||
## Current-State Reconciliation
|
||||
|
||||
The 2026-08-25 audit refreshed `origin/main` and checked every finding against the current tree. Sixteen P2 findings remain open. `WRK-34` is already complete in commit `3679e59` and needs a tracker correction rather than another code change. All three integration-gate follow-ups remain open:
|
||||
|
||||
- a live worker command and gateway path for refreshing an alarm subscription's subtag watch-list;
|
||||
- the full WRK-32 recency limitation in the public reply-field comment;
|
||||
- publication of the already-bumped client release train.
|
||||
|
||||
## Design Principles
|
||||
|
||||
1. Preserve the existing worker STA and COM-affinity rules. Worker alarm-consumer construction, subscription changes, and disposal remain on the STA.
|
||||
2. Preserve the primary alarm-manager subscription during watch-list repair. A refresh must never unsubscribe and recreate the whole alarm feed.
|
||||
3. Make behavioral changes test-first and keep documentation in the same commit as the behavior it describes.
|
||||
4. Perform one protobuf regeneration wave for the watch-list command, the WRK-32 reply-field clarification, and the IPC-36 operator-facing comment correction.
|
||||
5. Publish clients only after every client-affecting change is integrated, all five client suites pass, the release commit is on `main`, and registry collision checks pass.
|
||||
6. Keep the canonical archreview tracker current at every batch boundary, including commit IDs and verification counts.
|
||||
|
||||
## Workstreams
|
||||
|
||||
### 1. Tracker And Low-Risk Currency
|
||||
|
||||
Correct WRK-34 to Done, repair the stale replay-watermark documentation (GWC-36), document the public `DrainEvents` command (IPC-38), reconcile `stillpending.md` (TST-33), wire the code-review README check into portable CI (TST-34), and replace the nonexistent `event` scope spelling with `events:read` (TST-36).
|
||||
|
||||
These edits form the first batch because they have a small conflict surface and establish an accurate tracker before code changes begin.
|
||||
|
||||
### 2. Gateway Correctness And Security
|
||||
|
||||
- **GWC-33:** stop faulted or closed worker clients from contributing frozen queue depth while retaining fault records for the configured grace period.
|
||||
- **GWC-35:** make snapshot reconciliation emit a Raise when a cached acknowledged alarm is replaced by an active unacknowledged instance.
|
||||
- **SEC-38:** consult and feed the cross-peer aggregate for every validly-shaped key ID, even when the per-peer key-partition cap routes the request through the fallback bucket. Aggregate cardinality remains bounded by the aggregate cache's existing eviction policy.
|
||||
- **SEC-39:** never let a null effective key ID clear the shared fallback partition.
|
||||
|
||||
Each change gets a focused regression test that first reproduces the filed failure mode.
|
||||
|
||||
### 3. Secondary Contracts
|
||||
|
||||
- **IPC-36:** move all six worker-launch environment-variable names into one net48-safe Contracts class, consume those constants on both sides, enumerate the launch contract in the launcher and worker design documentation, and test that the launcher sets every declared variable.
|
||||
- **IPC-37:** compare the retained gateway Galaxy descriptor with the descriptor served by the shared `ZB.MOM.WW.GalaxyRepository` package after normalizing source-only metadata.
|
||||
|
||||
This work makes the two non-protobuf peer contracts executable invariants rather than comments.
|
||||
|
||||
### 4. Client Alarm-Feed Parity
|
||||
|
||||
- **CLI-47:** explicitly render false-valued provider and snapshot status fields in .NET text output; document the deliberate protobuf-JSON omission behavior and Rust's explicit JSON shape.
|
||||
- **CLI-48:** add provider-status and clearing snapshot-status coverage in .NET, Go, and Java, including Go's missing `stream-alarms` test scaffold.
|
||||
- **CLI-49:** expose the two Go status payload aliases and update the alarm-feed alias documentation.
|
||||
- **CLI-50:** change Python `stream-alarms` from a one-message buffered sample to incremental streaming with an unbounded-by-default limit, while retaining timeout and explicit finite-limit behavior.
|
||||
|
||||
The Python change uses the same arrival-order streaming model as the other four CLIs. Documentation examples will request a finite bound when a bounded demonstration is desirable.
|
||||
|
||||
### 5. Live Alarm Watch-List Refresh
|
||||
|
||||
The complete periodic hot-swap design is selected over failover-edge-only refresh and unsubscribe/resubscribe.
|
||||
|
||||
#### Contract
|
||||
|
||||
Add an explicit worker command for updating an active alarm subscription's watch-list. Add an explicit subscription flag that distinguishes these cases when the initial list is empty:
|
||||
|
||||
- alarm-manager-only deployment with no fallback requested;
|
||||
- auto fallback configured, but discovery currently produced no targets.
|
||||
|
||||
The command is gateway-owned even though it travels through the shared `MxCommand` envelope. It returns a normal `MxCommandReply`; invalid mode/state or replacement-subscription failure returns a structured failure without destroying the existing subscription.
|
||||
|
||||
#### Worker
|
||||
|
||||
In auto mode, retain a failover composite even when fallback is enabled with an initially empty list. The composite retains its subscription expression and supports replacing its standby consumer on the STA:
|
||||
|
||||
1. construct the replacement standby;
|
||||
2. subscribe and warm it before altering the current child;
|
||||
3. attach transition forwarding;
|
||||
4. atomically replace the standby reference;
|
||||
5. dispose the previous standby;
|
||||
6. if standby is currently active, keep the mode on subtag and expose the replacement snapshot immediately.
|
||||
|
||||
If any pre-swap step fails, dispose the replacement and keep the old standby untouched. Forced alarm-manager mode rejects refresh as inapplicable. Forced subtag mode may replace its sole consumer through the same build-first/swap-second rule.
|
||||
|
||||
#### Gateway
|
||||
|
||||
Reuse the existing alarm reconciliation cadence; do not add another timer or configuration option. Resolve only when fallback can use subtags. Compare a canonical target fingerprint with the list last acknowledged by the worker, and send an update only after material change or recovery from an empty/failed resolution.
|
||||
|
||||
The failover edge still performs an immediate refresh attempt before publishing its coverage verdict. A successful update changes the recorded sent fingerprint, clears stale-coverage status, and triggers reconciliation. A failed update retains the previous fingerprint and publishes/logs the degraded reason.
|
||||
|
||||
This design repairs the original startup-outage case before failover whenever Galaxy Repository recovers during normal monitor operation, while preserving the warm primary and current provider mode.
|
||||
|
||||
### 6. Windows Gateway Coverage
|
||||
|
||||
**TST-35** extends windev `live` mode to run the net10 gateway suite after the serialized Worker tier and full solution build. The existing host lock remains the load-sensitivity control. Logs record the observed count rather than hard-coding the historical 879 or 1183 count as an invariant.
|
||||
|
||||
### 7. Protobuf Regeneration And Release
|
||||
|
||||
The watch-list command lands in the same regeneration wave as:
|
||||
|
||||
- WRK-32's explicit sequential stale-callback/recency limitation;
|
||||
- IPC-36's replacement of the internal environment-variable name with the public configuration key.
|
||||
|
||||
Regenerate Contracts and all five client trees according to `docs/Contracts.md` and `docs/ClientPackaging.md`, then run the codegen cleanliness checks and every affected suite.
|
||||
|
||||
After integration to `main`, publish .NET/Contracts, Python, Rust, and Go as `0.2.1`, and Java as `0.2.2`. Query each exact coordinate before publishing, never overwrite, create and push the Go module tag at the exact release commit, then query every registry coordinate again and update the packaging record and tracker with immutable release evidence.
|
||||
|
||||
## Verification Strategy
|
||||
|
||||
- macOS: clean `NonWindows.slnx` build and full gateway suite.
|
||||
- Focused gateway tests after each behavioral batch.
|
||||
- All five client build/test commands from `docs/ClientPackaging.md`.
|
||||
- `scripts/check-codegen.ps1`, client-version guard, and code-review README check.
|
||||
- windev: x86 Worker build plus Worker.Tests for worker/proto changes; `live` mode for TST-35 and the final integration gate.
|
||||
- Release: exact registry-coordinate checks and Go module resolution from the pushed tag.
|
||||
|
||||
## Completion Criteria
|
||||
|
||||
The continuation is complete only when all 16 open P2 findings and all three follow-ups are Done or explicitly blocked with evidence, WRK-34 is corrected in the tracker, every required verification is green, client releases are confirmed at their registries, and `archreview/2026-08-19/remediation/00-tracking.md` contains the commit and test evidence for every batch.
|
||||
@@ -0,0 +1,405 @@
|
||||
# Cycle 3 P2 And Follow-Up Implementation Plan
|
||||
|
||||
> **For Codex:** REQUIRED SUB-SKILL: Use `executeplan` to implement this plan task-by-task.
|
||||
|
||||
**Goal:** Close the 16 still-open cycle-3 P2 findings, correct WRK-34's tracker status, implement the three integration-gate follow-ups, and publish the completed client release train.
|
||||
|
||||
**Architecture:** Work proceeds in seven reviewable batches. Small tracker and documentation corrections land first; focused gateway, security, and secondary-contract fixes follow; client parity is completed before a single protobuf regeneration wave adds live alarm watch-list refresh and the remaining public comments; Windows CI is extended before the final windev gate; packages are published only from the integrated `main` release commit.
|
||||
|
||||
**Tech Stack:** .NET 10/C#, .NET Framework 4.8 x86, protobuf/Grpc.Tools, PowerShell 5.1/7, Go, Python/pytest, Rust/Cargo, Java/Gradle, Gitea Actions and package registries.
|
||||
|
||||
---
|
||||
|
||||
## Preconditions
|
||||
|
||||
- Work only in `.worktrees/c3-p2-followups` on `codex/c3-p2-followups`.
|
||||
- Preserve the clean baseline recorded on 2026-08-20/25: `NonWindows.slnx` builds with zero warnings and the gateway suite passes 1183/1183.
|
||||
- Follow red-green-refactor for every behavior change. Documentation-only changes use an explicit failing grep/check as their red gate.
|
||||
- After every batch, update `archreview/2026-08-19/remediation/00-tracking.md` with status, commits, and exact verification output before committing.
|
||||
- Worker/protobuf changes require windev x86 verification before Done.
|
||||
|
||||
### Task 1: Batch H — Tracker And Documentation Currency
|
||||
|
||||
**Findings:** WRK-34 tracker correction, GWC-36, IPC-38, TST-33, TST-34, TST-36
|
||||
|
||||
**Files:**
|
||||
- Modify: `archreview/2026-08-19/remediation/00-tracking.md`
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Sessions/EventSubscriberReplayAttachment.cs`
|
||||
- Modify: `docs/Grpc.md`
|
||||
- Modify: `stillpending.md`
|
||||
- Modify: `.gitea/workflows/ci.yml`
|
||||
- Modify: `REVIEW-PROCESS.md`
|
||||
- Modify: `CLAUDE.md`
|
||||
- Modify: `docs/Sessions.md`
|
||||
- Modify: `docs/DesignDecisions.md`
|
||||
|
||||
**Step 1: Capture the failing currency checks**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
git grep -n '`event` scope' -- CLAUDE.md docs/
|
||||
rg -n 'not safe to rely|Phase 5, not started|6 \[LiveMxAccessFact\]|wire behavior is unverified|still scheduled.*Phase 4' stillpending.md
|
||||
rg -n 'regen-readme.py|check-code-reviews-readme' .gitea/workflows/ci.yml
|
||||
rg -n 'DrainEvents' docs/Grpc.md
|
||||
```
|
||||
|
||||
Expected: stale scope and pending claims are found; CI and `DrainEvents` checks find no intended entry.
|
||||
|
||||
**Step 2: Correct GWC-36 and TST-36**
|
||||
|
||||
Change `OldestAvailableSequence` to document retained-oldest or `highest observed + 1` when an empty ring has a gap, with `0` only when `Gap` is false. Replace prose references to the nonexistent ``event`` scope with ``events:read`` without renaming identifiers such as `GatewayScopes.EventsRead`.
|
||||
|
||||
**Step 3: Correct IPC-38 and TST-33**
|
||||
|
||||
Add a `DrainEvents` command row to `docs/Grpc.md` covering the 10,000-event count cap, byte cap, truncation diagnostic, drain-until-empty contract, and removal from the normal session event stream. Reconcile the five stale `stillpending.md` claims and stamp the file as a historical inventory whose live status comes from archreview trackers.
|
||||
|
||||
**Step 4: Wire TST-34**
|
||||
|
||||
Add this portable-job step next to the existing repository consistency checks:
|
||||
|
||||
```yaml
|
||||
- name: Code-reviews README consistency
|
||||
run: python code-reviews/regen-readme.py --check
|
||||
```
|
||||
|
||||
Qualify the interpreter note in `REVIEW-PROCESS.md`: Windows may require `python`; macOS/Linux normally use `python3`, while CI uses its configured `python`.
|
||||
|
||||
**Step 5: Re-run the checks**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
python3 code-reviews/regen-readme.py --check
|
||||
git grep -n '`event` scope' -- CLAUDE.md docs/ || true
|
||||
rg -n 'not safe to rely|Phase 5, not started|6 \[LiveMxAccessFact\]|wire behavior is unverified|still scheduled.*Phase 4' stillpending.md || true
|
||||
rg -n 'DrainEvents' docs/Grpc.md
|
||||
dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx
|
||||
```
|
||||
|
||||
Expected: README check passes; stale greps return nothing; DrainEvents is documented; build has zero warnings/errors.
|
||||
|
||||
**Step 6: Update tracker and commit**
|
||||
|
||||
Mark the five P2 findings Done and WRK-34 Done (`3679e59`), record verification, then commit:
|
||||
|
||||
```bash
|
||||
git add .gitea/workflows/ci.yml REVIEW-PROCESS.md CLAUDE.md stillpending.md docs/Grpc.md docs/Sessions.md docs/DesignDecisions.md src/ZB.MOM.WW.MxGateway.Server/Sessions/EventSubscriberReplayAttachment.cs archreview/2026-08-19/remediation/00-tracking.md
|
||||
git commit -m "chore(GWC-36,IPC-38,TST-33,TST-34,TST-36): close cycle-3 currency gaps"
|
||||
```
|
||||
|
||||
### Task 2: Batch I — Gateway Correctness And Security
|
||||
|
||||
**Findings:** GWC-33, GWC-35, SEC-38, SEC-39
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs`
|
||||
- Test: `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs`
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Alarms/GatewayAlarmMonitor.cs`
|
||||
- Test: `src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs`
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs`
|
||||
- Test: `src/ZB.MOM.WW.MxGateway.Tests/Security/Authorization/ApiKeyFailureLimiterTests.cs`
|
||||
- Modify: `docs/Authentication.md`
|
||||
- Modify: `archreview/2026-08-19/remediation/00-tracking.md`
|
||||
|
||||
**Step 1: RED for GWC-33**
|
||||
|
||||
Add a worker-client test that stages an event, faults the client without disposing it, and asserts the aggregate queue-depth observable stops counting that client while the faulted object remains alive. Run:
|
||||
|
||||
```bash
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter FullyQualifiedName~WorkerClientTests
|
||||
```
|
||||
|
||||
Expected: the new assertion fails because the registration is disposed only by `DisposeAsync`.
|
||||
|
||||
**Step 2: GREEN for GWC-33**
|
||||
|
||||
Make queue-depth registration disposal idempotent and call it on both terminal transitions and disposal. Do not clear the faulted client's diagnostic state or shorten `FaultedGraceSeconds` retention. Re-run the WorkerClient filter and commit the focused change.
|
||||
|
||||
**Step 3: RED/GREEN for GWC-35**
|
||||
|
||||
Add a monitor test whose first reconcile contains `ActiveAcked` and whose next reconcile carries a newer `Active` instance. Assert one `Raise` repair carrying the incoming instance identity. Verify red, then generalize the both-present state delta:
|
||||
|
||||
```csharp
|
||||
AlarmTransitionKind kind = incoming.Value.CurrentState == AlarmConditionState.ActiveAcked
|
||||
? AlarmTransitionKind.Acknowledge
|
||||
: AlarmTransitionKind.Raise;
|
||||
```
|
||||
|
||||
Keep the existing duplicate suppression keyed by original raise timestamp. Re-run `GatewayAlarmMonitorAttachOrderTests`.
|
||||
|
||||
**Step 4: RED/GREEN for SEC-38**
|
||||
|
||||
Add a limiter test that fills all 32 peer key-ID partitions, then attacks one valid victim key ID through the fallback from multiple peers. Assert the victim aggregate trips. Change resolution/recording so every validly-shaped presented key ID consults and feeds its aggregate even if its peer partition fell back; continue using the resolved partition key for the peer layer and the existing capped aggregate eviction for cardinality. Update class remarks and `docs/Authentication.md`.
|
||||
|
||||
**Step 5: RED/GREEN for SEC-39**
|
||||
|
||||
Add a test that records fallback failures, calls `Reset` with a null effective key ID, and asserts the fallback remains tracked. Tighten ownership:
|
||||
|
||||
```csharp
|
||||
if (effectiveKeyId is not null
|
||||
&& string.Equals(effectiveKeyId, partition.KeyId, StringComparison.Ordinal))
|
||||
{
|
||||
RemovePartition(partitionKey);
|
||||
}
|
||||
```
|
||||
|
||||
**Step 6: Verify, track, and commit**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~WorkerClientTests|FullyQualifiedName~GatewayAlarmMonitorAttachOrderTests|FullyQualifiedName~ApiKeyFailureLimiterTests"
|
||||
dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx
|
||||
```
|
||||
|
||||
Expected: all filtered tests pass; zero warnings/errors. Update tracker with per-finding commits and counts.
|
||||
|
||||
### Task 3: Batch J — Secondary Contract Guards
|
||||
|
||||
**Findings:** IPC-36, IPC-37
|
||||
|
||||
**Files:**
|
||||
- Create: `src/ZB.MOM.WW.MxGateway.Contracts/WorkerLaunchEnvironment.cs`
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerProcessLauncher.cs`
|
||||
- Modify: worker consumers under `src/ZB.MOM.WW.MxGateway.Worker/Bootstrap`, `Ipc`, and `MxAccess`
|
||||
- Test: `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerProcessLauncherTests.cs`
|
||||
- Create: `src/ZB.MOM.WW.MxGateway.Tests/Contracts/GalaxyContractParityTests.cs`
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj`
|
||||
- Modify: `docs/WorkerProcessLauncher.md`
|
||||
- Modify: `docs/MxAccessWorkerInstanceDesign.md`
|
||||
- Modify: `docs/Contracts.md`
|
||||
- Modify: `archreview/2026-08-19/remediation/00-tracking.md`
|
||||
|
||||
**Step 1: RED for IPC-36**
|
||||
|
||||
Change the launcher test to enumerate the six constants expected on `WorkerLaunchEnvironment` and assert each is present in `ProcessStartInfo.Environment`. Run the test and confirm compilation fails because the shared class does not exist.
|
||||
|
||||
**Step 2: GREEN for IPC-36**
|
||||
|
||||
Create a net48-safe static class containing six `public const string` names. Replace both gateway and worker literals with these constants; remove the duplicate public constants from `WorkerProcessLauncher` unless compatibility requires obsolete aliases. Add a launch-environment table with variable, source configuration key, and worker default. Refresh Write/Write2 wording.
|
||||
|
||||
**Step 3: RED for IPC-37**
|
||||
|
||||
Add `GalaxyContractParityTests` that converts both loaded descriptors with `ToProto()`, clears `SourceCodeInfo`, normalizes `Options.CsharpNamespace` and file `Name`, then compares deterministic bytes. Before the assertion, mutate a clone in a local test helper to prove the comparison reports the differing symbol/message clearly; the real descriptor assertion must pass at HEAD.
|
||||
|
||||
**Step 4: Document and verify**
|
||||
|
||||
Record the parity guard in the Contracts project comment and `docs/Contracts.md`. Run:
|
||||
|
||||
```bash
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~WorkerProcessLauncherTests|FullyQualifiedName~GalaxyContractParityTests"
|
||||
dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx
|
||||
```
|
||||
|
||||
Then update tracker and commit IPC-36/37.
|
||||
|
||||
### Task 4: Batch K — Client Alarm-Feed Parity
|
||||
|
||||
**Findings:** CLI-47, CLI-48, CLI-49, CLI-50
|
||||
|
||||
**Files:**
|
||||
- Modify: `clients/dotnet/ZB.MOM.WW.MxGateway.Client.Cli/MxGatewayClientCli.cs`
|
||||
- Test: `clients/dotnet/ZB.MOM.WW.MxGateway.Client.Tests/MxGatewayClientCliTests.cs`
|
||||
- Test/modify: `clients/go/cmd/mxgw-go/main.go`, `main_test.go`
|
||||
- Modify: `clients/go/mxgateway/types.go`
|
||||
- Test/modify: Java CLI and `MxGatewayCliTests.java`
|
||||
- Modify/test: `clients/python/src/zb_mom_ww_mxgateway_cli/commands.py`, `clients/python/tests/`
|
||||
- Modify: all affected client READMEs and `docs/CrossLanguageSmokeMatrix.md`
|
||||
- Modify: `archreview/2026-08-19/remediation/00-tracking.md`
|
||||
|
||||
**Step 1: RED/GREEN for CLI-47 and CLI-48**
|
||||
|
||||
In .NET, Go, and Java tests, feed `provider_status { degraded: false }` and `snapshot_status { truncated: false }`; assert exact text rows and protobuf-JSON behavior. Confirm .NET text fails as `{}` and Go lacks a stream-alarm harness. Hand-format .NET text to match the explicit rows; add the missing Go and Java coverage without changing their correct output.
|
||||
|
||||
**Step 2: RED/GREEN for CLI-49**
|
||||
|
||||
Add a Go compile-time consumer test that declares variables of `mxgateway.AlarmProviderStatus` and `mxgateway.AlarmSnapshotStatus`. Confirm red, then add aliases and update the five-arm `AlarmFeedMessage` comment.
|
||||
|
||||
**Step 3: RED/GREEN for CLI-50**
|
||||
|
||||
Add Python CLI tests proving default `stream-alarms` emits provider status, snapshot status, cached alarms, and later live messages incrementally until timeout/cancellation; add a finite `--max-messages` test. Change the default to `0` (unbounded) and print rows as consumed instead of collecting first. Keep the maximum bound for positive finite values and ensure iterator cleanup occurs in `finally`.
|
||||
|
||||
**Step 4: Verify all clients**
|
||||
|
||||
Run the authoritative commands from `docs/ClientPackaging.md`:
|
||||
|
||||
```bash
|
||||
dotnet test clients/dotnet/ZB.MOM.WW.MxGateway.Client.slnx
|
||||
(cd clients/go && go test ./...)
|
||||
(cd clients/python && python -m pytest)
|
||||
(cd clients/rust && cargo test --workspace)
|
||||
(cd clients/java && JAVA_HOME=/opt/homebrew/opt/openjdk@17 gradle test)
|
||||
```
|
||||
|
||||
Revert only proven generator churn; do not revert source changes. Update tracker and commit CLI-47–50.
|
||||
|
||||
### Task 5: Batch L — Live Watch-List Refresh And Protobuf Wave
|
||||
|
||||
**Follow-ups:** complete GWC-37 application path, WRK-32 public comment
|
||||
|
||||
**Files:**
|
||||
- Modify: `src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_gateway.proto`
|
||||
- Regenerate: Contracts and every client generated tree required by `docs/Contracts.md`
|
||||
- Modify/test: worker alarm command executor, handler, failover/subtag consumers, and Worker.Tests
|
||||
- Modify/test: `src/ZB.MOM.WW.MxGateway.Server/Alarms/GatewayAlarmMonitor.cs` and alarm monitor tests
|
||||
- Modify: alarm, worker protocol, configuration, and client contract documentation
|
||||
- Modify: `archreview/2026-08-19/remediation/00-tracking.md`
|
||||
|
||||
**Step 1: RED — contract round trip**
|
||||
|
||||
Add a protobuf round-trip test for:
|
||||
|
||||
```proto
|
||||
message UpdateAlarmWatchListCommand {
|
||||
repeated AlarmSubtagTarget watch_list = 1;
|
||||
}
|
||||
```
|
||||
|
||||
Add a new non-conflicting `MxCommand` oneof field and `MxCommandKind`, plus an explicit `fallback_enabled` field on `SubscribeAlarmsCommand`. Confirm the test cannot compile before editing the proto.
|
||||
|
||||
**Step 2: RED — worker routing and failure preservation**
|
||||
|
||||
Add Worker.Tests covering command routing, rejection without an active/fallback-capable subscription, replacement setup failure preserving the previous standby, primary-mode replacement keeping primary active, and standby-mode replacement keeping subtag active.
|
||||
|
||||
**Step 3: GREEN — worker hot-swap**
|
||||
|
||||
Implement `IAlarmCommandHandler.UpdateWatchList`. Retain the subscription expression. Refactor `FailoverAlarmConsumer` so standby replacement performs build/subscribe/attach before swap and disposes the old standby after swap. No COM operation may leave the STA. In auto mode, build the composite with an empty standby when `fallback_enabled` is true; leave an intentionally alarm-manager-only subscription unchanged.
|
||||
|
||||
**Step 4: RED — gateway periodic update**
|
||||
|
||||
Extend `GatewayAlarmMonitorProviderModeTests`/`AlarmFailoverEndToEndTests` with these scenarios:
|
||||
|
||||
- startup discovery fails/empty, later reconcile resolves targets, exactly one update command is sent;
|
||||
- unchanged targets send no update;
|
||||
- changed targets update the sent fingerprint only after an OK reply;
|
||||
- worker rejection retains the prior fingerprint and degraded coverage reason;
|
||||
- failover edge performs an immediate refresh before publishing status.
|
||||
|
||||
**Step 5: GREEN — gateway refresh path**
|
||||
|
||||
Reuse `ReconcileIntervalSeconds`. Resolve only when mode can use subtags, compare canonical full-target fingerprints, invoke the new command on material change/recovery, and reconcile after success. Do not add a timer or option.
|
||||
|
||||
**Step 6: Complete the shared proto comments**
|
||||
|
||||
In the same proto edit:
|
||||
|
||||
- replace the IPC-36 internal environment-variable name with `MxGateway:Worker:WriteCompletionWaitMilliseconds`;
|
||||
- describe WRK-32 correlation as recency-based per handle and name the sequential timed-out-predecessor stale-callback case, not only concurrent writes.
|
||||
|
||||
**Step 7: Regenerate once and run portable checks**
|
||||
|
||||
Follow `docs/Contracts.md` exactly, then run:
|
||||
|
||||
```bash
|
||||
pwsh scripts/check-codegen.ps1
|
||||
dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~ProtobufContractRoundTripTests|FullyQualifiedName~AlarmFailoverEndToEndTests|FullyQualifiedName~GatewayAlarmMonitorProviderModeTests|FullyQualifiedName~ClientProtoInputTests"
|
||||
```
|
||||
|
||||
Run all five client suites again because generated trees changed.
|
||||
|
||||
**Step 8: Verify worker on windev**
|
||||
|
||||
Commit the candidate, then run the repository's windev test mode at that SHA. Required evidence: x86 Worker build success and Worker.Tests with zero failures. Only then mark the watch-list/WRK-32 follow-ups Done in the tracker and commit any evidence-only tracker update.
|
||||
|
||||
### Task 6: Batch M — Windows Gateway Suite In Nightly CI
|
||||
|
||||
**Finding:** TST-35
|
||||
|
||||
**Files:**
|
||||
- Modify: `scripts/ci/windev-worker-ci.ps1`
|
||||
- Modify: `.gitea/workflows/ci.yml` comments if needed
|
||||
- Modify: `docs/GatewayTesting.md`
|
||||
- Modify: `scripts/ci/README.md`
|
||||
- Modify: `archreview/2026-08-19/remediation/00-tracking.md`
|
||||
|
||||
**Step 1: Add an acceptance check before behavior**
|
||||
|
||||
Document the expected `live` order and add a script-level/static assertion or dry-run seam showing gateway tests are absent before the change.
|
||||
|
||||
**Step 2: Extend live mode**
|
||||
|
||||
After Worker.Tests and full-slnx build, run:
|
||||
|
||||
```powershell
|
||||
Invoke-Native 'gateway tests on Windows' {
|
||||
dotnet test 'src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj' -c Release --no-build
|
||||
}
|
||||
```
|
||||
|
||||
Keep it before the opt-in live-MXAccess smoke, under the existing host lock. Log the actual result; do not enforce a historical count.
|
||||
|
||||
**Step 3: Verify on windev**
|
||||
|
||||
Run `live` mode at the committed SHA. Require Worker.Tests, gateway tests, full solution build, and live smoke to complete successfully. Update docs/tracker with date, SHA, counts, and duration; commit TST-35.
|
||||
|
||||
### Task 7: Batch N — Full Integration And Release
|
||||
|
||||
**Follow-up:** publish the five clients and Contracts
|
||||
|
||||
**Files:**
|
||||
- Modify after publication: `docs/ClientPackaging.md`
|
||||
- Modify: `archreview/2026-08-19/remediation/00-tracking.md`
|
||||
|
||||
**Step 1: Full local integration gate**
|
||||
|
||||
Run:
|
||||
|
||||
```bash
|
||||
git diff --check
|
||||
pwsh scripts/check-codegen.ps1
|
||||
pwsh scripts/check-client-versions.ps1
|
||||
python3 code-reviews/regen-readme.py --check
|
||||
dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx
|
||||
dotnet test src/ZB.MOM.WW.MxGateway.NonWindows.slnx --no-build --no-restore
|
||||
```
|
||||
|
||||
Run all five client suites. Require zero unexpected working-tree changes.
|
||||
|
||||
**Step 2: Final windev gate**
|
||||
|
||||
Run `windev-worker-ci.ps1 -Mode live` at the exact candidate SHA. Record Worker.Tests, gateway tests, live smoke, and full solution build evidence.
|
||||
|
||||
**Step 3: Integrate before release**
|
||||
|
||||
Use `superpowers:finishing-a-development-branch`. Merge/push the verified branch to `main`, confirm CI green, and resolve the exact release commit SHA. Do not publish artifacts from an unmerged feature commit.
|
||||
|
||||
**Step 4: Preflight exact coordinates**
|
||||
|
||||
Confirm these are absent (HTTP 404) immediately before publishing:
|
||||
|
||||
- NuGet `ZB.MOM.WW.MxGateway.Client` 0.2.1
|
||||
- NuGet `ZB.MOM.WW.MxGateway.Contracts` 0.2.1
|
||||
- PyPI `zb-mom-ww-mxaccess-gateway-client` 0.2.1
|
||||
- Cargo `zb-mom-ww-mxgateway-client` 0.2.1
|
||||
- Maven `com.zb.mom.ww.mxgateway:zb-mom-ww-mxgateway-client` 0.2.2
|
||||
- Git tag `clients/go/v0.2.1`
|
||||
|
||||
Abort rather than overwrite if any coordinate exists unexpectedly.
|
||||
|
||||
**Step 5: Publish from clean `main`**
|
||||
|
||||
Run the documented `scripts/pack-clients.ps1 -Publish` commands for .NET, Python, Rust, and Java with required credentials, then:
|
||||
|
||||
```powershell
|
||||
pwsh scripts/tag-go-module.ps1 -Version v0.2.1 -Push
|
||||
```
|
||||
|
||||
The Go tag must point to the exact integrated release commit.
|
||||
|
||||
**Step 6: Verify release and close tracker**
|
||||
|
||||
Query every exact registry coordinate for HTTP 200 and resolve the Go module/tag remotely. Update `docs/ClientPackaging.md` and the tracker with release SHA, coordinates, and verification date. Run the doc checks, commit the release record, and push it.
|
||||
|
||||
## Final Tasklist Exit Gate
|
||||
|
||||
- [ ] All 16 open P2 findings are Done with commit/test evidence.
|
||||
- [ ] WRK-34 is corrected to Done (`3679e59`).
|
||||
- [ ] Live watch-list refresh passes portable and windev tests.
|
||||
- [ ] WRK-32 public proto comment is regenerated through every client surface.
|
||||
- [ ] TST-35 nightly Windows gateway suite is proven on windev.
|
||||
- [ ] Full NonWindows gateway suite and all five client suites pass.
|
||||
- [ ] Worker.Tests and Windows gateway suite pass at the integrated release SHA.
|
||||
- [ ] Five client releases plus Contracts are confirmed in registries; Go tag resolves.
|
||||
- [ ] `archreview/2026-08-19/remediation/00-tracking.md` contains no stale Deferred row for completed work.
|
||||
+43
-4
@@ -227,6 +227,15 @@ ack. If the attribute has no writable ack-comment subtag configured, the RPC
|
||||
returns `FailedPrecondition`. In alarm-manager mode, `AlarmAckByName` is
|
||||
used as before.
|
||||
|
||||
**Acceptance is not application:** the ack is forwarded to the provider, and a
|
||||
successful return means the provider accepted the call, not that the
|
||||
acknowledgement was applied. On galaxies whose alarmed attributes carry a
|
||||
non-free-access security classification, `AlarmAckByName` returns `rc=0` and the
|
||||
alarm stays `UNACK_ALM` — it carries an operator name, not an authenticated
|
||||
identity. (Observed on the probe rig; the mechanism is inferred.) Confirm an ack
|
||||
by the resulting transition, never by the return code. See
|
||||
`docs/AlarmProbeFindings.md`.
|
||||
|
||||
**Degraded state visibility:** every subtag-mode transition carries
|
||||
`degraded = true` and `source_provider = ALARM_PROVIDER_MODE_SUBTAG` on the
|
||||
`OnAlarmTransitionEvent` and `ActiveAlarmSnapshot` proto fields. The
|
||||
@@ -240,6 +249,22 @@ monitoring (forced)") when subtag mode is the configured `Fallback:Mode=ForceSub
|
||||
as a fault. Metrics: `mxgateway.alarms.provider_mode` gauge (1 = alarmmgr,
|
||||
2 = subtag) and `mxgateway.alarms.provider_switches` counter.
|
||||
|
||||
**Standby-coverage visibility:** `degraded` also covers the case where the alarm
|
||||
manager is serving normally but the subtag standby behind it could not cover
|
||||
anything — a watch-list that resolved to zero targets, or Galaxy Repository
|
||||
discovery that was enabled and failed. That condition used to be a single startup
|
||||
log line, invisible until the failover that needed the standby found it empty; it
|
||||
is now a standing `AlarmProviderStatus` reason
|
||||
(`AlarmProviderReasons.SubtagWatchListEmpty` / `SubtagDiscoveryFailed`) carried on
|
||||
the feed and rendered as the amber `Alarm Manager (no subtag standby)` badge, which
|
||||
keeps the failover label from naming the wrong live provider. The watch-list the
|
||||
worker holds is fixed for the life of the subscription, so on the failover edge the
|
||||
monitor re-resolves and reports a difference
|
||||
(`AlarmProviderReasons.SubtagWatchListStale`, with both counts) rather than
|
||||
re-sending it — re-sending would rebuild the failover consumer on the primary that
|
||||
just failed. A monitor restart re-resolves and re-sends. See
|
||||
`docs/AlarmClientDiscovery.md`.
|
||||
|
||||
**Truncated-snapshot visibility:** `GetXmlCurrentAlarms2` caps its reply at
|
||||
`MxGateway:Alarms:MaxAlarmsPerFetch` and offers no confirmed "more available"
|
||||
flag, so a reply holding exactly the cap is treated as truncated. On such a
|
||||
@@ -558,9 +583,14 @@ Optional diagnostics:
|
||||
requests above the shared ceiling `GatewayContractInfo.MaxDrainEventsPerCommand`
|
||||
and the worker clamps to the same value, treating `max_events = 0` as "the default
|
||||
cap". By **bytes**: the worker sizes the reply while draining, against the
|
||||
negotiated worker-frame maximum, so a byte-heavy queue truncates instead of
|
||||
overshooting and events that do not fit stay queued. Truncation is reported in the
|
||||
reply's `DiagnosticMessage`; callers drain iteratively until an empty reply.
|
||||
negotiated worker-frame maximum less two envelope reserves, so a byte-heavy queue
|
||||
truncates instead of overshooting and events that do not fit stay queued. The
|
||||
reserve is subtracted twice because the reply has two boundaries to clear — the
|
||||
pipe frame, then the public gRPC send cap it is returned through by `Invoke` — and
|
||||
the validator keeps `Worker:MaxMessageBytes` within two reserves of
|
||||
`Protocol:MaxGrpcMessageBytes` so that budget always fits both. Truncation is
|
||||
reported in the reply's `DiagnosticMessage`; callers drain iteratively until an
|
||||
empty reply.
|
||||
- `ShutdownWorker`
|
||||
|
||||
Do not compress MXAccess semantics into generic verbs too early. A command enum
|
||||
@@ -903,10 +933,19 @@ Timeouts:
|
||||
|
||||
Cancellation:
|
||||
|
||||
- canceling the gRPC call should stop waiting in the gateway,
|
||||
- canceling the gRPC call stops the gateway waiting **and** forwards a
|
||||
`WorkerCancel` for the abandoned correlation, the same best-effort frame the
|
||||
command timeout sends — a client deadline or a disconnect is the common
|
||||
abandonment path, and leaving the correlation on the worker's single STA queue
|
||||
makes live callers queue behind work nobody is waiting for,
|
||||
- the cancel is sent only by the caller that owned the pending-entry removal (so
|
||||
a reply that won the race is never chased) and is skipped once the client is
|
||||
stopping, where every pending command fails anyway,
|
||||
- it cannot safely abort an in-flight COM call on the STA,
|
||||
- the worker should finish the COM call and discard or log the late reply if
|
||||
the correlation was canceled,
|
||||
- caller cancellation is reported as `CallerCancelled` (mapped to gRPC
|
||||
`Cancelled`), distinct from a gateway shutdown, so metrics name the real cause,
|
||||
- hard cancellation means killing the worker process.
|
||||
|
||||
## Event Delivery And Backpressure
|
||||
|
||||
@@ -52,31 +52,46 @@ LmxOpcUa write rights doesn't need a second account for the gw.
|
||||
|
||||
| Group | GID | DN | LmxOpcUa meaning | Suggested mxgw mapping |
|
||||
|---|---|---|---|---|
|
||||
| ReadOnly | 5501 | `ou=ReadOnly,ou=groups,dc=zb,dc=local` | Browse + read OPC UA nodes | `Browse` + `Subscribe` (read paths only) |
|
||||
| WriteOperate | 5502 | `ou=WriteOperate,ou=groups,dc=zb,dc=local` | Write FreeAccess / Operate attrs | `Write` (plain) |
|
||||
| WriteTune | 5504 | `ou=WriteTune,ou=groups,dc=zb,dc=local` | Write Tune attrs | `WriteSecured` (Tune only) |
|
||||
| WriteConfigure | 5505 | `ou=WriteConfigure,ou=groups,dc=zb,dc=local` | Write Configure attrs | `WriteSecured` (Configure) |
|
||||
| AlarmAck | 5503 | `ou=AlarmAck,ou=groups,dc=zb,dc=local` | Acknowledge alarms | gw alarm-ack RPC, when added |
|
||||
| ReadOnly | 5601 | `ou=ReadOnly,ou=groups,dc=zb,dc=local` | Browse + read OPC UA nodes | `Browse` + `Subscribe` (read paths only) |
|
||||
| WriteOperate | 5602 | `ou=WriteOperate,ou=groups,dc=zb,dc=local` | Write FreeAccess / Operate attrs | `Write` (plain) |
|
||||
| WriteTune | 5603 | `ou=WriteTune,ou=groups,dc=zb,dc=local` | Write Tune attrs | `WriteSecured` (Tune only) |
|
||||
| WriteConfigure | 5604 | `ou=WriteConfigure,ou=groups,dc=zb,dc=local` | Write Configure attrs | `WriteSecured` (Configure) |
|
||||
| AlarmAck | 5605 | `ou=AlarmAck,ou=groups,dc=zb,dc=local` | Acknowledge alarms | gw alarm-ack RPC, when added |
|
||||
|
||||
These are the shared config's OPC-perm (`560x`) family — see
|
||||
`scadaproj/infra/glauth/README.md`'s directory-layout table, which
|
||||
partitions groups into non-overlapping gid ranges per sister app
|
||||
(`SCADA-*` 55xx, OPC-perm 560x, `Gw*` 561x, `OtOpcUa-*` 57xx) so the
|
||||
three apps coexist in one directory.
|
||||
|
||||
**A user can be in multiple groups** — `othergroups = [...]` in the
|
||||
config is a list. `admin` is the canonical example (in every role
|
||||
group below).
|
||||
config is a list. `multi-role` is the canonical example (in every
|
||||
group in the file, not just the ones below).
|
||||
|
||||
## Pre-provisioned users
|
||||
|
||||
Per-role OPC-perm testers are named `opc-*` rather than the bare role name
|
||||
(`readonly`, `writeop`, …) because a tester username must not
|
||||
case-collide with a group name — see the naming rule in
|
||||
`scadaproj/infra/glauth/README.md`.
|
||||
|
||||
| Username | Password | UID | Primary group | Other groups | Capabilities |
|
||||
|---|---|---|---|---|---|
|
||||
| `readonly` | `readonly123` | 5001 | ReadOnly | — | Browse, read |
|
||||
| `writeop` | `writeop123` | 5002 | WriteOperate | — | + plain Write |
|
||||
| `writetune` | `writetune123` | 5005 | WriteTune | — | + WriteSecured (Tune) |
|
||||
| `writeconfig` | `writeconfig123` | 5006 | WriteConfigure | — | + WriteSecured (Configure) |
|
||||
| `alarmack` | `alarmack123` | 5003 | AlarmAck | — | Alarm acknowledgment |
|
||||
| `admin` | `admin123` | 5004 | ReadOnly | WriteOperate, AlarmAck, WriteTune, WriteConfigure | All roles |
|
||||
| `opc-readonly` | `password` | 5101 | ReadOnly | — | Browse, read |
|
||||
| `opc-writeop` | `password` | 5102 | WriteOperate | — | + plain Write |
|
||||
| `opc-writetune` | `password` | 5103 | WriteTune | — | + WriteSecured (Tune) |
|
||||
| `opc-writeconfig` | `password` | 5104 | WriteConfigure | — | + WriteSecured (Configure) |
|
||||
| `opc-alarmack` | `password` | 5105 | AlarmAck | — | Alarm acknowledgment |
|
||||
| `multi-role` | `password` | 5005 | SCADA-Admins | every other group in the file, incl. all five OPC-perm groups + `GwAdmin`/`GwReader` | All roles, all apps (cross-app canonical login) |
|
||||
| `admin` | `password` | 5001 | SCADA-Admins | `GwAdmin`, `OtOpcUa-Admins` | Dashboard Administrator + ScadaBridge/OtOpcUa admin — **not** a member of any OPC-perm group, so it has no gw-side Write/WriteSecured/AlarmAck capability |
|
||||
| `serviceaccount` | `<service-account-password>` | 5999 | ReadOnly | — | LDAP search capability (for bind-then-search) |
|
||||
|
||||
For mxaccessgw dev, `admin` covers every gw-side capability test;
|
||||
`readonly` is the right "negative" case for proving Browse-OK /
|
||||
Write-denied.
|
||||
For mxaccessgw dev, `multi-role` covers every gw-side capability test
|
||||
(it is a member of all five OPC-perm groups as well as `GwAdmin`);
|
||||
`opc-readonly` is the right "negative" case for proving Browse-OK /
|
||||
Write-denied. `admin` is useful for dashboard-Administrator and
|
||||
ScadaBridge/OtOpcUa-admin testing but, per the table above, carries no
|
||||
OPC-perm group membership of its own.
|
||||
|
||||
The gateway dashboard uses two gateway-specific groups beyond the LmxOpcUa taxonomy:
|
||||
`GwAdmin` (gid 5610 → role `Administrator`) and `GwReader` (gid 5611 → role `Viewer`).
|
||||
@@ -121,7 +136,7 @@ principal cannot show that.
|
||||
|
||||
```
|
||||
DN: cn=admin,dc=zb,dc=local
|
||||
Password: admin123
|
||||
Password: password
|
||||
```
|
||||
|
||||
Construct the DN from the username; bind. Works on GLAuth because
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env pwsh
|
||||
<#
|
||||
.SYNOPSIS
|
||||
Fails a PR that changes a client's source without bumping its version
|
||||
past the last thing actually published (CLI-46).
|
||||
|
||||
.DESCRIPTION
|
||||
CLI-39 (2026-07-12 cycle) added publish-time guards that refuse to
|
||||
re-publish an already-published name+version
|
||||
(scripts/pack-clients.ps1's Assert-GiteaPackageNotPublished,
|
||||
scripts/tag-go-module.ps1's ClientVersion-vs-tag check). Those guards
|
||||
only fire when someone actually runs a publish. CLI-46 (2026-08-19
|
||||
cycle) found the same hazard recurring silently in between publishes:
|
||||
library source (including a real bug fix, commit a8f86b5) shipped
|
||||
while the version constant still equalled the already-published
|
||||
number, so nothing in the tree recorded that the published artifact
|
||||
and the in-repo source had diverged. This script is the PR-time half
|
||||
of that guard: catch the drift as soon as it happens, not at the next
|
||||
publish.
|
||||
|
||||
For each client this repo ships, the script:
|
||||
1. Finds the latest "publish tag" for that client, per the tag
|
||||
naming convention its own publish tooling actually uses.
|
||||
2. If no such tag exists, says so and passes (nothing to compare
|
||||
against yet — this is expected for every client except Go; see
|
||||
below).
|
||||
3. If a tag exists, reads the version recorded in-source at that tag
|
||||
and the version recorded in-source now. If `clients/<lang>` has
|
||||
changed since the tag AND the version constant is unchanged, that
|
||||
is exactly the CLI-46 pattern: source moved, the published
|
||||
coordinate did not, and nothing says so. Fail.
|
||||
|
||||
Tag conventions, as implemented today (verified against
|
||||
scripts/pack-clients.ps1, scripts/tag-go-module.ps1, and `git tag -l`):
|
||||
- Go is the only client with a git tag at all: `tag-go-module.ps1`
|
||||
creates `clients/go/vX.Y.Z` (CLI-21/CLI-39 already keep this tag's
|
||||
version in lockstep with clients/go/mxgateway/version.go's
|
||||
ClientVersion at *tag time*; this script is what keeps it in
|
||||
lockstep *between* tags).
|
||||
- .NET, Python, Rust, and Java publish straight to a package
|
||||
registry via `pack-clients.ps1 -Publish` (NuGet/PyPI/cargo/Maven
|
||||
API pushes) with no corresponding git tag of any kind — there is
|
||||
nothing in this repo's history to diff against for those four.
|
||||
The script looks for a same-shaped tag anyway
|
||||
(`clients/<lang>/v*`) so it stays correct if that ever changes,
|
||||
but today it will report "no tags found" and pass for all four.
|
||||
|
||||
A client with no tag is not a violation of anything — it means no
|
||||
release train has ever tagged that client, so there is no "since the
|
||||
last publish" window to check. The check exists to catch drift once a
|
||||
baseline exists, not to demand one.
|
||||
|
||||
.EXAMPLE
|
||||
pwsh scripts/check-client-versions.ps1
|
||||
#>
|
||||
|
||||
[CmdletBinding()]
|
||||
param()
|
||||
|
||||
Set-StrictMode -Version Latest
|
||||
$ErrorActionPreference = 'Stop'
|
||||
|
||||
$repoRoot = (Resolve-Path (Join-Path $PSScriptRoot '..')).Path
|
||||
Push-Location $repoRoot
|
||||
try {
|
||||
# One entry per official client. `TagGlob` is the tag naming pattern that
|
||||
# client's own publish tooling uses (see doc comment above); `VersionFile`
|
||||
# + `VersionPattern` locate the single version-of-record constant
|
||||
# docs/ClientPackaging.md's "Versioning" section names as authoritative
|
||||
# for that client; `SourceDir` is the diff scope (matches the literal
|
||||
# `clients/<lang>` scope described in the CLI-46 remediation task).
|
||||
$clients = @(
|
||||
[ordered]@{
|
||||
Name = 'dotnet'
|
||||
SourceDir = 'clients/dotnet'
|
||||
TagGlob = 'clients/dotnet/v*'
|
||||
VersionFile = 'clients/dotnet/ZB.MOM.WW.MxGateway.Client/ZB.MOM.WW.MxGateway.Client.csproj'
|
||||
VersionPattern = '<Version>([^<]+)</Version>'
|
||||
},
|
||||
[ordered]@{
|
||||
Name = 'go'
|
||||
SourceDir = 'clients/go'
|
||||
TagGlob = 'clients/go/v*'
|
||||
VersionFile = 'clients/go/mxgateway/version.go'
|
||||
VersionPattern = 'ClientVersion\s*=\s*"([^"]+)"'
|
||||
},
|
||||
[ordered]@{
|
||||
Name = 'python'
|
||||
SourceDir = 'clients/python'
|
||||
TagGlob = 'clients/python/v*'
|
||||
VersionFile = 'clients/python/src/zb_mom_ww_mxgateway/version.py'
|
||||
VersionPattern = '__version__\s*=\s*"([^"]+)"'
|
||||
},
|
||||
[ordered]@{
|
||||
Name = 'rust'
|
||||
SourceDir = 'clients/rust'
|
||||
TagGlob = 'clients/rust/v*'
|
||||
VersionFile = 'clients/rust/Cargo.toml'
|
||||
VersionPattern = 'version\s*=\s*"([^"]+)"'
|
||||
},
|
||||
[ordered]@{
|
||||
Name = 'java'
|
||||
SourceDir = 'clients/java'
|
||||
TagGlob = 'clients/java/v*'
|
||||
VersionFile = 'clients/java/build.gradle'
|
||||
VersionPattern = "version\s*=\s*'([^']+)'"
|
||||
}
|
||||
)
|
||||
|
||||
function Get-LatestTag {
|
||||
param([string]$Glob)
|
||||
|
||||
# --sort=-v:refname orders by version-aware descending semver, so the
|
||||
# first line is the newest tag matching the glob. Empty result means
|
||||
# no tag of this shape has ever been created for this client.
|
||||
$tags = git tag -l --sort='-v:refname' $Glob
|
||||
if (-not $tags) {
|
||||
return $null
|
||||
}
|
||||
return @($tags -split "`n")[0].Trim()
|
||||
}
|
||||
|
||||
function Get-VersionFromFile {
|
||||
param([string]$Path, [string]$Pattern)
|
||||
|
||||
if (-not (Test-Path $Path)) {
|
||||
throw "Version file not found: $Path"
|
||||
}
|
||||
$content = Get-Content -Path $Path -Raw
|
||||
$match = [regex]::Match($content, $Pattern)
|
||||
if (-not $match.Success) {
|
||||
throw "Could not find a version matching /$Pattern/ in $Path"
|
||||
}
|
||||
return $match.Groups[1].Value
|
||||
}
|
||||
|
||||
function Get-VersionFromFileAtRevision {
|
||||
param([string]$Revision, [string]$Path, [string]$Pattern)
|
||||
|
||||
$content = git show "${Revision}:${Path}" 2>$null
|
||||
if ($LASTEXITCODE -ne 0 -or -not $content) {
|
||||
# The version file did not exist (or was at a different path) at
|
||||
# that tag. Treat as "unknown" rather than failing the whole
|
||||
# check — the drift comparison below only needs "does the
|
||||
# current version still equal the tag's version", and an
|
||||
# unresolvable historical version can never equal the current
|
||||
# one, so this degrades to "assume it changed" (safe: it will
|
||||
# not mask a real drift).
|
||||
return $null
|
||||
}
|
||||
$match = [regex]::Match(($content -join "`n"), $Pattern)
|
||||
if (-not $match.Success) {
|
||||
return $null
|
||||
}
|
||||
return $match.Groups[1].Value
|
||||
}
|
||||
|
||||
$failures = New-Object System.Collections.Generic.List[string]
|
||||
|
||||
foreach ($client in $clients) {
|
||||
Write-Host ''
|
||||
Write-Host "== $($client.Name) ==" -ForegroundColor Cyan
|
||||
|
||||
$tag = Get-LatestTag -Glob $client.TagGlob
|
||||
if (-not $tag) {
|
||||
Write-Host " No publish tags matching '$($client.TagGlob)' — nothing to compare against yet. Pass." -ForegroundColor Yellow
|
||||
continue
|
||||
}
|
||||
|
||||
Write-Host " Latest publish tag: $tag"
|
||||
|
||||
$currentVersion = Get-VersionFromFile -Path $client.VersionFile -Pattern $client.VersionPattern
|
||||
Write-Host " Current version constant ($($client.VersionFile)): $currentVersion"
|
||||
|
||||
# Any diff at all in the client's source tree since its last publish
|
||||
# tag — matches the CLI-46 remediation task's literal check.
|
||||
git diff --quiet $tag -- $client.SourceDir
|
||||
$sourceChanged = ($LASTEXITCODE -ne 0)
|
||||
|
||||
if (-not $sourceChanged) {
|
||||
Write-Host " No changes under $($client.SourceDir) since $tag. Pass." -ForegroundColor Green
|
||||
continue
|
||||
}
|
||||
|
||||
$tagVersion = Get-VersionFromFileAtRevision -Revision $tag -Path $client.VersionFile -Pattern $client.VersionPattern
|
||||
Write-Host " Source under $($client.SourceDir) changed since $tag (tag's version constant: $(if ($tagVersion) { $tagVersion } else { '<unresolvable>' }))."
|
||||
|
||||
if ($currentVersion -eq $tagVersion) {
|
||||
$failures.Add("$($client.Name): source changed since publish tag '$tag' but $($client.VersionFile) still reads '$currentVersion' (the version at '$tag'). Bump the version before another publish (CLI-46).")
|
||||
Write-Host " FAIL: version constant unchanged since the publish tag." -ForegroundColor Red
|
||||
}
|
||||
else {
|
||||
Write-Host " Version constant already bumped ('$tagVersion' -> '$currentVersion') since $tag. Pass." -ForegroundColor Green
|
||||
}
|
||||
}
|
||||
|
||||
Write-Host ''
|
||||
if ($failures.Count -gt 0) {
|
||||
Write-Host 'Client version freshness check FAILED:' -ForegroundColor Red
|
||||
foreach ($failure in $failures) {
|
||||
Write-Host " - $failure" -ForegroundColor Red
|
||||
}
|
||||
exit 1
|
||||
}
|
||||
|
||||
Write-Host 'Client version freshness check passed.' -ForegroundColor Green
|
||||
}
|
||||
finally {
|
||||
Pop-Location
|
||||
}
|
||||
@@ -11,11 +11,15 @@ code back so a regression turns the job red.
|
||||
| File | Runs on | Role |
|
||||
|---|---|---|
|
||||
| `run-windev-ci.sh <build\|test\|live>` | Linux CI job | Writes the SSH key/known-hosts, base64-encodes a bootstrap, SSHes to windev, returns the remote exit code. |
|
||||
| `windev-worker-ci.ps1 -Sha -Mode` | windev | Locks the CI clone, checks out the SHA, runs the x86 build / `Worker.Tests` / live smoke. |
|
||||
| `windev-worker-ci.ps1 -Sha -Mode` | windev | Locks the CI clone, checks out the SHA, and runs the requested cumulative Windows tiers. |
|
||||
| `windev.known_hosts` | committed | Pinned windev host keys so `StrictHostKeyChecking=yes` has no TOFU prompt. Host keys are public; committing is intentional. |
|
||||
|
||||
Modes are cumulative: `build` = x86 Worker build; `test` = build + `Worker.Tests`;
|
||||
`live` = test + full-`slnx` build + live-MXAccess smoke (`MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`).
|
||||
`live` = test + full-`slnx` build + the net10 gateway suite + live-MXAccess smoke
|
||||
(`MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`). The `live` order is x86 Worker build,
|
||||
`Worker.Tests`, full solution build, gateway tests, then live smoke. One host lock covers
|
||||
checkout and every tier, so concurrent runs cannot overlap; each native command is checked
|
||||
immediately and the first nonzero exit aborts the remaining tiers and propagates to CI.
|
||||
|
||||
CI jobs (`.gitea/workflows/ci.yml`): `windows-x86` runs `test` per push/PR; `nightly-windev`
|
||||
runs `live` on the `0 6 * * *` schedule and opens a Gitea issue on failure.
|
||||
|
||||
@@ -16,8 +16,8 @@
|
||||
Modes (cumulative):
|
||||
build x86 Worker build only (net48/CS0246/x86 compile guard — always fast).
|
||||
test build + Worker.Tests (x86). The per-push default.
|
||||
live test + the live-MXAccess smoke (MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1) and a full
|
||||
slnx build (surfaces Windows-only analyzer diagnostics, e.g. xUnit1030). Nightly.
|
||||
live test + full slnx build + the net10 gateway suite + the live-MXAccess smoke
|
||||
(MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1). Nightly.
|
||||
|
||||
Runs under Windows PowerShell 5.1 (`powershell.exe`), so it avoids PS7-only syntax and
|
||||
checks $LASTEXITCODE after every native command rather than relying on stderr-as-error
|
||||
@@ -112,6 +112,7 @@ try {
|
||||
|
||||
$workerProj = 'src/ZB.MOM.WW.MxGateway.Worker/ZB.MOM.WW.MxGateway.Worker.csproj'
|
||||
$workerTests = 'src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj'
|
||||
$gatewayTests = 'src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj'
|
||||
$integrationTests = 'src/ZB.MOM.WW.MxGateway.IntegrationTests/ZB.MOM.WW.MxGateway.IntegrationTests.csproj'
|
||||
|
||||
# build: always. The x86/net48 compile is the floor guard (catches CS0246 on
|
||||
@@ -127,6 +128,10 @@ try {
|
||||
# that never fire on the macOS/Linux NonWindows build.
|
||||
Invoke-Native 'full slnx build' { dotnet build src/ZB.MOM.WW.MxGateway.slnx -c Release }
|
||||
|
||||
Invoke-Native 'gateway tests on Windows' {
|
||||
dotnet test $gatewayTests -c Release --no-build
|
||||
}
|
||||
|
||||
$env:MXGATEWAY_RUN_LIVE_MXACCESS_TESTS = '1'
|
||||
try {
|
||||
Invoke-Native 'live-MXAccess smoke' {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -130,6 +130,7 @@ message MxCommand {
|
||||
WriteSecuredBulkCommand write_secured_bulk = 41;
|
||||
WriteSecured2BulkCommand write_secured2_bulk = 42;
|
||||
ReadBulkCommand read_bulk = 43;
|
||||
UpdateAlarmWatchListCommand update_alarm_watch_list = 44;
|
||||
PingCommand ping = 100;
|
||||
GetSessionStateCommand get_session_state = 101;
|
||||
GetWorkerInfoCommand get_worker_info = 102;
|
||||
@@ -174,6 +175,7 @@ enum MxCommandKind {
|
||||
MX_COMMAND_KIND_WRITE_SECURED_BULK = 32;
|
||||
MX_COMMAND_KIND_WRITE_SECURED2_BULK = 33;
|
||||
MX_COMMAND_KIND_READ_BULK = 34;
|
||||
MX_COMMAND_KIND_UPDATE_ALARM_WATCH_LIST = 35;
|
||||
MX_COMMAND_KIND_PING = 100;
|
||||
MX_COMMAND_KIND_GET_SESSION_STATE = 101;
|
||||
MX_COMMAND_KIND_GET_WORKER_INFO = 102;
|
||||
@@ -344,8 +346,8 @@ enum AlarmProviderMode {
|
||||
// Subscription expression follows the canonical
|
||||
// `\\<machine>\Galaxy!<area>` format (literal "Galaxy" provider). The
|
||||
// worker spins up a wnwrapConsumer-backed subscription on its STA on
|
||||
// first call; subsequent calls are an error (use UnsubscribeAlarms then
|
||||
// SubscribeAlarms to reconfigure).
|
||||
// first call; subsequent calls are an error. Use UpdateAlarmWatchList to
|
||||
// replace the standby targets without restarting the primary subscription.
|
||||
message SubscribeAlarmsCommand {
|
||||
string subscription_expression = 1;
|
||||
// UNSPECIFIED = auto-failover/failback. ALARMMGR/SUBTAG force one provider.
|
||||
@@ -354,6 +356,17 @@ message SubscribeAlarmsCommand {
|
||||
// alarmmgr mode; in subtag mode it bounds what the consumer can observe.
|
||||
repeated AlarmSubtagTarget watch_list = 3;
|
||||
AlarmFailoverConfig failover = 4;
|
||||
// True when auto mode should retain subtag-fallback capability even if the
|
||||
// gateway's initial discovery produced an empty watch list. False leaves an
|
||||
// empty list as an intentional alarm-manager-only subscription.
|
||||
bool fallback_enabled = 5;
|
||||
}
|
||||
|
||||
// Replace the live alarm subscription's subtag standby targets. The worker
|
||||
// constructs and attaches the replacement before swapping it into service, so
|
||||
// a setup failure leaves the existing standby and active provider unchanged.
|
||||
message UpdateAlarmWatchListCommand {
|
||||
repeated AlarmSubtagTarget watch_list = 1;
|
||||
}
|
||||
|
||||
// Tear down the worker's alarm consumer. No-op if no subscription is
|
||||
@@ -537,11 +550,14 @@ message MxCommandReply {
|
||||
MxValue return_value = 6;
|
||||
// Correlated per-item outcome rows. For WRITE / WRITE2 / WRITE_SECURED /
|
||||
// WRITE_SECURED2 replies the worker holds the reply for a bounded window
|
||||
// (default 1.5 s, MXGATEWAY_WORKER_WRITE_COMPLETION_WAIT_MS) waiting for
|
||||
// (default 1.5 s, MxGateway:Worker:WriteCompletionWaitMilliseconds) waiting for
|
||||
// the matching MXAccess OnWriteComplete callback and copies its status rows
|
||||
// here, so statuses[0] carries the real MXAccess commit outcome (success OR
|
||||
// failure) while protocol_status/hresult still describe command acceptance
|
||||
// only. Empty statuses on a write reply means the completion did not arrive
|
||||
// only. Callback correlation is recency-based per item handle: after the
|
||||
// bounded wait, a sequential write's callback supersedes a timed-out
|
||||
// predecessor, and a later stale callback for that predecessor is ignored.
|
||||
// Empty statuses on a write reply means the completion did not arrive
|
||||
// within the window — the write is unconfirmed, not failed. Correlation is
|
||||
// best-effort per (server_handle, item_handle): MXAccess's callback carries
|
||||
// no transaction id, so concurrent writes to the same item within the
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
namespace ZB.MOM.WW.MxGateway.Contracts;
|
||||
|
||||
/// <summary>Names of environment variables shared by the gateway worker launcher and worker process.</summary>
|
||||
public static class WorkerLaunchEnvironment
|
||||
{
|
||||
/// <summary>Bootstrap nonce used to authenticate the worker handshake.</summary>
|
||||
public const string WorkerNonce = "MXGATEWAY_WORKER_NONCE";
|
||||
|
||||
/// <summary>Per-attempt named-pipe connection timeout in milliseconds.</summary>
|
||||
public const string PipeConnectAttemptTimeoutMilliseconds =
|
||||
"MXGATEWAY_WORKER_PIPE_CONNECT_ATTEMPT_TIMEOUT_MS";
|
||||
|
||||
/// <summary>Bounded wait for a Write/Write2/WriteSecured/WriteSecured2 completion callback in milliseconds.</summary>
|
||||
public const string WriteCompletionWaitMilliseconds =
|
||||
"MXGATEWAY_WORKER_WRITE_COMPLETION_WAIT_MS";
|
||||
|
||||
/// <summary>Capacity of the worker's outbound MXAccess event queue.</summary>
|
||||
public const string EventQueueCapacity = "MXGATEWAY_EVENT_QUEUE_CAPACITY";
|
||||
|
||||
/// <summary>Worker alarm polling cadence in milliseconds.</summary>
|
||||
public const string AlarmPollIntervalMilliseconds = "MXGATEWAY_ALARM_POLL_INTERVAL_MS";
|
||||
|
||||
/// <summary>Maximum alarm records requested in one current-alarm fetch.</summary>
|
||||
public const string MaxAlarmsPerFetch = "MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH";
|
||||
}
|
||||
@@ -7,7 +7,7 @@
|
||||
<PropertyGroup>
|
||||
<IsPackable>true</IsPackable>
|
||||
<PackageId>ZB.MOM.WW.MxGateway.Contracts</PackageId>
|
||||
<Version>0.2.0</Version>
|
||||
<Version>0.2.1</Version>
|
||||
<Authors>Joseph Doherty</Authors>
|
||||
<Company>ZB MOM WW</Company>
|
||||
<Copyright>Copyright (c) ZB MOM WW. All rights reserved.</Copyright>
|
||||
@@ -37,7 +37,9 @@
|
||||
the wire-identical Galaxy types from the ZB.MOM.WW.GalaxyRepository package. This proto is
|
||||
still the generation source of truth for the language clients (Go/Rust/Java/Python compile it
|
||||
by path; the .NET client consumes Contracts.Proto.Galaxy.* via this project reference) and for
|
||||
clients/proto/proto-inputs.json. Removing it breaks all five clients — do not delete. -->
|
||||
clients/proto/proto-inputs.json. GalaxyContractParityTests normalizes its descriptor against
|
||||
the descriptor served by the package and fails on wire drift. Removing it breaks all five
|
||||
clients — do not delete. -->
|
||||
<Protobuf Include="Protos\galaxy_repository.proto" ProtoRoot="Protos" OutputDir="Generated" GrpcOutputDir="Generated" GrpcServices="Both" />
|
||||
</ItemGroup>
|
||||
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Alarms;
|
||||
using ZB.MOM.WW.MxGateway.Server.Configuration;
|
||||
|
||||
@@ -19,14 +18,18 @@ public sealed class EmptyAlarmWatchListResolver : IAlarmWatchListResolver
|
||||
/// <summary>Shared no-op instance.</summary>
|
||||
public static readonly EmptyAlarmWatchListResolver Instance = new();
|
||||
|
||||
private static readonly IReadOnlyList<AlarmSubtagTarget> Empty = [];
|
||||
private static readonly AlarmWatchListResolution Empty = new(
|
||||
[],
|
||||
DiscoveryAttempted: false,
|
||||
DiscoveryFailed: false,
|
||||
DiscoveryFailureMessage: null);
|
||||
|
||||
private EmptyAlarmWatchListResolver()
|
||||
{
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public Task<IReadOnlyList<AlarmSubtagTarget>> ResolveAsync(
|
||||
public Task<AlarmWatchListResolution> ResolveAsync(
|
||||
AlarmsOptions options,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
|
||||
@@ -514,21 +514,37 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
// AuthenticateUser resolves an ArchestrA user id for the WriteSecured call.
|
||||
// Credentials are env-overridable so the test honors the gateway's "do not
|
||||
// log secrets" rule and works against either MXAccess's own user store or
|
||||
// the LmxOpcUa-baseline GLAuth-bridged ArchestrA identity (admin/admin123).
|
||||
// the LmxOpcUa-baseline GLAuth-bridged ArchestrA identity (admin/password —
|
||||
// see glauth.md; the shared GLAuth's dev-user password is "password", not
|
||||
// the old per-box "admin123").
|
||||
MxCommandReply authReply = await fixture.Service.Invoke(
|
||||
CreateAuthenticateUserRequest(sessionId, serverHandle, verifyUser, verifyPassword),
|
||||
new TestServerCallContext()).ConfigureAwait(false);
|
||||
recordedOutput.WriteLine(
|
||||
$"AuthenticateUser status={authReply.ProtocolStatus.Code} hresult={authReply.Hresult} user_id={authReply.AuthenticateUser?.UserId}");
|
||||
|
||||
// AuthenticateUser is allowed to fail (the underlying provider may reject
|
||||
// the credential pair); we use the returned user id if non-zero and fall
|
||||
// back to 0 ("operator only" / no verifier) so the parity assertion holds.
|
||||
int currentUserId = authReply.ProtocolStatus.Code == ProtocolStatusCode.Ok
|
||||
// AuthenticateUser is allowed to fail — MXAccess parity means a rejected
|
||||
// credential is a legitimate reply, not a bug, so this must never be an
|
||||
// Assert. But a silent fall-through to "operator only" (currentUserId 0)
|
||||
// quietly demotes the rest of this test from an authenticated round-trip
|
||||
// to a bare failure-reply-shape check, which a stale rig credential (e.g.
|
||||
// TST-31) would otherwise hide indefinitely in nightly runs. Log the
|
||||
// outcome loudly — never the password (verifyUser only) — so that
|
||||
// degradation is visible in the test output even though it never fails
|
||||
// the test by itself.
|
||||
bool authenticateUserSucceeded = authReply.ProtocolStatus.Code == ProtocolStatusCode.Ok
|
||||
&& authReply.AuthenticateUser is not null
|
||||
&& authReply.AuthenticateUser.UserId != 0
|
||||
? authReply.AuthenticateUser.UserId
|
||||
: 0;
|
||||
&& authReply.AuthenticateUser.UserId != 0;
|
||||
recordedOutput.WriteLine(
|
||||
authenticateUserSucceeded
|
||||
? $"AuthenticateUser SUCCEEDED for user={verifyUser} user_id={authReply.AuthenticateUser!.UserId}"
|
||||
: $"AuthenticateUser FAILED (credential rejected or not recognized by the provider) "
|
||||
+ $"for user={verifyUser} status={authReply.ProtocolStatus.Code} hresult={authReply.Hresult} "
|
||||
+ $"diagnostic={authReply.DiagnosticMessage} -- WriteSecured below will run as an "
|
||||
+ $"unauthenticated (currentUserId=0) failure-reply-shape check only, not an "
|
||||
+ $"authenticated round-trip. If this is unexpected, verify "
|
||||
+ $"MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_USER/_PASSWORD against the rig's ArchestrA "
|
||||
+ $"identity (see glauth.md).");
|
||||
|
||||
int currentUserId = authenticateUserSucceeded ? authReply.AuthenticateUser!.UserId : 0;
|
||||
|
||||
MxCommandReply writeSecuredReply = await fixture.Service.Invoke(
|
||||
CreateWriteSecuredRequest(
|
||||
@@ -1429,7 +1445,7 @@ public sealed class WorkerLiveMxAccessSmokeTests(ITestOutputHelper output)
|
||||
string verifyUser = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_USER")
|
||||
?? "admin";
|
||||
string verifyPassword = Environment.GetEnvironmentVariable("MXGATEWAY_LIVE_MXACCESS_WRITE_SECURED_PASSWORD")
|
||||
?? "admin123";
|
||||
?? "password";
|
||||
return (verifyUser, verifyPassword);
|
||||
}
|
||||
|
||||
|
||||
@@ -15,4 +15,32 @@ public static class AlarmProviderReasons
|
||||
/// distinguish a forced subtag mode from an unexpected degraded failover.
|
||||
/// </summary>
|
||||
public const string ForcedSubtag = "Forced subtag mode (configuration)";
|
||||
|
||||
/// <summary>
|
||||
/// Reason prefix set when the effective mode can need the subtag watch-list
|
||||
/// (<c>Auto</c> or <c>ForceSubtag</c>) but the list resolved to zero targets, so the
|
||||
/// subtag provider covers no alarms (GWC-37). In <c>Auto</c> this is a warning about the
|
||||
/// standby, not about the currently-serving alarm manager, which is why the dashboard
|
||||
/// renders it against the <c>ALARMMGR</c> mode as a coverage badge rather than a failover.
|
||||
/// </summary>
|
||||
public const string SubtagWatchListEmpty =
|
||||
"Subtag watch-list is empty: the subtag provider covers no alarms";
|
||||
|
||||
/// <summary>
|
||||
/// Reason prefix set when Galaxy Repository alarm discovery was enabled but failed, so
|
||||
/// the watch-list carries only what configuration contributed and may be incomplete
|
||||
/// (GWC-37). Distinct from <see cref="SubtagWatchListEmpty"/>: discovery failing with a
|
||||
/// usable config-only list is a completeness warning, not a total loss of coverage.
|
||||
/// </summary>
|
||||
public const string SubtagDiscoveryFailed =
|
||||
"Galaxy Repository alarm discovery failed: the subtag watch-list may be incomplete";
|
||||
|
||||
/// <summary>
|
||||
/// Reason prefix set when a re-resolution at the failover edge produced a different
|
||||
/// watch-list than the one sent at <c>SubscribeAlarms</c> (GWC-37). The worker keeps the
|
||||
/// list it was given for the life of the subscription, so the difference is reported
|
||||
/// rather than applied; a monitor restart re-resolves and re-sends.
|
||||
/// </summary>
|
||||
public const string SubtagWatchListStale =
|
||||
"Subtag watch-list differs from the one the worker is using";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Alarms;
|
||||
|
||||
/// <summary>
|
||||
/// The outcome of one subtag watch-list resolution: the composed targets plus enough
|
||||
/// provenance for the caller to tell an empty list that means "nothing was configured"
|
||||
/// from one that means "discovery was configured and silently produced nothing"
|
||||
/// (GWC-37). <see cref="IAlarmWatchListResolver.ResolveAsync"/> deliberately never throws
|
||||
/// on a discovery failure, so without this the two are indistinguishable at the call site
|
||||
/// and a Galaxy Repository outage at startup can only be found in a log line.
|
||||
/// </summary>
|
||||
/// <param name="Targets">The resolved watch-list, in order; possibly empty.</param>
|
||||
/// <param name="DiscoveryAttempted">
|
||||
/// Galaxy Repository discovery was enabled for this resolution
|
||||
/// (<c>Fallback:Discovery:UseGalaxyRepository</c>).
|
||||
/// </param>
|
||||
/// <param name="DiscoveryFailed">
|
||||
/// Discovery was attempted and failed, so <paramref name="Targets"/> holds only what
|
||||
/// configuration contributed.
|
||||
/// </param>
|
||||
/// <param name="DiscoveryFailureMessage">
|
||||
/// The discovery failure's message, or <c>null</c> when discovery did not fail.
|
||||
/// </param>
|
||||
public sealed record AlarmWatchListResolution(
|
||||
IReadOnlyList<AlarmSubtagTarget> Targets,
|
||||
bool DiscoveryAttempted,
|
||||
bool DiscoveryFailed,
|
||||
string? DiscoveryFailureMessage);
|
||||
@@ -31,7 +31,7 @@ public sealed class AlarmWatchListResolver : IAlarmWatchListResolver
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<IReadOnlyList<AlarmSubtagTarget>> ResolveAsync(
|
||||
public async Task<AlarmWatchListResolution> ResolveAsync(
|
||||
AlarmsOptions options,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
@@ -39,6 +39,11 @@ public sealed class AlarmWatchListResolver : IAlarmWatchListResolver
|
||||
|
||||
AlarmDiscoveryOptions discovery = options.Fallback.Discovery;
|
||||
|
||||
// GWC-37: the caller cannot grade an empty watch-list without knowing whether discovery
|
||||
// was asked to contribute and whether it failed while doing so, because the failure is
|
||||
// deliberately swallowed below.
|
||||
string? discoveryFailureMessage = null;
|
||||
|
||||
// Config fallback area used only for explicit IncludeAttributes entries (which
|
||||
// carry no discovered area): discovery area, else the default area (may be empty).
|
||||
string configFallbackArea = string.IsNullOrEmpty(discovery.Area) ? options.DefaultArea : discovery.Area;
|
||||
@@ -68,6 +73,7 @@ public sealed class AlarmWatchListResolver : IAlarmWatchListResolver
|
||||
_logger.LogWarning(
|
||||
ex,
|
||||
"Galaxy Repository alarm-attribute discovery failed; continuing with configuration-only watch-list.");
|
||||
discoveryFailureMessage = ex.Message;
|
||||
rows = [];
|
||||
}
|
||||
|
||||
@@ -148,7 +154,11 @@ public sealed class AlarmWatchListResolver : IAlarmWatchListResolver
|
||||
_logger.LogDebug("Resolved alarm subtag watch-list with {TargetCount} target(s).", targets.Count);
|
||||
}
|
||||
|
||||
return targets;
|
||||
return new AlarmWatchListResolution(
|
||||
targets,
|
||||
DiscoveryAttempted: discovery.UseGalaxyRepository,
|
||||
DiscoveryFailed: discoveryFailureMessage is not null,
|
||||
discoveryFailureMessage);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -57,6 +57,21 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
private string _providerReason = string.Empty;
|
||||
private DateTimeOffset _providerSince = DateTimeOffset.UtcNow;
|
||||
|
||||
// GWC-37 subtag-coverage verdict, guarded by _sync: whether the watch-list the worker is
|
||||
// using can actually cover anything, and why not. Independent of _providerDegraded (which
|
||||
// means "the subtag provider is the one serving"): in Auto mode the alarm manager can be
|
||||
// serving perfectly while the standby behind it covers zero alarms, and that is invisible
|
||||
// until the failover that needs it. Folded into BuildProviderStatus so it reaches the feed,
|
||||
// the dashboard badge and every late-joining subscriber rather than only a startup log line.
|
||||
private bool _subtagCoverageMissing;
|
||||
private string _subtagCoverageReason = string.Empty;
|
||||
|
||||
// The full canonical watch-list fingerprint acknowledged by the worker and its alarm
|
||||
// references, guarded by _sync. The fingerprint advances only after an OK subscribe/update
|
||||
// reply, so a rejected replacement remains material and is retried on the next refresh edge.
|
||||
private string _acknowledgedWatchListFingerprint = string.Empty;
|
||||
private IReadOnlyCollection<string> _sentWatchListReferences = [];
|
||||
|
||||
// Whether the worker's most recent reconcile fetch was capped, guarded by _sync.
|
||||
// Written only by ApplyReconcile (and cleared with the cache), so it describes the last full
|
||||
// reconcile — not necessarily the current _alarms contents, which live transitions keep moving
|
||||
@@ -329,12 +344,22 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
|
||||
private async Task SubscribeAlarmsAsync(string sessionId, string subscription, CancellationToken cancellationToken)
|
||||
{
|
||||
IReadOnlyList<AlarmSubtagTarget> watchList = await _watchListResolver
|
||||
AlarmWatchListResolution resolution = await _watchListResolver
|
||||
.ResolveAsync(_options, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
IReadOnlyList<AlarmSubtagTarget> watchList = resolution.Targets;
|
||||
|
||||
AlarmProviderMode forcedMode = MapForcedMode(_options.Fallback.Mode);
|
||||
|
||||
// GWC-37: grade the coverage before the command goes out, so a Galaxy Repository outage at
|
||||
// startup is a standing status on the feed instead of one log line an operator reads weeks
|
||||
// later — when the failover that needed the watch-list finds it empty. Broadcast because
|
||||
// feed subscribers outlive the monitor's worker session and may already be attached.
|
||||
lock (_sync)
|
||||
{
|
||||
SetSubtagCoverage(GradeSubtagCoverage(forcedMode, resolution, staleAgainstSent: false), broadcast: true);
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Alarm subscribe: forcedMode={ForcedMode} configMode={ConfigMode} watchList={WatchListCount}.",
|
||||
forcedMode, _options.Fallback.Mode, watchList.Count);
|
||||
@@ -347,6 +372,7 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
{
|
||||
SubscriptionExpression = subscription,
|
||||
ForcedMode = forcedMode,
|
||||
FallbackEnabled = CanUseSubtagFallback(forcedMode),
|
||||
Failover = new AlarmFailoverConfig
|
||||
{
|
||||
ConsecutiveFailureThreshold = _options.Fallback.ConsecutiveFailureThreshold,
|
||||
@@ -377,6 +403,11 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
?? $"status {code}";
|
||||
throw new InvalidOperationException($"Worker rejected SubscribeAlarms: {diagnostic}");
|
||||
}
|
||||
|
||||
lock (_sync)
|
||||
{
|
||||
AcknowledgeWatchList(watchList);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReconcileLoopAsync(string sessionId, CancellationToken cancellationToken)
|
||||
@@ -389,6 +420,8 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
{
|
||||
try
|
||||
{
|
||||
await RefreshAlarmWatchListAsync(sessionId, broadcast: true, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
await ReconcileAsync(sessionId, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
@@ -518,6 +551,16 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
AlarmProviderMode toMode = change.Mode;
|
||||
string reason = change.Reason ?? string.Empty;
|
||||
|
||||
// The failover edge is the moment the subtag watch-list stops being theoretical. Refresh
|
||||
// it before publishing the status frame so recovered discovery is applied to the already
|
||||
// active standby and the operator sees the acknowledged coverage verdict atomically with
|
||||
// the provider switch.
|
||||
if (toMode == AlarmProviderMode.Subtag)
|
||||
{
|
||||
await RefreshAlarmWatchListAsync(sessionId, broadcast: false, cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
AlarmProviderStatus status;
|
||||
int fromModeInt;
|
||||
lock (_sync)
|
||||
@@ -565,14 +608,221 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RefreshAlarmWatchListAsync(
|
||||
string sessionId,
|
||||
bool broadcast,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
AlarmProviderMode forcedMode = MapForcedMode(_options.Fallback.Mode);
|
||||
if (!CanUseSubtagFallback(forcedMode))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
AlarmWatchListResolution resolution;
|
||||
try
|
||||
{
|
||||
resolution = await _watchListResolver.ResolveAsync(_options, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
exception,
|
||||
"Re-resolving the alarm subtag watch-list failed; keeping the last acknowledged watch list.");
|
||||
return;
|
||||
}
|
||||
|
||||
string resolvedFingerprint = CanonicalWatchListFingerprint(resolution.Targets);
|
||||
string acknowledgedFingerprint;
|
||||
lock (_sync)
|
||||
{
|
||||
acknowledgedFingerprint = _acknowledgedWatchListFingerprint;
|
||||
}
|
||||
|
||||
if (string.Equals(resolvedFingerprint, acknowledgedFingerprint, StringComparison.Ordinal))
|
||||
{
|
||||
lock (_sync)
|
||||
{
|
||||
SetSubtagCoverage(
|
||||
GradeSubtagCoverage(forcedMode, resolution, staleAgainstSent: false),
|
||||
broadcast);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
UpdateAlarmWatchListCommand update = new();
|
||||
update.WatchList.AddRange(resolution.Targets);
|
||||
WorkerCommandReply reply = await _sessionManager.InvokeAsync(
|
||||
sessionId,
|
||||
new WorkerCommand
|
||||
{
|
||||
Command = new MxCommand
|
||||
{
|
||||
Kind = MxCommandKind.UpdateAlarmWatchList,
|
||||
UpdateAlarmWatchList = update,
|
||||
},
|
||||
},
|
||||
cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
ProtocolStatusCode? code = reply.Reply?.ProtocolStatus?.Code;
|
||||
lock (_sync)
|
||||
{
|
||||
if (code == ProtocolStatusCode.Ok)
|
||||
{
|
||||
AcknowledgeWatchList(resolution.Targets);
|
||||
SetSubtagCoverage(
|
||||
GradeSubtagCoverage(forcedMode, resolution, staleAgainstSent: false),
|
||||
broadcast);
|
||||
return;
|
||||
}
|
||||
|
||||
string diagnostic = reply.Reply?.DiagnosticMessage
|
||||
?? reply.Reply?.ProtocolStatus?.Message
|
||||
?? $"status {code}";
|
||||
string reason = GradeSubtagCoverage(forcedMode, resolution, staleAgainstSent: true);
|
||||
SetSubtagCoverage($"{reason}; worker rejected watch-list update ({diagnostic})", broadcast);
|
||||
}
|
||||
}
|
||||
|
||||
private bool CanUseSubtagFallback(AlarmProviderMode forcedMode) =>
|
||||
forcedMode != AlarmProviderMode.Alarmmgr
|
||||
&& (forcedMode == AlarmProviderMode.Subtag
|
||||
|| _options.Fallback.Discovery.UseGalaxyRepository
|
||||
|| _options.Fallback.Discovery.IncludeAttributes.Length > 0);
|
||||
|
||||
// Caller holds _sync.
|
||||
private void AcknowledgeWatchList(IReadOnlyList<AlarmSubtagTarget> watchList)
|
||||
{
|
||||
_acknowledgedWatchListFingerprint = CanonicalWatchListFingerprint(watchList);
|
||||
_sentWatchListReferences = watchList
|
||||
.Select(target => target.AlarmFullReference)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static string CanonicalWatchListFingerprint(IEnumerable<AlarmSubtagTarget> watchList) =>
|
||||
string.Join(
|
||||
"\n",
|
||||
watchList
|
||||
.Select(target => string.Join(
|
||||
"|",
|
||||
FingerprintPart(target.AlarmFullReference),
|
||||
FingerprintPart(target.SourceObjectReference),
|
||||
FingerprintPart(target.ActiveSubtag),
|
||||
FingerprintPart(target.AckedSubtag),
|
||||
FingerprintPart(target.AckCommentSubtag),
|
||||
FingerprintPart(target.PrioritySubtag)))
|
||||
.Order(StringComparer.Ordinal));
|
||||
|
||||
private static string FingerprintPart(string? value)
|
||||
{
|
||||
string normalized = value ?? string.Empty;
|
||||
return $"{normalized.Length}:{normalized}";
|
||||
}
|
||||
|
||||
// Grades one resolution against the effective mode. Returns the reason the subtag coverage is
|
||||
// unsound, or an empty string when it is sound (or irrelevant, as under ForceAlarmManager).
|
||||
// Empty targets are the total-loss case; a failed discovery with a usable config-only list is
|
||||
// a completeness warning; drift is reported separately because the worker cannot be updated.
|
||||
private string GradeSubtagCoverage(
|
||||
AlarmProviderMode forcedMode,
|
||||
AlarmWatchListResolution resolution,
|
||||
bool staleAgainstSent)
|
||||
{
|
||||
// ForceAlarmManager never builds a subtag consumer, so its watch-list is genuinely unused.
|
||||
if (forcedMode == AlarmProviderMode.Alarmmgr)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
// Nothing was asked of discovery and nothing was configured: subtag fallback is simply not
|
||||
// in use in this deployment (Auto with an empty list subscribes alarmmgr-only), so there is
|
||||
// no coverage to warn about.
|
||||
if (resolution.Targets.Count == 0
|
||||
&& !resolution.DiscoveryAttempted
|
||||
&& _options.Fallback.Discovery.IncludeAttributes.Length == 0
|
||||
&& forcedMode != AlarmProviderMode.Subtag)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
List<string> reasons = [];
|
||||
if (resolution.Targets.Count == 0)
|
||||
{
|
||||
reasons.Add(AlarmProviderReasons.SubtagWatchListEmpty);
|
||||
}
|
||||
|
||||
if (resolution.DiscoveryFailed)
|
||||
{
|
||||
reasons.Add(resolution.DiscoveryFailureMessage is { Length: > 0 } message
|
||||
? $"{AlarmProviderReasons.SubtagDiscoveryFailed} ({message})"
|
||||
: AlarmProviderReasons.SubtagDiscoveryFailed);
|
||||
}
|
||||
|
||||
if (staleAgainstSent)
|
||||
{
|
||||
reasons.Add(
|
||||
$"{AlarmProviderReasons.SubtagWatchListStale} ({resolution.Targets.Count} target(s) now "
|
||||
+ $"resolve against {_sentWatchListReferences.Count} acknowledged by the worker)");
|
||||
}
|
||||
|
||||
return string.Join("; ", reasons);
|
||||
}
|
||||
|
||||
// Caller holds _sync. Applies a coverage verdict, logging and (optionally) broadcasting only
|
||||
// when it actually changed so a stable degraded state does not repeat a frame every pass.
|
||||
private void SetSubtagCoverage(string reason, bool broadcast)
|
||||
{
|
||||
bool missing = reason.Length > 0;
|
||||
if (missing == _subtagCoverageMissing && string.Equals(reason, _subtagCoverageReason, StringComparison.Ordinal))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_subtagCoverageMissing = missing;
|
||||
_subtagCoverageReason = reason;
|
||||
|
||||
if (missing)
|
||||
{
|
||||
_logger.LogWarning("Alarm subtag coverage is unsound: {Reason}", reason);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation("Alarm subtag coverage is sound.");
|
||||
}
|
||||
|
||||
if (broadcast)
|
||||
{
|
||||
BroadcastToAll(new AlarmFeedMessage { ProviderStatus = BuildProviderStatus() });
|
||||
}
|
||||
}
|
||||
|
||||
// Caller holds _sync. Builds an AlarmProviderStatus snapshot of the current state.
|
||||
//
|
||||
// GWC-37: degraded is the OR of "the subtag provider is the one serving" and "the subtag
|
||||
// watch-list cannot cover anything". The second is what makes an empty watch-list visible in
|
||||
// Auto mode, where the alarm manager is serving happily and the standby behind it is empty.
|
||||
// The reasons are concatenated when both apply, which is the worst case worth naming loudly:
|
||||
// failed over to a provider that covers nothing.
|
||||
private AlarmProviderStatus BuildProviderStatus()
|
||||
{
|
||||
bool degraded = _providerDegraded || _subtagCoverageMissing;
|
||||
string reason = (_providerReason, _subtagCoverageReason) switch
|
||||
{
|
||||
("", var coverage) => coverage,
|
||||
(var mode, "") => mode,
|
||||
(var mode, var coverage) => $"{mode} — {coverage}",
|
||||
};
|
||||
|
||||
return new AlarmProviderStatus
|
||||
{
|
||||
Mode = _providerMode,
|
||||
Degraded = _providerDegraded,
|
||||
Reason = _providerReason,
|
||||
Degraded = degraded,
|
||||
Reason = reason,
|
||||
Since = Timestamp.FromDateTimeOffset(_providerSince),
|
||||
};
|
||||
}
|
||||
@@ -671,15 +921,17 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic
|
||||
new AlarmFeedMessage { Transition = TransitionFromSnapshot(incoming.Value, AlarmTransitionKind.Raise) },
|
||||
incoming.Key);
|
||||
}
|
||||
else if (existing.CurrentState != incoming.Value.CurrentState
|
||||
&& incoming.Value.CurrentState == AlarmConditionState.ActiveAcked)
|
||||
else if (existing.CurrentState != incoming.Value.CurrentState)
|
||||
{
|
||||
// The alarm was already known but the worker now reports it acknowledged: the
|
||||
// live Acknowledge transition never reached the feed. Without this the acked
|
||||
// state is absorbed silently by the snapshot replace below and subscribers show
|
||||
// the alarm unacked until it clears.
|
||||
// The alarm was already known but the worker now reports a different state. An
|
||||
// ActiveAcked snapshot repairs a missed Acknowledge. Any other incoming active
|
||||
// state repairs a missed Raise for the incoming instance (including an
|
||||
// acknowledged alarm reference that was cleared and re-raised between passes).
|
||||
AlarmTransitionKind kind = incoming.Value.CurrentState == AlarmConditionState.ActiveAcked
|
||||
? AlarmTransitionKind.Acknowledge
|
||||
: AlarmTransitionKind.Raise;
|
||||
Broadcast(
|
||||
new AlarmFeedMessage { Transition = TransitionFromSnapshot(incoming.Value, AlarmTransitionKind.Acknowledge) },
|
||||
new AlarmFeedMessage { Transition = TransitionFromSnapshot(incoming.Value, kind) },
|
||||
incoming.Key);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,13 +18,15 @@ public interface IAlarmWatchListResolver
|
||||
/// <param name="options">Alarm configuration carrying discovery and subtag-name settings.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <returns>
|
||||
/// The resolved <see cref="AlarmSubtagTarget"/> watch-list, possibly empty.
|
||||
/// The resolved <see cref="AlarmSubtagTarget"/> watch-list, possibly empty, together
|
||||
/// with the discovery provenance the caller needs to grade it.
|
||||
/// Discovery being unavailable never throws — it yields an empty (or
|
||||
/// config-only) list and the caller decides what to do with it. Cancellation
|
||||
/// config-only) list with <see cref="AlarmWatchListResolution.DiscoveryFailed"/> set,
|
||||
/// and the caller decides what to do with it. Cancellation
|
||||
/// is the one exception: a triggered <paramref name="cancellationToken"/>
|
||||
/// still propagates an <see cref="OperationCanceledException"/>.
|
||||
/// </returns>
|
||||
Task<IReadOnlyList<AlarmSubtagTarget>> ResolveAsync(
|
||||
Task<AlarmWatchListResolution> ResolveAsync(
|
||||
AlarmsOptions options,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
@@ -606,6 +606,15 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
||||
// wrapped in a WorkerEnvelope and the outbound write faults the whole session. Fail fast
|
||||
// at startup rather than mid-traffic. Only checked when both knobs are themselves in range so the
|
||||
// message is not doubled up with the individual range errors.
|
||||
//
|
||||
// IPC-33: the coupling also has an upper half. The worker budgets a DrainEvents reply against the
|
||||
// negotiated frame maximum less two reserves, and that same MxCommandReply is then returned
|
||||
// verbatim through Invoke, where the gateway's gRPC MaxSendMessageSize (= MaxGrpcMessageBytes)
|
||||
// applies. Letting the pipe maximum run arbitrarily far above the gRPC cap lets the worker pack a
|
||||
// reply the pipe carries and the gRPC layer then refuses to send — with the drained events already
|
||||
// removed from the worker queue, so they are lost to both the caller and the event stream. Keeping
|
||||
// the pipe maximum within two reserves of the gRPC cap makes "frame max - 2x reserve" a budget that
|
||||
// is always at or below the gRPC cap, so the wrapper always has a full reserve of margin.
|
||||
private static void ValidateFrameSizeHeadroom(
|
||||
WorkerOptions worker,
|
||||
ProtocolOptions protocol,
|
||||
@@ -628,6 +637,20 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase<GatewayOption
|
||||
+ $"{WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes}-byte worker-frame envelope reserve "
|
||||
+ $"(>= {requiredWorkerMax}).");
|
||||
}
|
||||
|
||||
long permittedWorkerMax =
|
||||
(long)protocol.MaxGrpcMessageBytes + (2L * WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes);
|
||||
if (worker.MaxMessageBytes > permittedWorkerMax)
|
||||
{
|
||||
builder.Add(
|
||||
$"MxGateway:Worker:MaxMessageBytes ({worker.MaxMessageBytes}) must be at most "
|
||||
+ $"MxGateway:Protocol:MaxGrpcMessageBytes ({protocol.MaxGrpcMessageBytes}) plus twice the "
|
||||
+ $"{WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes}-byte worker-frame envelope reserve "
|
||||
+ $"(<= {permittedWorkerMax}). The worker budgets a DrainEvents reply against the negotiated "
|
||||
+ "frame maximum less two reserves; a pipe maximum further above the gRPC cap lets the worker "
|
||||
+ "pack a reply the pipe carries but the gRPC send cap refuses, losing the already-drained "
|
||||
+ "events. Lower MxGateway:Worker:MaxMessageBytes or raise MxGateway:Protocol:MaxGrpcMessageBytes.");
|
||||
}
|
||||
}
|
||||
|
||||
private static void AddIfBlank(string? value, string message, ValidationBuilder builder)
|
||||
|
||||
@@ -58,9 +58,12 @@ public sealed class WorkerOptions
|
||||
/// The maximum worker-frame (pipe) message size in bytes. Must stay at least
|
||||
/// <see cref="Workers.WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes"/> above
|
||||
/// <see cref="ProtocolOptions.MaxGrpcMessageBytes"/> so a maximally-sized accepted gRPC payload
|
||||
/// still fits one worker frame; the gateway conveys this value to the worker in the
|
||||
/// handshake (<c>GatewayHello.max_frame_bytes</c>). Default is the 16 MB public gRPC cap
|
||||
/// plus that reserve.
|
||||
/// still fits one worker frame, and at most <em>twice</em> that reserve above it so a
|
||||
/// <c>DrainEvents</c> reply budgeted against this maximum (frame max less two reserves) still
|
||||
/// fits the gRPC send cap it is returned through — otherwise the worker can pack a reply the
|
||||
/// pipe carries and the gRPC layer refuses, losing the already-drained events (IPC-33). The
|
||||
/// gateway conveys this value to the worker in the handshake
|
||||
/// (<c>GatewayHello.max_frame_bytes</c>). Default is the 16 MB public gRPC cap plus one reserve.
|
||||
/// </summary>
|
||||
public int MaxMessageBytes { get; init; } =
|
||||
(16 * 1024 * 1024) + Workers.WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes;
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
|
||||
/// <summary>
|
||||
/// Handler for <see cref="AuthenticatedDashboardRoleRequirement"/>: succeeds only when the caller
|
||||
/// is authenticated AND holds one of the requirement's roles. It reads no options at all, which
|
||||
/// is the point — there is no configuration value that can turn this into a bypass (SEC-40).
|
||||
/// </summary>
|
||||
public sealed class AuthenticatedDashboardRoleHandler
|
||||
: AuthorizationHandler<AuthenticatedDashboardRoleRequirement>
|
||||
{
|
||||
/// <inheritdoc />
|
||||
protected override Task HandleRequirementAsync(
|
||||
AuthorizationHandlerContext context,
|
||||
AuthenticatedDashboardRoleRequirement requirement)
|
||||
{
|
||||
if (context.User.Identity?.IsAuthenticated != true)
|
||||
{
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
foreach (string role in requirement.RequiredRoles)
|
||||
{
|
||||
if (context.User.IsInRole(role))
|
||||
{
|
||||
context.Succeed(requirement);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Server.Dashboard;
|
||||
|
||||
/// <summary>
|
||||
/// Requirement satisfied ONLY by an authenticated principal holding one of
|
||||
/// <see cref="RequiredRoles"/> as a real role claim. Unlike
|
||||
/// <see cref="DashboardAuthorizationRequirement"/> there is no environment bypass: it is a
|
||||
/// distinct requirement type, so <see cref="DashboardAuthorizationHandler"/> —
|
||||
/// an <c>AuthorizationHandler<DashboardAuthorizationRequirement></c> — is never invoked for
|
||||
/// it and its <c>Authentication:Mode=Disabled</c> / anonymous-localhost arms cannot reach it.
|
||||
/// The type distinction is the enforcement; adding a bypass arm to the other handler cannot leak
|
||||
/// into this one by accident.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Introduced for SEC-40 to gate the alarm feed (<c>AlarmsHub</c> and the <c>/alarms</c> page),
|
||||
/// which is gateway-central and session-less and therefore carries no per-session provenance the
|
||||
/// SEC-25 tag ACL could scope it by. Requiring a real dashboard role claim is the narrowing that
|
||||
/// is actually available: it keeps galaxy-wide alarm metadata — alarm reference (a full tag path),
|
||||
/// state, severity, category, area, acknowledging operator — away from the read-only environment
|
||||
/// bypasses, which exist for local development and unauthenticated deployments rather than as a
|
||||
/// grant of plant-wide visibility. The same pattern the shared Secrets UI relies on: a policy the
|
||||
/// dashboard bypass handler cannot satisfy because the requirement type differs.
|
||||
/// </remarks>
|
||||
/// <param name="requiredRoles">Roles any one of which satisfies the requirement.</param>
|
||||
public sealed class AuthenticatedDashboardRoleRequirement(IReadOnlyList<string> requiredRoles)
|
||||
: IAuthorizationRequirement
|
||||
{
|
||||
/// <summary>Gets the roles, any one of which satisfies this requirement.</summary>
|
||||
public IReadOnlyList<string> RequiredRoles { get; } = requiredRoles;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the requirement satisfied by an authenticated caller holding either dashboard role.
|
||||
/// Same role set as <see cref="DashboardAuthorizationRequirement.AnyDashboardRole"/>; the
|
||||
/// difference is entirely that no environment bypass applies.
|
||||
/// </summary>
|
||||
public static AuthenticatedDashboardRoleRequirement AnyDashboardRole { get; } =
|
||||
new([DashboardRoles.Admin, DashboardRoles.Viewer]);
|
||||
}
|
||||
@@ -12,7 +12,14 @@
|
||||
<NavRailItem Href="/sessions" Text="Sessions" />
|
||||
<NavRailItem Href="/workers" Text="Workers" />
|
||||
<NavRailItem Href="/events" Text="Events" />
|
||||
<NavRailItem Href="/alarms" Text="Alarms" />
|
||||
@* Gated on the SAME policy the /alarms page and AlarmsHub enforce (SEC-40), for the
|
||||
same reason the Secrets link below is: the page denies an environment-bypass
|
||||
caller outright, so an ungated link would be a dead end. *@
|
||||
<AuthorizeView Policy="@DashboardAuthenticationDefaults.AlarmsPolicy">
|
||||
<Authorized>
|
||||
<NavRailItem Href="/alarms" Text="Alarms" />
|
||||
</Authorized>
|
||||
</AuthorizeView>
|
||||
</NavRailSection>
|
||||
<NavRailSection Title="Galaxy" Key="galaxy">
|
||||
<NavRailItem Href="/galaxy" Text="Repository" />
|
||||
|
||||
@@ -1,6 +1,15 @@
|
||||
@page "/alarms"
|
||||
@implements IAsyncDisposable
|
||||
@using Microsoft.AspNetCore.Authorization
|
||||
@using ZB.MOM.WW.MxGateway.Server.Alarms
|
||||
@* SEC-40: the alarm feed needs a real authenticated dashboard role, not the read-only
|
||||
environment bypasses (Authentication:Mode=Disabled, anonymous localhost) that satisfy the
|
||||
ViewerPolicy MapRazorComponents<App>() applies. This page reads the gateway-central alarm
|
||||
monitor in process, exactly the surface AlarmsHub serves to external clients, so it carries the
|
||||
SAME policy as the hub — gating only the hub would leave the disclosure one navigation away.
|
||||
Alarm metadata stays galaxy-wide for an authenticated Viewer: the monitor has no per-session
|
||||
provenance, so the SEC-25 tag ACL has nothing to intersect. *@
|
||||
@attribute [Authorize(Policy = DashboardAuthenticationDefaults.AlarmsPolicy)]
|
||||
@inject IDashboardLiveDataService LiveData
|
||||
@inject IOptions<GatewayOptions> GatewayOptions
|
||||
@inject IGatewayAlarmService AlarmService
|
||||
@@ -191,13 +200,13 @@
|
||||
private Task? _pollTask;
|
||||
|
||||
private DashboardAlarmProviderStatus _providerStatus = DashboardAlarmProviderStatus.Healthy;
|
||||
private Task? _providerStatusTask;
|
||||
private Task? _statusFeedTask;
|
||||
|
||||
/// <inheritdoc />
|
||||
protected override void OnInitialized()
|
||||
{
|
||||
_pollTask = PollLoopAsync();
|
||||
_providerStatusTask = ProviderStatusLoopAsync();
|
||||
_statusFeedTask = StatusFeedLoopAsync();
|
||||
}
|
||||
|
||||
private string? ProviderStatusTitle()
|
||||
@@ -210,8 +219,13 @@
|
||||
// The badge tracks the central monitor directly rather than looping back through
|
||||
// /hubs/alarms: the alarm service is an in-process multi-subscriber fan-out, so a
|
||||
// server-rendered page needs no SignalR client, no loopback socket and no auth token.
|
||||
// Alarm rows still come from the 3-second poll below — this loop only feeds the badge.
|
||||
private async Task ProviderStatusLoopAsync()
|
||||
// This loop feeds the two gateway-status indicators — the provider badge and the
|
||||
// truncation banner — from the feed's own status frames, so both move as soon as the
|
||||
// monitor's verdict changes instead of on the next 3-second tick. Alarm rows still come
|
||||
// from the poll below, which also re-asserts the truncation verdict as its reconcile
|
||||
// baseline: both sources read the same monitor verdict, so they cannot disagree for
|
||||
// longer than one tick, and neither one is synthesized here.
|
||||
private async Task StatusFeedLoopAsync()
|
||||
{
|
||||
while (!_cts.IsCancellationRequested)
|
||||
{
|
||||
@@ -221,16 +235,30 @@
|
||||
.StreamAsync(alarmFilterPrefix: null, _cts.Token)
|
||||
.ConfigureAwait(false))
|
||||
{
|
||||
if (message.PayloadCase != AlarmFeedMessage.PayloadOneofCase.ProviderStatus)
|
||||
switch (message.PayloadCase)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
case AlarmFeedMessage.PayloadOneofCase.ProviderStatus:
|
||||
await InvokeAsync(() =>
|
||||
{
|
||||
_providerStatus = DashboardAlarmProviderStatus.FromFeed(message);
|
||||
StateHasChanged();
|
||||
}).ConfigureAwait(false);
|
||||
break;
|
||||
|
||||
await InvokeAsync(() =>
|
||||
{
|
||||
_providerStatus = DashboardAlarmProviderStatus.FromFeed(message);
|
||||
StateHasChanged();
|
||||
}).ConfigureAwait(false);
|
||||
// Every subscriber is primed with this frame at open, so a page that
|
||||
// attaches mid-truncation gets the caveat without waiting for an edge —
|
||||
// no page-side priming needed.
|
||||
case AlarmFeedMessage.PayloadOneofCase.SnapshotStatus:
|
||||
await InvokeAsync(() =>
|
||||
{
|
||||
_snapshotTruncated = message.SnapshotStatus.Truncated;
|
||||
StateHasChanged();
|
||||
}).ConfigureAwait(false);
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
@@ -239,9 +267,13 @@
|
||||
}
|
||||
catch
|
||||
{
|
||||
// The monitor completes a subscriber's stream when it falls behind, and
|
||||
// again when the monitor restarts. Both are recoverable by resubscribing;
|
||||
// the badge holds its last value in the meantime.
|
||||
// The monitor drops a subscriber whose queue it cannot write to, completing
|
||||
// that stream with an error; short of cancellation or disposal that is the
|
||||
// only way this enumeration ends. A monitor restart is NOT one of them — it
|
||||
// keeps the channel and pushes the cleared status frames through it — so this
|
||||
// catch is the fell-behind case, recoverable by resubscribing. The badge and
|
||||
// banner hold their last values meanwhile, and the resubscribe is primed with
|
||||
// the current ones.
|
||||
}
|
||||
|
||||
try
|
||||
@@ -321,7 +353,7 @@
|
||||
};
|
||||
}
|
||||
|
||||
// Fault handling sits inside the loop, matching ProviderStatusLoopAsync: a query or render
|
||||
// Fault handling sits inside the loop, matching StatusFeedLoopAsync: a query or render
|
||||
// fault on one tick is transient (a provider blip, a momentarily unavailable session), so it
|
||||
// is surfaced on the page and retried on the next tick rather than ending polling for the
|
||||
// life of the page. Cancellation is the only exit. The loop method itself therefore cannot
|
||||
@@ -398,6 +430,10 @@
|
||||
{
|
||||
DashboardAlarmQueryResult result = await LiveData.QueryAlarmsAsync(_cts.Token);
|
||||
_queryError = result.Error;
|
||||
// Kept alongside the feed's snapshot_status frame rather than replaced by it: this is
|
||||
// the reconcile baseline. Both read the same monitor verdict, so the poll can only
|
||||
// confirm what the frame already showed — but it also re-establishes the banner for a
|
||||
// page whose feed subscription is mid-resubscribe after the monitor dropped it.
|
||||
_snapshotTruncated = result.SnapshotTruncated;
|
||||
_workerPid = result.WorkerProcessId;
|
||||
_lastRefresh = DateTimeOffset.UtcNow;
|
||||
@@ -416,7 +452,7 @@
|
||||
// Drained together, not one after the other: the wedged dispatcher this bound exists
|
||||
// for blocks both loops at once, so sequential drains would time out twice and make
|
||||
// the real bound 10 seconds. DrainAsync tolerates a null task.
|
||||
await Task.WhenAll(DrainAsync(_pollTask), DrainAsync(_providerStatusTask))
|
||||
await Task.WhenAll(DrainAsync(_pollTask), DrainAsync(_statusFeedTask))
|
||||
.ConfigureAwait(false);
|
||||
|
||||
_cts.Dispose();
|
||||
|
||||
@@ -98,6 +98,8 @@ else
|
||||
<tr><th scope="row">Dashboard enabled</th><td>@Snapshot.Configuration.Dashboard.Enabled</td></tr>
|
||||
<tr><th scope="row">Anonymous localhost</th><td>@Snapshot.Configuration.Dashboard.AllowAnonymousLocalhost</td></tr>
|
||||
<tr><th scope="row">Snapshot interval</th><td>@Snapshot.Configuration.Dashboard.SnapshotIntervalMilliseconds ms</td></tr>
|
||||
<tr><th scope="row">Recent fault limit</th><td>@Snapshot.Configuration.Dashboard.RecentFaultLimit</td></tr>
|
||||
<tr><th scope="row">Recent session limit</th><td>@Snapshot.Configuration.Dashboard.RecentSessionLimit</td></tr>
|
||||
<tr><th scope="row">Show tag values</th><td>@Snapshot.Configuration.Dashboard.ShowTagValues</td></tr>
|
||||
<tr><th scope="row">Untagged session visibility</th><td>@Snapshot.Configuration.Dashboard.UntaggedSessionVisibility</td></tr>
|
||||
<tr><th scope="row">Worker protocol</th><td>@Snapshot.Configuration.Protocol.WorkerProtocolVersion</td></tr>
|
||||
|
||||
@@ -30,6 +30,14 @@ public sealed record DashboardAlarmProviderStatus(
|
||||
/// </summary>
|
||||
public const string ForcedSubtagLabel = "Subtag monitoring (forced)";
|
||||
|
||||
/// <summary>
|
||||
/// Badge label shown when the alarm manager is serving normally but the subtag standby
|
||||
/// behind it covers nothing — an empty or undiscoverable watch-list (GWC-37). The feed is
|
||||
/// healthy today; what is degraded is the fallback that would take over tomorrow, so the
|
||||
/// badge names the standby rather than claiming the feed has failed over.
|
||||
/// </summary>
|
||||
public const string NoSubtagStandbyLabel = "Alarm Manager (no subtag standby)";
|
||||
|
||||
private const string HealthyBadge = "bg-success";
|
||||
private const string DegradedBadge = "bg-warning text-dark";
|
||||
|
||||
@@ -69,8 +77,18 @@ public sealed record DashboardAlarmProviderStatus(
|
||||
&& status.Mode == AlarmProviderMode.Subtag
|
||||
&& string.Equals(reason, AlarmProviderReasons.ForcedSubtag, StringComparison.Ordinal);
|
||||
|
||||
string label = !degraded ? AlarmManagerLabel : forced ? ForcedSubtagLabel : DegradedLabel;
|
||||
string badge = !degraded ? HealthyBadge : forced ? ForcedSubtagBadge : DegradedBadge;
|
||||
// GWC-37: degraded WITHOUT subtag mode means the alarm manager is still serving and the
|
||||
// standby is the unsound half. "Subtag monitoring (degraded)" would misreport which
|
||||
// provider is live, so this state gets its own label; the amber badge is shared because
|
||||
// the operator action — fix discovery — is equally overdue in both.
|
||||
bool standbyOnlyWarning = degraded && status.Mode != AlarmProviderMode.Subtag;
|
||||
|
||||
string label = !degraded
|
||||
? AlarmManagerLabel
|
||||
: standbyOnlyWarning ? NoSubtagStandbyLabel : forced ? ForcedSubtagLabel : DegradedLabel;
|
||||
string badge = !degraded
|
||||
? HealthyBadge
|
||||
: standbyOnlyWarning ? DegradedBadge : forced ? ForcedSubtagBadge : DegradedBadge;
|
||||
|
||||
return new DashboardAlarmProviderStatus(
|
||||
Mode: status.Mode,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user