From 9958f800264949a8f0d1f399413bb94ae3cb928b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:01:18 -0400 Subject: [PATCH 01/41] docs(plans): perf review remediation plan --- .../2026-08-15-perf-review-remediation.md | 624 ++++++++++++++++++ ...8-15-perf-review-remediation.md.tasks.json | 31 + 2 files changed, 655 insertions(+) create mode 100644 docs/plans/2026-08-15-perf-review-remediation.md create mode 100644 docs/plans/2026-08-15-perf-review-remediation.md.tasks.json diff --git a/docs/plans/2026-08-15-perf-review-remediation.md b/docs/plans/2026-08-15-perf-review-remediation.md new file mode 100644 index 0000000..71e8f89 --- /dev/null +++ b/docs/plans/2026-08-15-perf-review-remediation.md @@ -0,0 +1,624 @@ +# Performance Review Remediation Implementation Plan + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers-extended-cc:executing-plans to implement this plan task-by-task (or superpowers-extended-cc:subagent-driven-development when executing in-session). + +**Goal:** Resolve every actionable finding from the 2026-08-15 architectural performance review — six High findings, the Medium tier, and the worthwhile Low/hygiene items — without changing any MXAccess parity behavior or public contract. + +**Architecture:** Two phases. Phase A is gateway-side (.NET 10, builds and tests locally on macOS via `NonWindows.slnx`); Phase B is worker-side (.NET Framework 4.8 x86, which does **not** compile on this Mac — Phase B tasks are edited here and verified in one consolidated pass on the windev box via the `psbridge` skill, Task 24). No `.proto` changes anywhere in this plan, so no client regeneration is needed. All work happens on branch `perf/review-remediation`. + +**Tech Stack:** ASP.NET Core gRPC, System.Threading.Channels, SignalR, Microsoft.Data.Sqlite, .NET Framework 4.8 STA/COM interop, protobuf (Google.Protobuf). + +--- + +## Ground rules for every implementer (read before your task) + +- **Build gate:** `TreatWarningsAsErrors=true`, `Nullable=enable`, analyzers at latest. New warnings fail the build — fix them, never suppress. +- **Style:** follow `docs/style-guides/CSharpStyleGuide.md` — file-scoped namespaces, `sealed` by default, `Async` suffix, MXAccess-aligned names. Match the comment density and idiom of the file you're editing. +- **Parity is sacred:** do not change MXAccess-visible semantics (event ordering, `OperationComplete` behavior, write-completion reply shape, per-tag ReadBulk timeout meaning). These tasks change *mechanics* (waits, locks, allocations), never observable protocol behavior, except where a task explicitly says otherwise. +- **Never synthesize events.** Nothing in this plan may fabricate an `MxEvent`. +- **Docs in the same commit:** when a task changes configuration, event mechanics, security behavior, or lifecycle rules, the named docs must be updated in that task's commit. +- **Worker code (Phase B) does not compile on this machine.** `LangVersion=latest` applies, so modern syntax is fine, but only net48-era BCL APIs exist (no `Span`-taking stream overloads, no `ArgumentNullException.ThrowIfNull` — check what the file already uses). Match the existing worker idioms exactly. Verification is Task 24. +- **Tests:** gateway tests use the FakeWorkerHarness (`src/ZB.MOM.WW.MxGateway.Tests`), no MXAccess needed. Run only your task's filter, not the full suite (full suite runs once per phase). +- **Commit after every task**, message style: `perf(): ` (or `fix(...)` for the two correctness bugs). + +Verification commands used throughout: + +```bash +# Gateway build (macOS-safe) +dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx +# Targeted gateway tests +dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj --filter "FullyQualifiedName~" +``` + +--- + +# Phase A — Gateway (local verification) + +### Task 1: Named-pipe buffer sizes + +**Classification:** small +**Estimated implement time:** ~3 min +**Parallelizable with:** Tasks 2, 3, 4, 5, 6, 8, 10, 11, 12, 13, 14 + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionWorkerClientFactory.cs` (`CreatePipe`, ~line 157) +- Modify: `docs/WorkerFrameProtocol.md` (add a short "Pipe buffers" note) +- Test: `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/` (existing factory/e2e tests must stay green; no new test — buffer size isn't observable through the .NET API) + +**Why:** the current 5-arg `NamedPipeServerStream` overload passes `inBufferSize: 0, outBufferSize: 0`. A zero-quota byte-mode pipe forces every write to rendezvous with a pending read — lock-step IPC, and the exact failure class behind the historical windev suite wedge. + +**Step 1: Change the overload** + +```csharp +private const int PipeBufferSizeBytes = 128 * 1024; + +private static NamedPipeServerStream CreatePipe(string pipeName) +{ + return new NamedPipeServerStream( + pipeName, + PipeDirection.InOut, + maxNumberOfServerInstances: 1, + PipeTransmissionMode.Byte, + PipeOptions.Asynchronous, + inBufferSize: PipeBufferSizeBytes, + outBufferSize: PipeBufferSizeBytes); +} +``` + +Add a comment stating *why* (zero-quota rendezvous behavior; reference the windev wedge). Note: on Unix these sizes are advisory (Unix domain socket), which is fine — the fix targets Windows production. + +**Step 2:** `dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx` → 0 errors. +**Step 3:** `dotnet test ... --filter "FullyQualifiedName~GatewayEndToEndFakeWorkerSmokeTests"` → PASS. +**Step 4:** Update `docs/WorkerFrameProtocol.md` with a 3–4 line "Pipe buffers" paragraph. Commit: `perf(ipc): give worker pipes real OS buffers instead of zero-quota rendezvous` + +--- + +### Task 2: Metrics — pull-gauge for worker queue depth, lock-free command counters + +**Classification:** standard +**Estimated implement time:** ~5 min +**Parallelizable with:** Tasks 1, 3, 4, 5, 6, 8, 10, 11, 12, 13, 14 (NOT Task 7 — both edit `WorkerClient.cs`) + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetrics.cs` (`SetWorkerEventQueueDepth` ~290; `CommandStarted/Succeeded/Failed` ~202–247; gauge wiring ~91; snapshot ~461–492) +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs` (call sites ~303 and ~602) +- Test: `src/ZB.MOM.WW.MxGateway.Tests/Metrics/` (extend the existing GatewayMetrics test class) + +**Why:** `SetWorkerEventQueueDepth` takes the process-wide `_syncRoot` twice per event for every session, and the single scalar makes the gauge last-writer-wins across sessions (a correctness bug). The command counters take the same global lock 2–3× per RPC. + +**Step 1 (failing test):** add a test that registers two worker-queue-depth sources reporting 3 and 4 and asserts the snapshot/gauge reports 7; add a test that `CommandStarted`×N from parallel tasks yields exactly N with no lock (behavioral: just correctness of count). + +**Step 2 (implement):** +- Mirror the existing GWC-15 pattern verbatim: add `RegisterWorkerEventQueueDepthSource(Func depth)` returning an `IDisposable` handle, a `ConcurrentDictionary>` of sources, and make `GetWorkerEventQueueDepth` sum the sources (clamp negatives). Delete `SetWorkerEventQueueDepth` and the `_workerEventQueueDepth` field. +- `WorkerClient`: at construction (or first use), register a source returning its staged+channel depth via `Volatile.Read` of a field the stage/consume paths maintain with `Interlocked` — the hot path does **no** metrics call at all anymore. Dispose the registration in `DisposeAsync`. +- Command counters: `_commandsStarted/_commandsSucceeded/_commandsFailed` become `long` updated with `Interlocked.Increment`; `_commandFailuresByMethod` becomes `ConcurrentDictionary` (follow the existing `EventReceived` pattern in the same file). Snapshot reads with `Interlocked.Read`. + +**Step 3:** run the Metrics test filter → PASS. **Step 4:** grep the repo for `SetWorkerEventQueueDepth` — zero hits outside tests you updated. +**Step 5:** Commit: `perf(metrics): pull-model worker queue gauge (fixes last-writer-wins), Interlocked command counters` + +--- + +### Task 3: Distributor — copy-on-write subscriber snapshot + +**Classification:** high-risk (core event fan-out concurrency) +**Estimated implement time:** ~5 min +**Parallelizable with:** Tasks 1, 2, 4, 5, 6, 8, 10, 11, 12, 13, 14 + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs` (pump loop ~600; register/unregister paths; the "snapshot-free enumerator" remark ~71) +- Test: `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/` (existing SessionEventDistributor tests must stay green; add one test if a register-during-pump race test doesn't already exist) + +**Why:** `_subscribers.Values` (the property) locks the whole `ConcurrentDictionary` and materializes a snapshot list **per event**, contradicting the adjacent comment. + +**Step 1 (implement):** maintain a `volatile Subscriber[] _subscriberSnapshot` rebuilt inside the existing registration lock on every register/unregister (the set is tiny and mutates rarely). The pump iterates the array. Keep the dictionary if other paths use keyed lookup; the array is purely the fan-out view. Update the ~71 remark to describe the actual mechanism. Semantics to preserve exactly: a subscriber registered mid-iteration may miss the in-flight event ("late subscribers see events after they register") — the array snapshot preserves this naturally. + +**Step 2:** run the distributor/replay test filters (`FullyQualifiedName~SessionEventDistributor`, `~Replay`) → PASS. The replay-handoff atomicity tests are the critical gate here. +**Step 3:** Commit: `perf(events): copy-on-write subscriber snapshot in fan-out pump` + +--- + +### Task 4: Dashboard event mirror — viewer gating + +**Classification:** standard +**Estimated implement time:** ~5 min +**Parallelizable with:** Tasks 1, 2, 3, 5, 6, 8, 10, 11, 12, 13, 14 + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHub.cs` +- Create: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHubViewerRegistry.cs` +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardEventBroadcaster.cs` (Publish, ~39) +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardServiceCollectionExtensions.cs` (register the registry) +- Modify: `docs/GatewayDashboardDesign.md` (mirror gating paragraph) +- Test: create `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/EventsHubViewerRegistryTests.cs` + extend the existing DashboardEventBroadcaster tests + +**Why:** with `ShowTagValues=false` (default), `Publish` deep-clones every event and dispatches to a SignalR group that is empty in the steady state. No viewer gate exists anywhere on the path. + +**Step 1 (failing test):** broadcaster with zero registered viewers for the session performs **no clone and no send** (assert via a counting fake hub-clients/`IHubContext` seam, matching however the existing broadcaster tests fake SignalR); with one viewer, behavior is unchanged (redacted clone sent). + +**Step 2 (implement):** +- `EventsHubViewerRegistry` (singleton): `ConcurrentDictionary` session→viewer count, `Increment(sessionId)`, `Decrement(sessionId)`, `HasViewers(sessionId)`. Track per-connection subscribed sessions in a `ConcurrentDictionary>` keyed by connection id so `OnDisconnectedAsync` can decrement everything that connection held. +- `EventsHub`: `SubscribeSession`/`UnsubscribeSession` update the registry alongside the group add/remove; override `OnDisconnectedAsync` to release the connection's sessions. Keep the existing SEC-25 remark intact. +- `DashboardEventBroadcaster.Publish`: first line after the null-guards becomes `if (!viewerRegistry.HasViewers(sessionId)) { return; }` — before the redact/clone. +- Do **not** attempt lazy mirror-lease start in this task (it interacts with distributor lifecycle); the gate above removes ~all of the waste already. Note this decision in the doc paragraph. + +**Step 3:** run Dashboard test filter → PASS. **Step 4:** update `docs/GatewayDashboardDesign.md`. Commit: `perf(dashboard): gate event mirror on live viewers — no clone, no send for unwatched sessions` + +--- + +### Task 5: Snapshot pipeline — idle gating, cached config, keyed refresh cadence + +**Classification:** standard +**Estimated implement time:** ~5 min +**Parallelizable with:** Tasks 1, 2, 3, 4, 6, 8, 10, 11, 12, 13, 14 + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotPublisher.cs` (~69–83) +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHub.cs` (connection counting) +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs` (~103 config rebuild, ~163–164 + ~267 API-key refresh) +- Modify: `docs/GatewayDashboardDesign.md` +- Test: extend existing snapshot service/publisher tests under `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/` + +**Why:** the 1 Hz tick runs an API-key SQLite read, a registry sort, a metrics snapshot, and a rebuild of the *static* effective-configuration record, broadcast to `Clients.All`, forever, with zero viewers. + +**Step 1 (failing tests):** (a) effective configuration object is reference-identical across two snapshot builds; (b) API-key summaries refresh at most once per configured interval (inject `TimeProvider`, follow the file's existing time idiom); (c) publisher with zero connections does not enumerate the snapshot source (fake the hub context; count pulls). + +**Step 2 (implement):** +- Cache `EffectiveGatewayConfiguration` in a field on first build (it's startup-static; add a comment saying so). +- `RefreshApiKeySummariesAsync`: skip unless `RefreshInterval` (new private constant, 15 s) has elapsed since the last successful refresh. +- `DashboardSnapshotHub`: `OnConnectedAsync`/`OnDisconnectedAsync` maintain an `int` connection count on a small singleton (or reuse the Task 4 registry class with a well-known key — implementer's choice, keep it simple). Publisher checks the count each tick: zero connections → `await Task.Delay(interval)` and skip both the snapshot build and the broadcast. First connection after idle gets a fresh snapshot on its next tick (≤1 interval of staleness — acceptable; pages also seed from `IDashboardSnapshotService` directly on load). + +**Step 3:** dashboard test filter → PASS. Docs paragraph. Commit: `perf(dashboard): idle-gate the snapshot tick; cache static config; bound key-list refresh` + +--- + +### Task 6: Reply ownership transfer in `MapCommandReply` + +**Classification:** small +**Estimated implement time:** ~3 min +**Parallelizable with:** Tasks 1, 2, 3, 4, 5, 8, 10, 11, 12, 13, 14 + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGrpcMapper.cs` (~74) +- Test: existing mapper/service tests under `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/` + +**Why:** every `WorkerCommandReply` is parsed fresh from one pipe frame and completed to exactly one awaiter; the gRPC handler is its only consumer. Events already got this treatment under GWC-07 — replies still deep-copy, which doubles the largest hot-path message on bulk reads. + +**Step 1 (verify precondition, in-code):** confirm (grep) that no caller of `WorkerClient.InvokeAsync` retains `reply.Reply` after mapping — the review found the Invoke path clean; `GatewayAlarmMonitor` and `DashboardLiveDataService` own their separate replies. If you find a second consumer, STOP and surface it — that's a plan defect. + +**Step 2 (implement):** `return reply.Reply.Clone();` → `return reply.Reply;` with a GWC-07-style ownership comment: the worker reply object is single-consumer by construction (one frame → one `PendingCommand` completion → one mapper call); the mapper transfers ownership to the gRPC response. + +**Step 3:** run `FullyQualifiedName~MxAccessGrpcMapper` + the fake-worker smoke filter → PASS. Commit: `perf(grpc): transfer reply ownership instead of deep-cloning every worker reply` + +--- + +### Task 7: WorkerClient — pooled-timer timeout, single sizing pass, `WorkerCancel` on timeout + +**Classification:** high-risk (IPC concurrency + protocol behavior) +**Estimated implement time:** ~5 min +**Parallelizable with:** Tasks 3, 4, 5, 8, 10, 11, 12, 13, 14 (NOT Task 2 — both edit `WorkerClient.cs`; run after Task 2) + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs` (InvokeAsync ~226–270; timeout path) +- Modify: `docs/GatewayProcessDesign.md` (command timeout → cancel-forwarding note) +- Test: extend `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/` worker-client tests (the fakes there already exercise timeout paths) + +**Why:** each Invoke churns a linked CTS + `Task.Delay` timer + `WhenAny`; `CalculateSize` runs twice (protobuf doesn't memoize); and on timeout the gateway never tells the worker, so a timed-out COM call keeps occupying the STA and an envelope still queued gets written anyway. + +**Step 1 (failing test):** on command timeout, the client enqueues a `WorkerCancel` envelope carrying the timed-out correlation id (assert via the fake connection's written-frame log). + +**Step 2 (implement):** +- Replace the CTS/Delay/WhenAny block with `await pendingCommand.Task.WaitAsync(timeout, cancellationToken)` wrapped in a `try/catch (TimeoutException)` / `(OperationCanceledException)` mapping to the exact same `WorkerClientErrorCode`s and messages as today (tests depend on them). +- On the timeout path, after `RemovePendingCommandAsFailed`, best-effort enqueue a `WorkerCancel` envelope for the correlation id (fire-and-forget with a swallow-and-log; never let cancel failure mask the timeout exception). The worker already handles `WorkerCancel` (`WorkerPipeSession` → `CancelCommand`). +- Thread the already-computed `envelopeSize` into the frame write path if the writer API allows passing a known size; if the writer's public surface would have to change more than trivially, skip this sub-item and leave a `// PERF:` note — the timer and cancel fixes carry the task. + +**Step 3:** worker-client test filter → PASS, including existing timeout tests unchanged. Docs note. Commit: `perf(ipc): WaitAsync command timeouts + forward WorkerCancel so a timed-out COM call frees the STA` + +--- + +### Task 8: Audit pipeline — startup bootstrap, background writer, retention + +**Classification:** high-risk (security/audit semantics) +**Estimated implement time:** ~5 min (split if it runs long: 8a writer, 8b retention) +**Parallelizable with:** Tasks 1, 2, 3, 4, 5, 6, 7, 10, 11, 12, 13, 14 + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Audit/SqliteCanonicalAuditStore.cs` (per-op `EnsureTableAsync` ~52–54, ~94, ~131–136) +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Audit/CanonicalAuditWriter.cs` (~35) +- Create: `src/ZB.MOM.WW.MxGateway.Server/Security/Audit/ChannelAuditWriter.cs` (bounded channel + hosted drain) +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/AuthStoreServiceCollectionExtensions.cs` (DI wiring + hosted service) +- Modify: `docs/DesignDecisions.md` (audit is asynchronous best-effort, bounded, with retention) +- Test: create `src/ZB.MOM.WW.MxGateway.Tests/Security/Audit/ChannelAuditWriterTests.cs` + +**Why:** constraint denials await a SQLite insert inline per denied tag inside bulk RPC loops — sequential round-trips into the same DB file the auth store uses, each with a redundant `CREATE TABLE IF NOT EXISTS`, into a table with no retention. + +**Step 1 (failing tests):** (a) `WriteAsync` returns without touching the store (enqueue-only) and the event lands in the store shortly after (drain); (b) when the bounded channel (capacity 4096) is full, `WriteAsync` drops (oldest or newest — pick drop-write/newest for simplicity) and increments a counter, never blocks; (c) retention sweep deletes rows older than the configured window. + +**Step 2 (implement):** +- `ChannelAuditWriter : ICanonicalAuditWriter` (or whatever the current writer interface is named — read `CanonicalAuditWriter.cs` first): bounded `Channel` (`BoundedChannelFullMode.DropWrite`), a `BackgroundService` drain that batches up to 64 events into one transaction per drain pass. The audit contract is already documented best-effort — say so in the class doc. +- Table bootstrap: run `EnsureTableAsync` once from the drain service's `StartAsync` (and from the store's first list call via a `Lazy`/latch); remove the per-insert and per-list calls. +- Retention: in the same drain service, once per hour, `DELETE FROM audit_event WHERE timestamp < now - RetentionDays` (new `SecurityOptions`/audit option, default 90 days, validated ≥1 in `GatewayOptionsValidator`); document in `docs/GatewayConfiguration.md`. +- Wire DI so `ConstraintEnforcer.RecordDenialAsync` transparently goes through the channel writer — **no signature changes** at the enforcer/service layer. +- Flush-on-shutdown: drain the channel in `StopAsync` with a 2 s cap. + +**Step 3:** audit test filter + `FullyQualifiedName~ConstraintEnforcer` → PASS. Docs (`DesignDecisions.md`, `GatewayConfiguration.md`). Commit: `perf(audit): bounded async audit writer with batched inserts, one-time bootstrap, retention sweep` + +--- + +### Task 9: Parallel session teardown in sweep and shutdown + +**Classification:** high-risk (lifecycle concurrency) +**Estimated implement time:** ~4 min +**Parallelizable with:** Tasks 10, 11, 12, 13, 14 (edits only `SessionManager.cs` + docs; run any time) + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs` (`CloseExpiredLeasesAsync` ~256–296, `ShutdownAsync` ~301–329) +- Modify: `docs/Sessions.md` (teardown parallelism note) +- Test: extend `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/` session-manager tests + +**Why:** both loops `await CloseSessionCoreAsync` strictly sequentially, each bounded by the 10 s worker-shutdown timeout — a mass expiry with hung workers stalls slot reclamation, and 50-session shutdown exceeds any host stop-timeout. + +**Step 1 (failing test):** two sessions whose fake worker shutdowns each take T complete a sweep in ~T, not ~2T (the fake harness supports delayed shutdown; if not, add a delay knob to the fake). + +**Step 2 (implement):** wrap both loops in `Parallel.ForEachAsync` with `MaxDegreeOfParallelism = 4` (named constant, comment why: bounded so a mass expiry can't stampede worker teardown). `TryBeginCloseIfExpired` already makes per-session close idempotent/exclusive — state that in a comment; that's the invariant making this safe. Preserve the existing sweep precedence (lease-expiry → faulted → detach-grace) by keeping the *selection* phase sequential and parallelizing only the close calls on the selected set. + +**Step 3:** session-manager filter → PASS. Docs. Commit: `perf(sessions): bounded-parallel teardown in lease sweep and shutdown` + +--- + +### Task 10: Dashboard live-data subscription cap + +**Classification:** standard +**Estimated implement time:** ~4 min +**Parallelizable with:** Tasks 1–9, 11, 12, 13, 14 + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardLiveDataService.cs` (~61–70, `_subscribed`) +- Modify: `docs/GatewayDashboardDesign.md` +- Test: extend existing live-data tests under `src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/` + +**Why:** every tag any viewer ever inspected stays advised on the shared worker session forever. + +**Step 1 (failing test):** subscribing tag #257 when the cap is 256 unsubscribes the least-recently-read tag first (assert the fake session sees an `UnsubscribeBulk`/equivalent for the evicted tag). + +**Step 2 (implement):** replace `_subscribed` (set) with an LRU: `Dictionary>` + `LinkedList` under the existing `_gate` (already serialized — no new locking). Cap at 256 (named constant; comment the sizing rationale: one browse page of tags plus headroom). On read of an already-subscribed tag, move to front. On insert past cap, evict from the back and call the session's unsubscribe for the evicted batch. On `InvalidateSession`, clear both structures (existing behavior). + +**Step 3:** dashboard filter → PASS. Docs. Commit: `perf(dashboard): LRU cap on the shared live-read session's advised set` + +--- + +### Task 11: Alarm monitor — cached `CurrentAlarms` projection + +**Classification:** small +**Estimated implement time:** ~3 min +**Parallelizable with:** Tasks 1–10, 12, 13, 14 + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Alarms/GatewayAlarmMonitor.cs` (~90–99 + every mutation site under `_sync`) +- Test: extend `src/ZB.MOM.WW.MxGateway.Tests/Alarms/` monitor tests + +**Why:** `CurrentAlarms` clones the full alarm set under the broadcast lock on every call. + +**Step 1 (failing test):** two consecutive `CurrentAlarms` calls with no intervening transition return the same cached array instance; a transition invalidates it. + +**Step 2 (implement):** add `private IReadOnlyList? _currentAlarmsCache;` — `CurrentAlarms` builds it (still cloning, still under `_sync`) only when null; every mutation path that touches the alarm dictionary (`ApplyTransition`, reconcile apply, clear) nulls it under `_sync`. Callers already treat the result as read-only. + +**Step 3:** alarms filter → PASS. Commit: `perf(alarms): memoize CurrentAlarms projection, invalidate on mutation` + +--- + +### Task 12: Request-logging middleware — hoisted logger, bearer redaction fix + +**Classification:** small (contains a security fix) +**Estimated implement time:** ~4 min +**Parallelizable with:** Tasks 1–11, 13, 14 + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GatewayRequestLoggingMiddlewareExtensions.cs` (~29–38) +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GatewayLogRedactor.cs` (~54–77) +- Test: extend `src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/` redactor tests + +**Why:** `CreateLogger` (factory lock + DI resolve) per request; and — the security half — `RedactClientIdentity` passes any bearer credential that doesn't contain `mxgw_` through **unredacted** into log scope, violating the "never log secrets" convention. + +**Step 1 (failing test):** `RedactClientIdentity("Bearer eyJhbGciOi...")` (a non-mxgw token) returns a redacted form (e.g. `Bearer [redacted]`), never the raw token. Keep the existing mxgw-shaped redaction (`mxgw__***`) intact — those tests must still pass. + +**Step 2 (implement):** +- Redactor: any `authorization`-style value that is not recognized as an mxgw key redacts to a fixed `"[redacted]"` (preserve scheme word only). This is fail-closed. +- Middleware: resolve the `ILogger` once outside the per-request lambda (category-keyed, not request-keyed) via the app's `ILoggerFactory` at `Use...` registration time; keep the scope construction as-is (it carries per-request fields the log pipeline consumes — do not conditionalize it on log level in this task; note as considered-and-skipped since scope consumers may be added at runtime). + +**Step 3:** diagnostics filter → PASS. Commit: `fix(logging): fail-closed bearer redaction; hoist per-request logger creation` + +--- + +### Task 13: Auth-path hygiene — span token parse, limiter partition keys + +**Classification:** small +**Estimated implement time:** ~4 min +**Parallelizable with:** Tasks 1–12, 14 + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcAuthorizationInterceptor.cs` (~153 `TryResolveKeyId`) +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/CachingApiKeyVerifier.cs` (~229 `TryParseKeyId`) +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs` (~244–269, ~398) +- Test: existing auth tests under `src/ZB.MOM.WW.MxGateway.Tests/Security/` must stay green; add parse-equivalence cases + +**Step 1 (failing test):** parse-equivalence table test: for a set of tokens (well-formed, missing `_`, empty, extra `_`), the new span parser returns exactly what `Split('_')` logic returned. + +**Step 2 (implement):** replace `Split('_')` in both parsers with `IndexOf('_')` twice over a `ReadOnlySpan`/string (no arrays, no substrings until the final key-id slice). In the limiter, compute the composite partition key once per RPC and pass it to both `Check` and `Reset` (or add an overload taking the precomputed key) instead of concatenating twice. + +**Step 3:** security filter → PASS. Commit: `perf(auth): allocation-free token parsing; single partition-key build per RPC` + +--- + +### Task 14: Bulk constraint loops, caches, and per-call hygiene + +**Classification:** standard +**Estimated implement time:** ~5 min +**Parallelizable with:** Tasks 1–13 (NOT Task 6 if the mapper edit collides — it doesn't; different files) + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs` (bulk loops ~466–troughs at 494/551/612/680; double session resolve ~104/126) +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ConstraintEnforcer.cs` (~214–215 LINQ; expose `HasReadConstraints`/`HasWriteConstraints` if not present) +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/GatewayApiKeyIdentityMapper.cs` (~39–42 cache cliff) +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Sessions/SparseArrayExpander.cs` (~123–126 capacity hints) +- Test: existing constraint/service tests under `src/ZB.MOM.WW.MxGateway.Tests/` + one new eviction test + +**Step 1 (failing test):** constraint-blob cache: inserting entry `MaxCachedConstraintBlobs + 1` evicts the oldest instead of refusing to cache (FIFO like `GalaxyGlobMatcher` — copy its idiom). + +**Step 2 (implement):** +- Bulk loops: hoist a single `identity has no read/write constraints` check before each per-item loop → unconstrained keys take an O(1) fast path (no per-item async interface dispatch, no denial bookkeeping allocation). +- Glob matching: replace the two `.Any(lambda)` calls with `for` loops over the glob lists. +- Denied-path double clone: build the filtered command directly (new message, copy allowed entries in) instead of `command.Clone()` then clear-and-refill; `MapCommand`'s own clone stays (that one is the load-bearing no-aliasing copy). +- Session double-resolve: add/`use` a `SessionManager` overload accepting the already-resolved `GatewaySession` (or have the service pass the session it resolved); keep the not-found exception behavior identical. +- `SparseArrayExpander`: set `RepeatedField.Capacity = length` (per element type) before the fill loops. + +**Step 3:** run `FullyQualifiedName~ConstraintEnforcer`, `~MxAccessGatewayService`, `~SparseArray` filters → PASS. Commit: `perf(grpc): O(1) unconstrained bulk fast path, direct filtered-command build, cache eviction, capacity hints` + +--- + +### Task 15: Phase A gate — full gateway suite + +**Classification:** trivial (verification only) +**Estimated implement time:** ~5 min wall (suite runtime) +**Parallelizable with:** none (runs after Tasks 1–14) + +Run, in order: + +```bash +dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx +dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj +``` + +Expected: 0 build errors, full suite green, clean process exit (0 surviving testhost). Fix anything red before Phase B. Commit only if fixes were needed. + +--- + +# Phase B — Worker (.NET Framework 4.8; verified on windev in Task 24) + +> Phase B implementers: you cannot compile. Be conservative — minimal diffs, match file idioms, net48 BCL only. Every task here lands as an unverified commit that Task 24 builds and tests remotely; keep commits clean so a failure bisects trivially. + +### Task 16: Event drain loop — wake signal instead of 25 ms poll + +**Classification:** high-risk (event path liveness) +**Estimated implement time:** ~5 min +**Parallelizable with:** Tasks 18, 19, 21, 22, 23 + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs` (add wake handle; `Enqueue` sets it) +- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs` (~18 `EventDrainInterval`, ~345–372 drain loop) +- Modify: `docs/MxAccessWorkerInstanceDesign.md` (drain-loop paragraph ~381) +- Test: extend `src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/` event-queue tests + `Ipc/` pipe-session tests (they run on windev) + +**Why:** the drain loop polls at 25 ms with no wake from `Enqueue` — a 25 ms latency floor on every burst from idle, 40 wakeups/s per idle worker, and less burst absorption before the 10k queue faults the session. + +**Implement:** +- `MxAccessEventQueue`: add a `SemaphoreSlim _signal = new(0, 1)` (or an `AsyncAutoResetEvent`-shaped helper if the codebase has one — check first). `Enqueue` releases it (cap at 1, swallow `SemaphoreFullException`). Expose `Task WaitForEventsAsync(TimeSpan timeout, CancellationToken ct)`. +- Drain loop: when a drain returns empty, `await queue.WaitForEventsAsync(EventDrainInterval, ct)` instead of `Task.Delay` — the 25 ms becomes a *fallback* ceiling, not the floor; a signaled wait returns immediately. Loop structure otherwise unchanged (fault handling, batch size). +- Doc paragraph: drain is signal-driven with a 25 ms fallback tick. +- Tests: enqueue-after-idle results in a drain without waiting for the fallback interval (windev-run; write it now). + +Commit: `perf(worker): signal-driven event drain — removes the 25 ms latency floor and idle wakeups` + +--- + +### Task 17: Event queue capacity — launcher-configurable + +**Classification:** standard +**Estimated implement time:** ~5 min +**Parallelizable with:** Tasks 18, 19, 21, 22, 23 (NOT Task 16 — both edit `MxAccessEventQueue.cs`/`WorkerPipeSession.cs`; run after 16) + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Configuration/WorkerOptions.cs` (+`EventQueueCapacity`, default 10000) and `GatewayOptionsValidator.cs` (≥1000, ≤1_000_000) +- Modify: `src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerProcessLauncher.cs` (new env var, mirror the `WorkerWriteCompletionWaitEnvironmentVariableName` pattern at ~25–29 and ~186–187 exactly) +- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Bootstrap/WorkerOptionsParser.cs` / `WorkerOptions.cs` / `EnvironmentVariableWorkerEnvironment.cs` (read it, following the write-completion variable's path) +- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs` (~52: pass capacity to `new MxAccessEventQueue(...)`) +- Modify: `docs/GatewayConfiguration.md` (+`MxGateway:Worker:EventQueueCapacity`), `docs/MxAccessWorkerInstanceDesign.md` (capacity paragraph ~371) +- Test: gateway side — validator test + launcher env-var test (these run locally); worker side — parser test (windev) + +**Why:** the 10,000 default is headroom-critical (overflow faults the session) but not configurable without a rebuild. + +**Implement:** copy the `WriteCompletionWaitMilliseconds` plumbing end to end under a new name (`MXGW_EVENT_QUEUE_CAPACITY` shaped like the existing variable's naming). Absent/invalid env value → default 10000 (never crash the worker on a bad value; log and default). + +Note the gateway-side files here don't overlap Phase A tasks — safe after Task 15. + +Commit: `perf(worker): launcher-configurable event queue capacity` + +--- + +### Task 18: STA completion waits — message-driven, not sleep-polled + +**Classification:** high-risk (STA/pump semantics) +**Estimated implement time:** ~5 min +**Parallelizable with:** Tasks 16, 17, 19, 21, 22, 23 + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessWriteCompletionCache.cs` (~97–118 wait loop) +- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessValueCache.cs` (~135–150 wait loop) +- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Sta/StaMessagePump.cs` (if it doesn't already expose a bounded "pump until signaled or timeout" primitive) +- Modify: `docs/MxAccessWorkerInstanceDesign.md` +- Test: extend `src/ZB.MOM.WW.MxGateway.Worker.Tests/Sta/` + `MxAccess/` cache tests (windev) + +**Why:** both waits run `pumpStep(); ...; Thread.Sleep(5)` on the STA — during each 5 ms sleep no messages pump, so COM event dispatch stalls in 5 ms bites for up to 1.5 s (writes) / 1 s per tag (ReadBulk). + +**Implement:** +- Add a wake to both caches: the update path (`OnWriteComplete` recording a completion / `OnDataChange` recording a value) signals a Win32 auto-reset event (`AutoResetEvent` is fine — it wraps one). +- Replace `Thread.Sleep(pollIntervalMs)` with a pump-integrated wait: `MsgWaitForMultipleObjectsEx(1, [waitHandle], remainingMs-capped-at-50, QS_ALLINPUT, MWMO_INPUTAVAILABLE)`; on `WAIT_OBJECT_0 + 1` (message arrived) run `pumpStep()` and re-check; on `WAIT_OBJECT_0` (signaled) re-check the entry immediately. The existing `StaMessagePump`/`StaRuntime` already use exactly this Win32 pattern (~`StaRuntime.cs:255–261`) — reuse/extract their P/Invoke declarations, do not duplicate. +- **Semantics unchanged:** timeouts, deadline math, return values, and the unconfirmed-empty-statuses reply shape stay byte-identical. Only the *waiting mechanism* changes: latency to observe a completion drops from ≤5 ms granularity to immediate, and the pump keeps running throughout the wait. +- **Do not** change the plain-`Write` completion-wait default in this task. The 1.5 s default is a documented OtOpcUa contract (`MxGateway:Worker:WriteCompletionWaitMilliseconds` is already configurable). Leave a doc note that operators with pure fire-and-forget write workloads can lower it. + +Commit: `perf(worker): message-driven completion waits — the STA pumps continuously while waiting` + +--- + +### Task 19: Handle registry — reverse index, cached views, O(1) removals + +**Classification:** standard +**Estimated implement time:** ~5 min +**Parallelizable with:** Tasks 16, 17, 18, 21, 22, 23 + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessHandleRegistry.cs` (`ItemHandles`/`ServerHandles`/`AdviceHandles` properties ~14–26; `RemoveAdviceHandles` ~137–148; `UnregisterServerHandle` ~46–65) +- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessSession.cs` (`TryGetCachedReadFor` ~988–1000) +- Test: extend `src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/` registry tests (windev) + +**Why:** the sorted list properties re-sort and copy the whole table on **every access**, `TryGetCachedReadFor` reads `ItemHandles` once per ReadBulk tag (O(tags × items·log items)), and advice/server removals do full LINQ scans (O(n²) bulk teardown). + +**Implement:** +- Reverse index: `Dictionary>` server→(tagAddress→itemHandle) — or flat `Dictionary<(int,int-packed + tag)>` — maintained on register/unregister. `TryGetCachedReadFor` becomes two dictionary probes (the file's own comment already asks for this). +- Cached materialization: memoize each sorted array with a version stamp bumped on any mutation; property returns the cached array when the version matches. Registry is STA-confined (verify: no locking in the file today ⇒ single-threaded by contract — state it in a comment), so no locking needed. +- Removals: secondary index advice-by-item (`Dictionary>` keyed on the packed `(serverHandle, itemHandle)` the item table already uses) so `RemoveAdviceHandles`/`UnregisterServerHandle` stop scanning. + +Commit: `perf(worker): reverse tag index + memoized views + indexed removals in the handle registry` + +--- + +### Task 20: Event conversion — exact-format timestamps, compiled status accessors + +**Classification:** standard +**Estimated implement time:** ~5 min +**Parallelizable with:** Tasks 16, 17, 18, 19, 21, 22, 23 (different files) + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventMapper.cs` (~360–377 timestamp parse) +- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Conversion/MxStatusProxyConverter.cs` (~96–109 reflection reads) +- Test: extend `src/ZB.MOM.WW.MxGateway.Worker.Tests/Conversion/` (windev) — these files have solid existing tests; add exact-format cases + +**Implement:** +- Timestamps: try `DateTime.TryParseExact` against a small cached array of the observed MXAccess formats (`M/d/yyyy h:mm:ss.fff tt` and its zero-padded/24 h siblings — derive the list from the existing tests' fixture strings) **first**, falling back to the existing two-stage `TryParse` chain so behavior never regresses on an unexpected locale. Order: exact formats → current-culture → invariant (today's chain). +- Status fields: replace the per-read `field.GetValue` with delegates compiled once per field via `Expression.Lambda>` (net48-safe) cached alongside the existing `FieldInfo` cache. Same values out, no boxing per event. + +Commit: `perf(worker): exact-format timestamp parse and compiled status-field accessors on the event path` + +--- + +### Task 21: Event queue drain — size memoized at enqueue + +**Classification:** standard +**Estimated implement time:** ~4 min +**Parallelizable with:** Tasks 18, 19, 20, 22, 23 (NOT 16/17 — same file; run after them) + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs` (~249–269 byte-budgeted `Drain`; enqueue path ~152–170) +- Test: extend the windev event-queue tests: budget math unchanged for a mixed-size batch + +**Why:** `Drain(maxEvents, maxTotalBytes)` calls `CalculateSize()` per event **inside** the queue lock the STA needs to enqueue — a large drain stalls COM callbacks. + +**Implement:** compute `CalculateSize()` once at enqueue time (outside any lock — the caller owns the event exclusively there) and store it on the queue's node/wrapper alongside the event; `Drain` uses the memoized size. The WRK-21 never-strand-the-head guarantee is untouched (same comparisons, precomputed operand). Events are never mutated after enqueue (WRK-11 no-clone contract) so the memoized size cannot go stale — say so in a comment. + +Commit: `perf(worker): memoize event frame size at enqueue; drain stops sizing under the STA's lock` + +--- + +### Task 22: Worker frame writer/reader — pooled buffers + +**Classification:** standard +**Estimated implement time:** ~5 min +**Parallelizable with:** Tasks 16, 17, 18, 19, 20, 21, 23 + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWriter.cs` (~467 per-frame `new byte[]`) +- Modify: `src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameReader.cs` (~33 per-frame prefix buffer) +- Test: windev `Ipc/` frame tests must stay green (they're thorough — rely on them) + +**Why:** the worker side allocates a fresh frame buffer + prefix buffer per frame while the gateway side already pools (`ArrayPool`, GWC-30) — the fix was applied on one side only. `System.Buffers` is already referenced by the worker (its reader uses `ArrayPool.Shared`). + +**Implement:** mirror the gateway codec: rent the frame buffer from `ArrayPool.Shared`, write prefix+payload into it, return in a `finally`; hoist the 4-byte prefix buffer to an instance field on the reader (single-reader by contract — copy the gateway reader's comment). Exact same wire bytes. + +Commit: `perf(worker): pooled frame buffers — brings the net48 codec up to the gateway side's GWC-30 pattern` + +--- + +### Task 23: Alarm consumer — cheap parse, truncation detection, configurable cadence + +**Classification:** high-risk (alarm correctness) +**Estimated implement time:** ~5 min (split 23a parse / 23b truncation+config if long) +**Parallelizable with:** Tasks 16, 17, 19, 20, 21, 22 + +**Files:** +- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/WnWrapAlarmConsumer.cs` (~402–437 parse; ~50 `DefaultMaxAlarmsPerFetch`; ~323–330 snapshot rebuild; `ComputeTransitions` absence rule ~356) +- Modify: `src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs` (~22 hard-coded 500 ms) +- Modify: `docs/GatewayConfiguration.md`, `docs/DesignDecisions.md` (alarm sections) +- Test: extend windev `MxAccess/` alarm-consumer tests — the truncation test is the important one + +**Implement (three independent sub-changes):** +1. **Parse cost:** in the per-alarm extraction, replace the ~14 `SelectSingleNode(child)` XPath calls with one pass over `alarmNode.ChildNodes` switching on `Name` (same fields, same defaults for absent children). Keep `XmlDocument` (an `XmlReader` rewrite is a bigger change than the win justifies once XPath is gone). Reuse the snapshot dictionary across polls (clear-and-refill → swap two dictionaries) only if trivially safe; otherwise skip — the XPath removal is the payload. +2. **Truncation cliff (correctness fix):** when the fetch returns exactly `maxAlarmsPerFetch` records, treat the snapshot as **truncated**: log a warning (rate-limited, identifiers only) and suppress the absence-implies-Clear inference in `ComputeTransitions` for that poll (present alarms still update; nothing is cleared on the evidence of a capped fetch). Add the test: 1024-record fetch + a known alarm missing from it → no Clear transition emitted, warning logged. +3. **Cadence + cap configurable:** plumb `MxGateway:Alarms:PollIntervalMilliseconds` (default 500, min 100) and `MaxAlarmsPerFetch` (default 1024) through the existing env-var pattern (as in Task 17). Gateway-side option + validator + launcher env, worker-side parse. + +Commit: `fix(alarms): truncation-safe transitions; perf: single-pass alarm parse; configurable poll cadence` + +--- + +### Task 24: Phase B verification on windev (psbridge) + +**Classification:** high-risk (this is the gate for every Phase B commit) +**Estimated implement time:** ~10 min wall +**Parallelizable with:** none (after all Phase B tasks) + +**Steps:** +1. Invoke the `psbridge` skill and follow it (it covers exec/push/deploy against the Windows box). +2. Push/pull the branch to windev (whatever the skill's established flow is — the repo has a remote the Windows box shares; `git pull` the branch there). +3. On windev, run in order and capture output: + ```powershell + dotnet build src/ZB.MOM.WW.MxGateway.slnx + dotnet build src/ZB.MOM.WW.MxGateway.Worker/ZB.MOM.WW.MxGateway.Worker.csproj -p:Platform=x86 + dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86 + dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj + ``` +4. Any failure: fix on the Mac, commit, re-run the failed leg. Bisect by commit if the failure isn't obvious — Phase B commits are deliberately one-task-each. +5. If psbridge is unreachable: STOP and report — Phase B remains "edited, unverified"; do not merge. + +Live MXAccess smoke (`MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`, `WorkerLiveMxAccessSmokeTests`) if provider state is available on windev; otherwise record why skipped, per `docs/GatewayTesting.md`. + +--- + +### Task 25: Wrap-up — docs sweep, umbrella index, review deltas + +**Classification:** small +**Estimated implement time:** ~5 min +**Parallelizable with:** none (last) + +**Files:** +- Verify each task's doc edits landed (`gateway.md`, `docs/Sessions.md`, `docs/GatewayConfiguration.md`, `docs/GatewayDashboardDesign.md`, `docs/DesignDecisions.md`, `docs/MxAccessWorkerInstanceDesign.md`, `docs/WorkerFrameProtocol.md`) +- Modify: `../scadaproj/CLAUDE.md` — **only if** a fact the umbrella index records changed (new `MxGateway:Worker:EventQueueCapacity` / alarm options are config, not indexed facts; expected outcome: no umbrella change needed — verify, don't assume) +- Check: no `.proto` diffs (`git diff main -- '*.proto'` must be empty) + +Commit anything found: `docs: remediation plan doc sweep` + +--- + +## Explicitly deferred (decided, not forgotten) + +| Finding | Why deferred | +|---|---| +| Value-cache triple clone per `OnDataChange` | Removing the defensive copies needs a GWC-07-style aliasing audit across cache consumers; risk outweighs the win until profiled. | +| net48 pipe-read cancellation | Benign in practice (worker exits after shutdown); a correct fix means restructuring stream teardown for a path that only fires at exit. | +| Control-frame completion coupled to event batch drain | Documented, bounded (≤128 frames) behavior of the two-class writer design; revisit only if heartbeat latency shows up in metrics. | +| Blazor pages' loopback SignalR hop | Works correctly; in-process `WatchSnapshotsAsync` consumption is a dashboard refactor with payoff only at viewer counts the product doesn't target. | +| Event-path triple async-iterator flattening | LOW-rated; touches the most invariant-dense code in the gateway for two `MoveNextAsync` hops per event. Reconsider after Tasks 3/4 land and if profiling still shows it. | + +## Execution notes for the orchestrator + +- Branch: `git checkout -b perf/review-remediation` before Task 1. +- Implementer subagents run on **Opus** per the user's instruction; reviewer chain per each task's Classification. +- Parallel dispatch waves (no file overlap): **Wave 1:** 1, 3, 4, 5, 6, 8 · **Wave 2:** 2, 9, 10, 11, 12, 13, 14 · then 7 (after 2) · then 15 · **Wave 3 (Phase B):** 16, 18, 19, 20, 22, 23 · then 17, 21 (after 16) · then 24 · then 25. (Waves are a suggestion; the per-task `Parallelizable with` fields are the contract.) +- Each implementer gets: its full task text, the ground rules block, and nothing else — the `Files:` block is the scope contract. diff --git a/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json b/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json new file mode 100644 index 0000000..44e8522 --- /dev/null +++ b/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json @@ -0,0 +1,31 @@ +{ + "planPath": "docs/plans/2026-08-15-perf-review-remediation.md", + "tasks": [ + {"id": 1, "subject": "Task 1: Named-pipe buffer sizes", "status": "pending"}, + {"id": 2, "subject": "Task 2: Metrics pull-gauge + Interlocked counters", "status": "pending"}, + {"id": 3, "subject": "Task 3: Distributor copy-on-write subscriber snapshot", "status": "pending"}, + {"id": 4, "subject": "Task 4: Dashboard event mirror viewer gating", "status": "pending"}, + {"id": 5, "subject": "Task 5: Snapshot pipeline idle gating + cached config", "status": "pending"}, + {"id": 6, "subject": "Task 6: Reply ownership transfer in MapCommandReply", "status": "pending"}, + {"id": 7, "subject": "Task 7: WorkerClient WaitAsync timeout + WorkerCancel", "status": "pending", "blockedBy": [2]}, + {"id": 8, "subject": "Task 8: Audit pipeline background writer + retention", "status": "pending"}, + {"id": 9, "subject": "Task 9: Parallel session teardown", "status": "pending"}, + {"id": 10, "subject": "Task 10: Dashboard live-data subscription cap", "status": "pending"}, + {"id": 11, "subject": "Task 11: Alarm monitor cached CurrentAlarms", "status": "pending"}, + {"id": 12, "subject": "Task 12: Logging middleware hoist + bearer redaction fix", "status": "pending"}, + {"id": 13, "subject": "Task 13: Auth-path span parsing + limiter keys", "status": "pending"}, + {"id": 14, "subject": "Task 14: Bulk constraint loops, caches, hygiene", "status": "pending"}, + {"id": 15, "subject": "Task 15: Phase A gate — full gateway suite", "status": "pending", "blockedBy": [1,2,3,4,5,6,7,8,9,10,11,12,13,14]}, + {"id": 16, "subject": "Task 16: Event drain wake signal", "status": "pending", "blockedBy": [15]}, + {"id": 17, "subject": "Task 17: Event queue capacity env plumbing", "status": "pending", "blockedBy": [16]}, + {"id": 18, "subject": "Task 18: STA message-driven completion waits", "status": "pending", "blockedBy": [15]}, + {"id": 19, "subject": "Task 19: Handle registry reverse index + O(1) removals", "status": "pending", "blockedBy": [15]}, + {"id": 20, "subject": "Task 20: Event conversion TryParseExact + compiled accessors", "status": "pending", "blockedBy": [15]}, + {"id": 21, "subject": "Task 21: Drain size memoized at enqueue", "status": "pending", "blockedBy": [16, 17]}, + {"id": 22, "subject": "Task 22: Worker frame writer/reader pooled buffers", "status": "pending", "blockedBy": [15]}, + {"id": 23, "subject": "Task 23: Alarm consumer parse + truncation + cadence", "status": "pending", "blockedBy": [15]}, + {"id": 24, "subject": "Task 24: Phase B verification on windev (psbridge)", "status": "pending", "blockedBy": [16,17,18,19,20,21,22,23]}, + {"id": 25, "subject": "Task 25: Wrap-up docs sweep + umbrella check", "status": "pending", "blockedBy": [24]} + ], + "lastUpdated": "2026-08-15T00:00:00Z" +} From da8463534b030b10b5cc6b39e057be9ad0ddf5b0 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:02:43 -0400 Subject: [PATCH 02/41] perf(ipc): give worker pipes real OS buffers instead of zero-quota rendezvous --- docs/WorkerFrameProtocol.md | 11 ++++++++++ .../Sessions/SessionWorkerClientFactory.cs | 21 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/docs/WorkerFrameProtocol.md b/docs/WorkerFrameProtocol.md index b623707..36fe955 100644 --- a/docs/WorkerFrameProtocol.md +++ b/docs/WorkerFrameProtocol.md @@ -164,6 +164,17 @@ class drains both queues to empty, and the heartbeat loop guarantees one arrives within a heartbeat interval, so worst-case residency is a few envelope references for seconds — not a leak. +## Pipe Buffers + +The gateway creates each worker pipe with an explicit 128 KiB kernel buffer per +direction (`SessionWorkerClientFactory.PipeBufferSizeBytes`) rather than the zero +quota the short `NamedPipeServerStream` overloads request. A zero-quota byte-mode +pipe makes every write rendezvous with a pending read, so a writer with no reader +parked blocks until one arrives — the failure class behind the historical windev +full-suite wedge. A real quota decouples writer latency from reader scheduling and +lets the flush coalescing above actually pay off. On Unix hosts, where named pipes +are Unix domain sockets, the sizes are advisory. + ## Verification The frame protocol lives in `ZB.MOM.WW.MxGateway.Worker.Ipc` (`WorkerFrameReader`, diff --git a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionWorkerClientFactory.cs b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionWorkerClientFactory.cs index 4a14f6b..ab416d9 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionWorkerClientFactory.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionWorkerClientFactory.cs @@ -12,6 +12,18 @@ namespace ZB.MOM.WW.MxGateway.Server.Sessions; /// Factory for creating worker clients and launching worker processes. public sealed class SessionWorkerClientFactory : ISessionWorkerClientFactory { + /// + /// Kernel buffer quota requested for each direction of a worker pipe. A zero quota — what the + /// short overloads request — makes every byte-mode write + /// rendezvous with a pending read, so writer latency is coupled to reader scheduling and a + /// writer with no reader parked blocks indefinitely. That is the failure class behind the + /// historical windev full-suite wedge (all tests reported, testhost never exiting). A real + /// quota lets a whole frame land in the kernel and the writer return. 128 KiB comfortably + /// holds the control traffic and typical event batches without reserving nonpaged pool per + /// session for the rare maximum-sized frame, which still streams through in chunks. + /// + private const int PipeBufferSizeBytes = 128 * 1024; + private readonly IWorkerProcessLauncher _workerProcessLauncher; private readonly GatewayMetrics _metrics; private readonly TimeProvider _timeProvider; @@ -155,6 +167,11 @@ public sealed class SessionWorkerClientFactory : ISessionWorkerClientFactory /// Creates a named pipe for worker communication. /// The pipe name. /// Named pipe server stream. + /// + /// The buffer sizes are explicit so the pipe is not created with a zero quota; see + /// . On Unix hosts (the macOS test matrix, where named pipes + /// are Unix domain sockets) the sizes are advisory — the fix targets Windows production. + /// private static NamedPipeServerStream CreatePipe(string pipeName) { return new NamedPipeServerStream( @@ -162,7 +179,9 @@ public sealed class SessionWorkerClientFactory : ISessionWorkerClientFactory PipeDirection.InOut, maxNumberOfServerInstances: 1, PipeTransmissionMode.Byte, - PipeOptions.Asynchronous); + PipeOptions.Asynchronous, + inBufferSize: PipeBufferSizeBytes, + outBufferSize: PipeBufferSizeBytes); } /// Waits for a client to connect to the pipe. From 6c5218913bbd5c466bd88d01ee6bcb630b6bee07 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:07:33 -0400 Subject: [PATCH 03/41] =?UTF-8?q?perf(dashboard):=20gate=20event=20mirror?= =?UTF-8?q?=20on=20live=20viewers=20=E2=80=94=20no=20clone,=20no=20send=20?= =?UTF-8?q?for=20unwatched=20sessions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DashboardEventBroadcaster.Publish ran a deep protobuf Clone (redaction is on by default) and a group SendAsync for every event of every session, before anything checked whether a dashboard client was actually watching. In the steady state the session:{id} group is empty, so that work was thrown away per event. SignalR does not expose group membership, so EventsHub now mirrors its own add/remove into a singleton EventsHubViewerRegistry, and OnDisconnectedAsync releases everything a dropped connection held (SessionDetailsPage disposes the connection rather than unsubscribing). Publish returns early when the session has no viewers, before the redaction clone. Watched sessions behave exactly as before. Lazy mirror-lease start/stop was deliberately not attempted — it entangles the dashboard with SessionEventDistributor subscribe lifetime for no saving beyond this gate; recorded in docs/GatewayDashboardDesign.md. --- docs/GatewayDashboardDesign.md | 27 ++- .../DashboardServiceCollectionExtensions.cs | 3 + .../Hubs/DashboardEventBroadcaster.cs | 17 ++ .../Dashboard/Hubs/EventsHub.cs | 38 +++- .../Dashboard/Hubs/EventsHubViewerRegistry.cs | 147 ++++++++++++++++ .../DashboardEventBroadcasterTests.cs | 78 ++++++++- .../Dashboard/EventsHubViewerRegistryTests.cs | 165 ++++++++++++++++++ 7 files changed, 465 insertions(+), 10 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHubViewerRegistry.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/EventsHubViewerRegistryTests.cs diff --git a/docs/GatewayDashboardDesign.md b/docs/GatewayDashboardDesign.md index e7fb554..5267fdc 100644 --- a/docs/GatewayDashboardDesign.md +++ b/docs/GatewayDashboardDesign.md @@ -167,7 +167,7 @@ bearer). Each hub class is `[Authorize(Policy = HubClientsPolicy)]`. |---|---|---|---|---| | `DashboardSnapshotHub` | `/hubs/snapshot` | `DashboardSnapshotPublisher` (BackgroundService consuming `IDashboardSnapshotService.WatchSnapshotsAsync`) | `DashboardSnapshot` | Sent to all connected clients on every snapshot tick; new connections receive the current snapshot synchronously in `OnConnectedAsync`. | | `AlarmsHub` | `/hubs/alarms` | `AlarmsHubPublisher` (BackgroundService consuming `IGatewayAlarmService.StreamAsync(filter: null)`) | `AlarmFeedMessage` (`active_alarm` / `snapshot_complete` / `transition`) | Connected clients auto-join `__alarms__`; all clients receive every message. Publisher auto-reconnects every 5s on stream faults. | -| `EventsHub` | `/hubs/events` | `DashboardEventBroadcaster` invoked by each session's internal dashboard-mirror subscriber on its `SessionEventDistributor` (registered when the session becomes Ready) | `MxEvent` | Clients call `SubscribeSession(sessionId)` to join `session:{id}`. The dashboard is a first-class distributor subscriber, so it receives the session's events whether or not a gRPC client is streaming. It sees RAW session events — not the per-gRPC-subscriber `AfterWorkerSequence` filtering that `EventStreamService` applies at its own boundary — because the dashboard is a separate LDAP-authenticated monitoring view meant to show the session's full event activity. Tag values are stripped from the mirrored `MxEvent` copy by `DashboardEventBroadcaster` when `Dashboard:ShowTagValues` is false (the default) — event metadata (tag reference, quality, status, timestamps) still renders, but the value fields are blanked, so no value leaks through this seam. The per-session hub ACL that would scope a Viewer to specific sessions is still outstanding (SEC-25 / remediation roadmap item 12); the value redaction is the near-term hardening that closes the value-leak seam independently of that ACL. | +| `EventsHub` | `/hubs/events` | `DashboardEventBroadcaster` invoked by each session's internal dashboard-mirror subscriber on its `SessionEventDistributor` (registered when the session becomes Ready) | `MxEvent` | Clients call `SubscribeSession(sessionId)` to join `session:{id}`, which also registers them in `EventsHubViewerRegistry` — the mirror is gated on that registry (see "Mirror gating" below). The dashboard is a first-class distributor subscriber, so it receives the session's events whether or not a gRPC client is streaming. It sees RAW session events — not the per-gRPC-subscriber `AfterWorkerSequence` filtering that `EventStreamService` applies at its own boundary — because the dashboard is a separate LDAP-authenticated monitoring view meant to show the session's full event activity. Tag values are stripped from the mirrored `MxEvent` copy by `DashboardEventBroadcaster` when `Dashboard:ShowTagValues` is false (the default) — event metadata (tag reference, quality, status, timestamps) still renders, but the value fields are blanked, so no value leaks through this seam. The per-session hub ACL that would scope a Viewer to specific sessions is still outstanding (SEC-25 / remediation roadmap item 12); the value redaction is the near-term hardening that closes the value-leak seam independently of that ACL. | `DashboardPageBase` opens a `DashboardSnapshotHub` connection via the connection factory in `OnInitializedAsync`, seeds `Snapshot` synchronously from @@ -191,6 +191,31 @@ Avoid pushing every MXAccess data-change event into a wider broadcast group. The current design routes events strictly through `session:{id}` groups; the snapshot hub continues to carry aggregate event counters and rates. +### Mirror gating + +Each session's dashboard-mirror subscriber calls +`DashboardEventBroadcaster.Publish` for every event the session produces, +independently of whether any browser is watching that session. SignalR does not +expose group membership, so the broadcaster cannot ask whether `session:{id}` is +empty. `EventsHubViewerRegistry` (singleton) supplies that answer: `EventsHub` +mirrors its own `AddToGroup` / `RemoveFromGroup` calls into it, and +`OnDisconnectedAsync` releases every subscription a dropped connection held — the +only reliable signal for a browser tab that closes without unsubscribing. +`Publish` returns immediately when `HasViewers(sessionId)` is false, **before** +the redaction clone. That matters because redaction is on by default +(`Dashboard:ShowTagValues` false), so the unwatched steady state — nobody on any +session-details page — previously paid a deep protobuf clone plus a send to an +empty group for every event of every session. Behaviour for a watched session is +unchanged. + +The mirror subscriber itself is still registered on the `SessionEventDistributor` +for the session's whole lifetime; only the per-event work is gated. Starting and +stopping the mirror lease lazily with the first and last viewer was considered +and deliberately not done — it entangles the dashboard with distributor +subscribe/unsubscribe lifetime (and with the replay/sequence bookkeeping that +attaching a subscriber mid-stream implies) for no additional saving beyond the +clone and send this gate already removes. + ## Pages ### Dashboard home diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardServiceCollectionExtensions.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardServiceCollectionExtensions.cs index 658d42e..c28f859 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardServiceCollectionExtensions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardServiceCollectionExtensions.cs @@ -47,6 +47,9 @@ public static class DashboardServiceCollectionExtensions services.AddSingleton(); services.AddScoped(); services.AddScoped(); + // Singleton: EventsHub instances are transient (one per hub invocation), so the + // subscriber bookkeeping they share with the broadcaster must outlive them. + services.AddSingleton(); services.AddSingleton(); services.AddHostedService(); services.AddHostedService(); diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardEventBroadcaster.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardEventBroadcaster.cs index b81125e..33597d6 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardEventBroadcaster.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardEventBroadcaster.cs @@ -21,8 +21,15 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; /// the mirror independently of the still-outstanding per-session hub ACL /// (see ). /// +/// Hub context used to send to the session's group. +/// +/// Live-subscriber registry consulted before any per-event work is done. +/// +/// Gateway options supplying Dashboard:ShowTagValues. +/// Logger for best-effort mirror failures. public sealed class DashboardEventBroadcaster( IHubContext hubContext, + EventsHubViewerRegistry viewerRegistry, IOptions options, ILogger logger) : IDashboardEventBroadcaster { @@ -36,6 +43,16 @@ public sealed class DashboardEventBroadcaster( return; } + // Every session's dashboard-mirror subscriber calls Publish for every event, + // whether or not a browser is on that session's page. Without this gate the + // steady state — no dashboard viewer at all — still paid a deep protobuf + // clone (redaction is on by default) plus a send to an empty SignalR group + // per event. Bail before both. + if (!viewerRegistry.HasViewers(sessionId)) + { + return; + } + MxEvent outbound = _showTagValues ? mxEvent : RedactValues(mxEvent); // Wrap the Task acquisition in a try/catch so a hypothetical synchronous throw diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHub.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHub.cs index 1c5863c..53c5c1c 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHub.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHub.cs @@ -9,8 +9,14 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; /// session; sends messages to /// session:{id} as events arrive from the live gRPC stream. /// +/// +/// Group membership is mirrored into +/// because SignalR does not expose it, and the broadcaster consults the +/// registry to skip all mirror work for sessions nobody is watching. +/// +/// Registry tracking which sessions have live subscribers. [Authorize(Policy = DashboardAuthenticationDefaults.HubClientsPolicy)] -public sealed class EventsHub : Hub +public sealed class EventsHub(EventsHubViewerRegistry viewerRegistry) : Hub { /// Method name used to push individual MxEvent values to clients. public const string EventMessage = "MxEvent"; @@ -55,19 +61,43 @@ public sealed class EventsHub : Hub return Task.CompletedTask; } + // Register before joining the group: the reverse order would leave a window + // in which this connection is a group member but the broadcaster's gate still + // reports the session unwatched, silently dropping events it should receive. + viewerRegistry.AddViewer(Context.ConnectionId, sessionId); + return Groups.AddToGroupAsync(Context.ConnectionId, GroupName(sessionId)); } /// Unsubscribes the calling SignalR connection from the per-session events group. /// Session id to unsubscribe the caller from. /// A task representing the unsubscription operation. - public Task UnsubscribeSession(string sessionId) + public async Task UnsubscribeSession(string sessionId) { if (string.IsNullOrWhiteSpace(sessionId)) { - return Task.CompletedTask; + return; } - return Groups.RemoveFromGroupAsync(Context.ConnectionId, GroupName(sessionId)); + // Leave the group first, deregister after — the mirror stays enabled for the + // brief overlap rather than dropping events still owed to other subscribers. + await Groups.RemoveFromGroupAsync(Context.ConnectionId, GroupName(sessionId)).ConfigureAwait(false); + + viewerRegistry.RemoveViewer(Context.ConnectionId, sessionId); + } + + /// + /// Releases every session subscription the dropped connection held. A browser + /// tab that closes never calls , so without + /// this the session would look watched forever and the mirror would keep + /// cloning and sending events to an empty group. + /// + /// The exception that terminated the connection, if any. + /// A task representing the disconnect handling. + public override Task OnDisconnectedAsync(Exception? exception) + { + viewerRegistry.ReleaseConnection(Context.ConnectionId); + + return base.OnDisconnectedAsync(exception); } } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHubViewerRegistry.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHubViewerRegistry.cs new file mode 100644 index 0000000..0607dc3 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHubViewerRegistry.cs @@ -0,0 +1,147 @@ +using System.Collections.Concurrent; + +namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; + +/// +/// Tracks which sessions currently have at least one live +/// subscriber, so can skip the redaction +/// clone and the group send for sessions nobody is watching. +/// +/// +/// SignalR does not expose group membership, so the hub mirrors its own +/// AddToGroup/RemoveFromGroup calls here. In the steady state no +/// browser is on a session-details page, yet every session's dashboard-mirror +/// subscriber still called Publish for every event — a deep protobuf +/// clone (values are redacted by default) plus a send to an empty group, per +/// event, thrown away. This registry is the cheap gate in front of that work. +/// +/// Per-connection subscriptions are tracked as well, because a browser tab that +/// simply goes away never calls UnsubscribeSession; the hub's +/// OnDisconnectedAsync releases everything the connection held. +/// +/// +public sealed class EventsHubViewerRegistry +{ + private readonly ConcurrentDictionary _viewersBySession = new(StringComparer.Ordinal); + + private readonly ConcurrentDictionary> _sessionsByConnection = + new(StringComparer.Ordinal); + + /// + /// Records that is watching + /// . Repeat calls for the same pair are + /// idempotent, so one always clears them. + /// + /// SignalR connection id of the subscriber. + /// Session id being watched. + public void AddViewer(string connectionId, string sessionId) + { + if (string.IsNullOrWhiteSpace(connectionId) || string.IsNullOrWhiteSpace(sessionId)) + { + return; + } + + ConcurrentDictionary sessions = _sessionsByConnection.GetOrAdd( + connectionId, + static _ => new ConcurrentDictionary(StringComparer.Ordinal)); + + // The per-connection set is the source of truth for the count: only a + // subscription that was genuinely new increments the session's viewers. + if (!sessions.TryAdd(sessionId, 0)) + { + return; + } + + _viewersBySession.AddOrUpdate(sessionId, 1, static (_, count) => count + 1); + } + + /// + /// Records that stopped watching + /// . A removal with no matching + /// is a no-op, so the count cannot go negative. + /// + /// SignalR connection id of the subscriber. + /// Session id no longer being watched. + public void RemoveViewer(string connectionId, string sessionId) + { + if (string.IsNullOrWhiteSpace(connectionId) || string.IsNullOrWhiteSpace(sessionId)) + { + return; + } + + if (!_sessionsByConnection.TryGetValue(connectionId, out ConcurrentDictionary? sessions) + || !sessions.TryRemove(sessionId, out _)) + { + return; + } + + ReleaseSession(sessionId); + } + + /// + /// Releases every subscription held by . + /// Called from the hub's disconnect callback, which is the only reliable + /// signal for a browser tab that closed without unsubscribing. + /// + /// SignalR connection id that dropped. + public void ReleaseConnection(string connectionId) + { + if (string.IsNullOrWhiteSpace(connectionId)) + { + return; + } + + if (!_sessionsByConnection.TryRemove(connectionId, out ConcurrentDictionary? sessions)) + { + return; + } + + foreach (string sessionId in sessions.Keys) + { + // TryRemove, not a bare enumeration: a concurrent RemoveViewer on the + // same detached set must not let the session be decremented twice. + if (sessions.TryRemove(sessionId, out _)) + { + ReleaseSession(sessionId); + } + } + } + + /// Gets a value indicating whether any hub connection is watching the session. + /// Session id to test. + /// when at least one connection is subscribed. + public bool HasViewers(string sessionId) => + !string.IsNullOrEmpty(sessionId) + && _viewersBySession.TryGetValue(sessionId, out int count) + && count > 0; + + /// + /// Decrements the session's viewer count, dropping the entry entirely at + /// zero so the dictionary does not grow one key per session ever viewed. + /// The compare-and-swap loop keeps the decrement correct against a + /// concurrent on the same session. + /// + /// Session id whose count is released. + private void ReleaseSession(string sessionId) + { + while (true) + { + if (!_viewersBySession.TryGetValue(sessionId, out int count)) + { + return; + } + + if (count <= 1) + { + if (_viewersBySession.TryRemove(new KeyValuePair(sessionId, count))) + { + return; + } + } + else if (_viewersBySession.TryUpdate(sessionId, count - 1, count)) + { + return; + } + } + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardEventBroadcasterTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardEventBroadcasterTests.cs index 7ec3c86..701573c 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardEventBroadcasterTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardEventBroadcasterTests.cs @@ -11,16 +11,64 @@ namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard; /// Verifies that honours /// MxGateway:Dashboard:ShowTagValues (SEC-25): tag values are stripped /// from the mirrored copy when the flag is off, present when it is on, and the -/// shared source event is never mutated. +/// shared source event is never mutated. Also verifies the viewer gate — the +/// mirror does no work at all for a session nobody is watching. /// public sealed class DashboardEventBroadcasterTests { + /// An unwatched session costs neither a redaction clone nor a send. + [Fact] + public void Publish_WithNoRegisteredViewers_DoesNotCloneOrSend() + { + CapturingHubContext hubContext = new(); + EventsHubViewerRegistry viewers = new(); + DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: false, viewers); + MxEvent source = BuildEventWithValue(); + + broadcaster.Publish("session-1", source); + + Assert.Equal(0, hubContext.SendCount); + Assert.Null(hubContext.LastArgument); + } + + /// A viewer on a different session does not open the gate for this one. + [Fact] + public void Publish_WithViewersOnAnotherSessionOnly_DoesNotSend() + { + CapturingHubContext hubContext = new(); + EventsHubViewerRegistry viewers = new(); + viewers.AddViewer("conn-1", "session-2"); + DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: false, viewers); + + broadcaster.Publish("session-1", BuildEventWithValue()); + + Assert.Equal(0, hubContext.SendCount); + } + + /// Once the last viewer leaves, the mirror stops sending again. + [Fact] + public void Publish_AfterLastViewerLeaves_StopsSending() + { + CapturingHubContext hubContext = new(); + EventsHubViewerRegistry viewers = new(); + viewers.AddViewer("conn-1", "session-1"); + DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: false, viewers); + + broadcaster.Publish("session-1", BuildEventWithValue()); + Assert.Equal(1, hubContext.SendCount); + + viewers.RemoveViewer("conn-1", "session-1"); + broadcaster.Publish("session-1", BuildEventWithValue()); + + Assert.Equal(1, hubContext.SendCount); + } + /// Values are stripped from the mirror when ShowTagValues is off; metadata survives. [Fact] public void Publish_WhenShowTagValuesFalse_RedactsValuesButKeepsMetadata() { CapturingHubContext hubContext = new(); - DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: false); + DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: false, WatchedSession1()); MxEvent source = BuildEventWithValue(); broadcaster.Publish("session-1", source); @@ -43,7 +91,7 @@ public sealed class DashboardEventBroadcasterTests public void Publish_WhenShowTagValuesFalse_DoesNotMutateSourceEvent() { CapturingHubContext hubContext = new(); - DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: false); + DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: false, WatchedSession1()); MxEvent source = BuildEventWithValue(); broadcaster.Publish("session-1", source); @@ -61,7 +109,7 @@ public sealed class DashboardEventBroadcasterTests public void Publish_WhenShowTagValuesTrue_KeepsValues() { CapturingHubContext hubContext = new(); - DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: true); + DashboardEventBroadcaster broadcaster = Create(hubContext, showTagValues: true, WatchedSession1()); MxEvent source = BuildEventWithValue(); broadcaster.Publish("session-1", source); @@ -73,7 +121,10 @@ public sealed class DashboardEventBroadcasterTests Assert.NotNull(sent.OnAlarmTransition.LimitValue); } - private static DashboardEventBroadcaster Create(CapturingHubContext hubContext, bool showTagValues) + private static DashboardEventBroadcaster Create( + CapturingHubContext hubContext, + bool showTagValues, + EventsHubViewerRegistry viewers) { GatewayOptions gatewayOptions = new() { @@ -82,10 +133,20 @@ public sealed class DashboardEventBroadcasterTests return new DashboardEventBroadcaster( hubContext, + viewers, Options.Create(gatewayOptions), NullLogger.Instance); } + /// A registry with one hub connection watching session-1. + /// The populated registry. + private static EventsHubViewerRegistry WatchedSession1() + { + EventsHubViewerRegistry viewers = new(); + viewers.AddViewer("conn-1", "session-1"); + return viewers; + } + private static MxEvent BuildEventWithValue() { return new MxEvent @@ -117,6 +178,9 @@ public sealed class DashboardEventBroadcasterTests /// Gets the first argument of the most recent send call. public object? LastArgument => _clients.GroupProxy.LastArgument; + + /// Gets the number of send calls this fake has observed. + public int SendCount => _clients.GroupProxy.SendCount; } private sealed class CapturingHubClients : IHubClients @@ -148,6 +212,9 @@ public sealed class DashboardEventBroadcasterTests /// Gets the first argument of the most recent send call. public object? LastArgument { get; private set; } + /// Gets the number of send calls made through this proxy. + public int SendCount { get; private set; } + /// Records the send call arguments and completes synchronously. /// The SignalR method name. /// The method arguments. @@ -155,6 +222,7 @@ public sealed class DashboardEventBroadcasterTests /// A completed task. public Task SendCoreAsync(string method, object?[] args, CancellationToken cancellationToken = default) { + SendCount++; LastArgument = args.Length > 0 ? args[0] : null; return Task.CompletedTask; } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/EventsHubViewerRegistryTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/EventsHubViewerRegistryTests.cs new file mode 100644 index 0000000..e220b9c --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/EventsHubViewerRegistryTests.cs @@ -0,0 +1,165 @@ +using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; + +namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard; + +/// +/// Verifies , the seam +/// consults before cloning and sending +/// an event: a session is "watched" only while at least one hub connection +/// holds a subscription to it, and a dropped connection releases every +/// subscription it held. +/// +public sealed class EventsHubViewerRegistryTests +{ + /// A session with no subscriber is not watched. + [Fact] + public void HasViewers_WithNoSubscribers_IsFalse() + { + EventsHubViewerRegistry registry = new(); + + Assert.False(registry.HasViewers("session-1")); + } + + /// Adding then removing the only viewer flips the session back to unwatched. + [Fact] + public void AddViewer_ThenRemoveViewer_TogglesWatchedState() + { + EventsHubViewerRegistry registry = new(); + + registry.AddViewer("conn-1", "session-1"); + Assert.True(registry.HasViewers("session-1")); + + registry.RemoveViewer("conn-1", "session-1"); + Assert.False(registry.HasViewers("session-1")); + } + + /// Each connection counts once: the session stays watched until the last one leaves. + [Fact] + public void RemoveViewer_WithOtherConnectionsStillSubscribed_KeepsSessionWatched() + { + EventsHubViewerRegistry registry = new(); + + registry.AddViewer("conn-1", "session-1"); + registry.AddViewer("conn-2", "session-1"); + + registry.RemoveViewer("conn-1", "session-1"); + Assert.True(registry.HasViewers("session-1")); + + registry.RemoveViewer("conn-2", "session-1"); + Assert.False(registry.HasViewers("session-1")); + } + + /// A repeated subscribe from the same connection is idempotent, so one unsubscribe clears it. + [Fact] + public void AddViewer_CalledTwiceForSameConnection_CountsOnce() + { + EventsHubViewerRegistry registry = new(); + + registry.AddViewer("conn-1", "session-1"); + registry.AddViewer("conn-1", "session-1"); + + registry.RemoveViewer("conn-1", "session-1"); + + Assert.False(registry.HasViewers("session-1")); + } + + /// Viewer counts are tracked per session; unrelated sessions stay unwatched. + [Fact] + public void AddViewer_TracksSessionsIndependently() + { + EventsHubViewerRegistry registry = new(); + + registry.AddViewer("conn-1", "session-1"); + + Assert.True(registry.HasViewers("session-1")); + Assert.False(registry.HasViewers("session-2")); + } + + /// A dropped connection releases every session it held. + [Fact] + public void ReleaseConnection_ReleasesEverySessionTheConnectionHeld() + { + EventsHubViewerRegistry registry = new(); + + registry.AddViewer("conn-1", "session-1"); + registry.AddViewer("conn-1", "session-2"); + registry.AddViewer("conn-2", "session-2"); + + registry.ReleaseConnection("conn-1"); + + Assert.False(registry.HasViewers("session-1")); + + // conn-2 still watches session-2. + Assert.True(registry.HasViewers("session-2")); + } + + /// Releasing a connection twice does not double-decrement another connection's subscription. + [Fact] + public void ReleaseConnection_CalledTwice_DoesNotDropOtherViewers() + { + EventsHubViewerRegistry registry = new(); + + registry.AddViewer("conn-1", "session-1"); + registry.AddViewer("conn-2", "session-1"); + + registry.ReleaseConnection("conn-1"); + registry.ReleaseConnection("conn-1"); + + Assert.True(registry.HasViewers("session-1")); + } + + /// Unmatched removals cannot drive the count negative and strand a session as unwatched. + [Fact] + public void RemoveViewer_WithoutMatchingAdd_LeavesCountAtZero() + { + EventsHubViewerRegistry registry = new(); + + registry.RemoveViewer("conn-1", "session-1"); + registry.RemoveViewer("conn-1", "session-1"); + registry.ReleaseConnection("conn-1"); + + Assert.False(registry.HasViewers("session-1")); + + // A subsequent genuine subscribe must still register as exactly one viewer. + registry.AddViewer("conn-1", "session-1"); + Assert.True(registry.HasViewers("session-1")); + + registry.RemoveViewer("conn-1", "session-1"); + Assert.False(registry.HasViewers("session-1")); + } + + /// Blank connection or session ids are ignored rather than tracked. + [Theory] + [InlineData("", "session-1")] + [InlineData(" ", "session-1")] + [InlineData("conn-1", "")] + [InlineData("conn-1", " ")] + public void AddViewer_WithBlankIdentifiers_IsIgnored(string connectionId, string sessionId) + { + EventsHubViewerRegistry registry = new(); + + registry.AddViewer(connectionId, sessionId); + + Assert.False(registry.HasViewers(sessionId)); + Assert.False(registry.HasViewers("session-1")); + } + + /// Concurrent add/remove pairs settle at zero viewers, never at a stuck-on count. + [Fact] + public void AddAndRemoveViewer_UnderConcurrency_SettlesAtZero() + { + EventsHubViewerRegistry registry = new(); + + Parallel.For(0, 64, i => + { + string connectionId = $"conn-{i}"; + for (int pass = 0; pass < 50; pass++) + { + registry.AddViewer(connectionId, "session-1"); + registry.RemoveViewer(connectionId, "session-1"); + } + }); + + Assert.False(registry.HasViewers("session-1")); + } +} From e2ac5d117a960f1d8e282df8eaa3fb80f35cebb5 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:17:22 -0400 Subject: [PATCH 04/41] perf(audit): bounded async audit writer with batched inserts, one-time bootstrap, retention sweep --- docs/DesignDecisions.md | 25 ++ docs/GatewayConfiguration.md | 1 + .../Configuration/GatewayOptionsValidator.cs | 7 + .../Configuration/SecurityOptions.cs | 9 + .../Security/Audit/CanonicalAuditWriter.cs | 20 +- .../Security/Audit/ChannelAuditWriter.cs | 333 ++++++++++++++++++ .../Audit/SqliteCanonicalAuditStore.cs | 189 ++++++++-- .../AuthStoreServiceCollectionExtensions.cs | 28 +- .../Security/Audit/ChannelAuditWriterTests.cs | 271 ++++++++++++++ 9 files changed, 845 insertions(+), 38 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Server/Security/Audit/ChannelAuditWriter.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/Security/Audit/ChannelAuditWriterTests.cs diff --git a/docs/DesignDecisions.md b/docs/DesignDecisions.md index fcaeafd..2d1e479 100644 --- a/docs/DesignDecisions.md +++ b/docs/DesignDecisions.md @@ -228,6 +228,31 @@ Storage recommendation: administrators. - Require TLS when the gateway is reachable off-machine. +## Audit Pipeline + +Decision: audit is asynchronous, bounded, and swept. + +The canonical `IAuditWriter` contract has always been best-effort — a failed audit write is +logged and swallowed so it cannot abort the action that produced it. The registered writer is +`ChannelAuditWriter`, which makes the cost of that promise explicit: a producer enqueues onto a +4096-event bounded channel and returns, and `AuditDrainService` commits up to 64 buffered events +per transaction. This exists because constraint denials are emitted per denied tag inside bulk +RPC loops: a partially denied 1,000-tag request previously awaited 1,000 sequential SQLite +inserts — each re-running `CREATE TABLE IF NOT EXISTS` — against the same database file every +authenticated call reads. The schema bootstrap now runs once, from the drain's `StartAsync`. + +When the channel is full the newest event is dropped and counted rather than blocking the +producer: a stalled audit database must cost audit completeness, not gateway availability. Drops +are logged once and reported in aggregate on each sweep. Shutdown drains what is buffered under a +2-second cap. Where no hosted service runs — the `apikey` admin CLI — the writer falls back to the +synchronous path, so audit is never buffered into a channel nobody drains. + +`MxGateway:Security:AuditRetentionDays` (default 90, minimum 1) bounds the table: the drain sweeps +at startup and hourly, deleting older rows. Retention cannot be configured off. The sweep compares +through SQLite's `datetime()` rather than on the stored ISO-8601 text, because timestamps written +from a non-UTC offset do not sort lexicographically against a UTC cutoff; a row whose timestamp +cannot be parsed yields NULL and is kept rather than deleted. + ## Authorization Decision: start with scope checks by command category. diff --git a/docs/GatewayConfiguration.md b/docs/GatewayConfiguration.md index 23bf2a4..9d269e1 100644 --- a/docs/GatewayConfiguration.md +++ b/docs/GatewayConfiguration.md @@ -393,6 +393,7 @@ model requires otherwise. | `MxGateway:Security:ApiKeyFailureWindowSeconds` | `60` | Sliding-window length, in seconds, over which API-key verification failures are counted, for both the per-partition and the per-key-id aggregate layer. Must be greater than zero. | | `MxGateway:Security:ApiKeyFailureAggregateLimit` | `30` | Failed verifications for one key id counted across **all** transport peers within `ApiKeyFailureWindowSeconds` before that key id enters probe mode. This second layer bounds a distributed or source-rotating sprayer that never trips any single `(peer, key id)` partition. `0` disables the aggregate layer, leaving only per-partition counting. Must be zero or greater. | | `MxGateway:Security:ApiKeyFailureProbeIntervalSeconds` | `5` | Minimum interval, in seconds, between probe admissions for an over-limit partition or key-id aggregate. An over-limit state is a valve rather than a wall: one request per interval reaches the real verifier — exactly one, even when a burst arrives together at the interval boundary — so the holder of the correct secret always gets through and clears the state, while everything else is still refused before the store read. `0` blocks absolutely instead — **not recommended**, because an unauthenticated peer can then deny the key to its holder for the whole window. Must be zero or greater. | +| `MxGateway:Security:AuditRetentionDays` | `90` | Days of canonical audit history kept in the `audit_event` table. The audit drain sweeps once at startup and hourly thereafter, deleting rows older than this window; without it the table grows without bound inside the same SQLite file the authentication hot path reads. Rows whose timestamp SQLite cannot parse are never swept. Must be greater than zero — retention can be widened but not switched off. | | `MxGateway:Security:ApiKeyFailureTrackedPeers` | `4096` | Maximum distinct partitions tracked by the failure counter (a bounded LRU) so a spray of unique tokens cannot grow memory without limit. It cannot be used to flush an active block either: only a validly shaped `mxgw__` token mints a key-id partition (everything else lands on the sender's transport-peer partition), each address may mint at most 32 key-id partitions before the overflow collapses onto that address's fallback partition, and eviction prefers fully expired windows, never removing an over-limit partition until the map exceeds twice this cap. Must be greater than zero. | ## Galaxy Options diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs index 3732209..3f98691 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs @@ -100,6 +100,13 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase public int ApiKeyFailureTrackedPeers { get; init; } = 4096; + + /// + /// Gets how many days of canonical audit history the gateway keeps. The audit drain sweeps + /// audit_event once at startup and hourly thereafter, deleting rows older than this + /// window; without it the table grows without bound in the same SQLite file the + /// authentication hot path reads. Must be greater than zero — audit retention cannot be + /// disabled by configuration, only widened. Default is 90 days. + /// + public int AuditRetentionDays { get; init; } = 90; } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/CanonicalAuditWriter.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/CanonicalAuditWriter.cs index 593849b..a03fdfb 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/CanonicalAuditWriter.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/CanonicalAuditWriter.cs @@ -3,24 +3,26 @@ using ZB.MOM.WW.Audit; namespace ZB.MOM.WW.MxGateway.Server.Security.Audit; /// -/// Best-effort over the MxGateway-owned -/// . It honours the canonical +/// Best-effort, synchronous over the MxGateway-owned +/// . It honours the canonical /// contract: a failed audit write is swallowed and logged /// rather than propagated, so it can never abort the user-facing action that produced it. /// /// -/// This is the single sink through which ALL MxGateway audit flows — the library admin -/// verbs (via ) and the gateway's own -/// dashboard / constraint-denial producers, which write canonical events directly. The -/// best-effort wrapping here also closes the gap that the library's +/// This is the durable bottom of the audit pipeline. Callers reach it two ways: through +/// — the registered , which +/// enqueues and lets batch events onto the sink — and +/// directly, when there is no drain to batch behind (the apikey CLI, and any host +/// shutdown window), where writing through immediately is the only way the event survives. +/// The best-effort wrapping here also closes the gap that the library's /// SqliteApiKeyAuditStore.AppendAsync propagated exceptions. /// public sealed class CanonicalAuditWriter( - SqliteCanonicalAuditStore store, + IAuditEventSink sink, ILogger logger) : IAuditWriter { /// - /// Persists a canonical audit event to the underlying . + /// Persists a canonical audit event to the underlying . /// Any failure is caught, logged, and swallowed rather than propagated to the caller. /// /// The canonical audit event to persist. @@ -32,7 +34,7 @@ public sealed class CanonicalAuditWriter( try { - await store.InsertAsync(auditEvent, cancellationToken).ConfigureAwait(false); + await sink.InsertAsync(auditEvent, cancellationToken).ConfigureAwait(false); } catch (Exception exception) { diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/ChannelAuditWriter.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/ChannelAuditWriter.cs new file mode 100644 index 0000000..3b204cf --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/ChannelAuditWriter.cs @@ -0,0 +1,333 @@ +using System.Threading.Channels; +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.MxGateway.Server.Configuration; + +namespace ZB.MOM.WW.MxGateway.Server.Security.Audit; + +/// +/// Bounded, non-blocking : enqueues onto a +/// fixed-capacity channel and returns, leaving to batch the +/// events onto the durable . +/// +/// +/// The canonical contract is already best-effort — a failed audit +/// write is swallowed rather than propagated. The channel makes the bound on that +/// promise explicit: audit can cost the calling RPC at most one enqueue, never a SQLite +/// round-trip, and at most events of memory. This matters on the +/// constraint-denial path, where a partially denied bulk RPC previously awaited one insert per +/// denied tag, serially, against the same database file the authentication hot path reads. +/// +/// When the channel is full the newest write is dropped () +/// and counted in . Dropping is the deliberate choice over blocking: +/// a stalled audit database must degrade audit completeness, not stall the gateway. Drops are +/// logged, and reports the running total on its sweep. +/// +/// +/// Until a drain attaches (), and again after it detaches, writes go +/// straight through to . Enqueueing into a channel nobody will +/// ever read would silently discard audit in the processes that have no hosted services — the +/// apikey admin CLI and the DI-only tests — so those keep the original synchronous path. +/// +/// +public sealed class ChannelAuditWriter : IAuditWriter +{ + /// + /// Maximum number of audit events buffered before writes start being dropped. Sized to + /// absorb a fully denied bulk RPC (the gateway's bulk request cap) plus headroom, so a + /// realistic burst is buffered rather than lost. + /// + public const int ChannelCapacity = 4096; + + private readonly CanonicalAuditWriter _directWriter; + private readonly ILogger _logger; + private readonly Channel _channel; + + private long _droppedCount; + private int _drainAttached; + private int _dropLogged; + + /// Creates the writer and its bounded buffer. + /// The synchronous writer used when no drain is attached. + /// Logger for drop diagnostics. + public ChannelAuditWriter(CanonicalAuditWriter directWriter, ILogger logger) + { + _directWriter = directWriter; + _logger = logger; + + // DropWrite discards the incoming item and still reports success to the producer, so the + // itemDropped callback is the only place a drop can be observed and counted. + _channel = Channel.CreateBounded( + new BoundedChannelOptions(ChannelCapacity) + { + FullMode = BoundedChannelFullMode.DropWrite, + SingleReader = true, + SingleWriter = false, + }, + itemDropped: RecordDrop); + } + + /// Gets the number of audit events dropped because the channel was full. + public long DroppedCount => Interlocked.Read(ref _droppedCount); + + /// Gets the reader the drain service consumes buffered events from. + public ChannelReader Reader => _channel.Reader; + + /// + /// Marks a drain as running, so subsequent writes enqueue instead of writing through. + /// Called by once its one-time bootstrap has completed. + /// + public void AttachDrain() => Volatile.Write(ref _drainAttached, 1); + + /// + /// Marks the drain as no longer running, so writes revert to the synchronous path. Called + /// at shutdown so late audit is still persisted rather than buffered into a dead channel. + /// + public void DetachDrain() => Volatile.Write(ref _drainAttached, 0); + + /// + /// Enqueues a canonical audit event for the drain to persist. Never blocks, never throws, + /// and never touches the store on the caller's thread while a drain is attached. + /// + /// The canonical audit event to persist. + /// Token honoured only by the direct write-through path. + /// A task that represents the asynchronous operation. + public Task WriteAsync(AuditEvent auditEvent, CancellationToken cancellationToken = default) + { + ArgumentNullException.ThrowIfNull(auditEvent); + + if (Volatile.Read(ref _drainAttached) == 0) + { + return _directWriter.WriteAsync(auditEvent, cancellationToken); + } + + // Under DropWrite a full channel still reports success (the discard surfaces through the + // itemDropped callback); a false here means the channel is completed, which is also a drop. + if (!_channel.Writer.TryWrite(auditEvent)) + { + RecordDrop(auditEvent); + } + + return Task.CompletedTask; + } + + /// Signals that no further events will be enqueued, so the drain loop can finish. + public void CompleteWriting() => _channel.Writer.TryComplete(); + + private void RecordDrop(AuditEvent auditEvent) + { + Interlocked.Increment(ref _droppedCount); + + // Log the first drop only; the running total is reported on the drain's periodic sweep, + // so a sustained overload cannot turn audit pressure into a log flood. + if (Interlocked.Exchange(ref _dropLogged, 1) == 0) + { + _logger.LogWarning( + "Audit channel is full ({Capacity} events); dropping audit event {EventId} (action {Action}). " + + "Audit is best-effort and bounded; further drops are reported in aggregate.", + ChannelCapacity, + auditEvent.EventId, + auditEvent.Action); + } + } +} + +/// +/// Drains onto the durable , +/// owns the one-time schema bootstrap, and sweeps audit rows past their retention window. +/// +/// +/// Batching is the point: up to buffered events are committed in a +/// single transaction, so a burst of constraint denials costs a handful of commits instead of +/// one per denied tag. The bootstrap runs here — before the writer starts enqueueing — so no +/// audit write ever pays a CREATE TABLE IF NOT EXISTS round-trip. +/// +/// The channel writer whose buffered events are drained. +/// The durable sink events are committed to. +/// Security options carrying the audit retention window. +/// Clock used for the retention cutoff and sweep interval. +/// Logger for bootstrap, drain and sweep diagnostics. +public sealed class AuditDrainService( + ChannelAuditWriter writer, + IAuditEventSink sink, + SecurityOptions security, + TimeProvider timeProvider, + ILogger logger) : BackgroundService +{ + /// Maximum number of audit events committed in one transaction per drain pass. + public const int MaxBatchSize = 64; + + /// How often the retention sweep runs while the gateway is up. + public static readonly TimeSpan RetentionSweepInterval = TimeSpan.FromHours(1); + + /// Upper bound on how long shutdown waits for the remaining buffered events. + private static readonly TimeSpan ShutdownDrainCap = TimeSpan.FromSeconds(2); + + /// + /// Bootstraps the audit table, runs one retention sweep, then attaches the drain so the + /// writer switches from synchronous write-through to enqueueing. + /// + /// Token to observe for cancellation. + /// A task that represents the asynchronous operation. + public override async Task StartAsync(CancellationToken cancellationToken) + { + try + { + await sink.EnsureInitializedAsync(cancellationToken).ConfigureAwait(false); + await SweepRetentionAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception exception) + { + // Audit is best-effort: a bootstrap failure must not take the gateway down. The + // sink's own latch will retry the schema check on the first write. + logger.LogWarning(exception, "Audit store bootstrap failed; audit writes will retry the schema check."); + } + + writer.AttachDrain(); + + await base.StartAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Detaches the drain (so late writes go straight to the sink) and gives the buffered + /// events a bounded window to reach the store. + /// + /// Token to observe for cancellation. + /// A task that represents the asynchronous operation. + public override async Task StopAsync(CancellationToken cancellationToken) + { + writer.DetachDrain(); + writer.CompleteWriting(); + + await base.StopAsync(cancellationToken).ConfigureAwait(false); + + using CancellationTokenSource drainCap = new(ShutdownDrainCap); + try + { + await DrainPendingAsync(drainCap.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + logger.LogWarning( + "Shutdown drain exceeded {CapSeconds}s; remaining buffered audit events were not persisted.", + ShutdownDrainCap.TotalSeconds); + } + } + + /// + /// Commits every event currently buffered, in batches of at most . + /// + /// Token to observe for cancellation. + /// The number of events persisted. + public async Task DrainPendingAsync(CancellationToken cancellationToken) + { + int persisted = 0; + List batch = new(MaxBatchSize); + + while (!cancellationToken.IsCancellationRequested) + { + batch.Clear(); + while (batch.Count < MaxBatchSize && writer.Reader.TryRead(out AuditEvent? auditEvent)) + { + batch.Add(auditEvent); + } + + if (batch.Count == 0) + { + break; + } + + try + { + await sink.InsertBatchAsync(batch, cancellationToken).ConfigureAwait(false); + persisted += batch.Count; + } + catch (Exception exception) + { + // Best-effort: the batch is lost, but the drain keeps going so one poisoned + // batch cannot stall every later audit event behind it. + logger.LogWarning( + exception, + "Failed to persist a batch of {Count} audit events; audit writes are best-effort and were suppressed.", + batch.Count); + } + } + + return persisted; + } + + /// + /// Deletes audit rows older than MxGateway:Security:AuditRetentionDays, and reports + /// any audit events dropped by channel pressure since the last sweep. + /// + /// Token to observe for cancellation. + /// A task that represents the asynchronous operation. + public async Task SweepRetentionAsync(CancellationToken cancellationToken) + { + DateTimeOffset cutoff = timeProvider.GetUtcNow() - TimeSpan.FromDays(security.AuditRetentionDays); + + try + { + int deleted = await sink.DeleteOlderThanAsync(cutoff, cancellationToken).ConfigureAwait(false); + if (deleted > 0) + { + logger.LogInformation( + "Audit retention sweep removed {Deleted} events older than {Cutoff:o} ({RetentionDays} days).", + deleted, + cutoff, + security.AuditRetentionDays); + } + } + catch (Exception exception) + { + logger.LogWarning(exception, "Audit retention sweep failed; it will be retried on the next interval."); + } + + long dropped = writer.DroppedCount; + if (dropped > 0) + { + logger.LogWarning( + "{Dropped} audit events have been dropped since startup because the audit channel was full.", + dropped); + } + } + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await Task.WhenAll( + DrainLoopAsync(stoppingToken), + RetentionLoopAsync(stoppingToken)).ConfigureAwait(false); + } + + private async Task DrainLoopAsync(CancellationToken stoppingToken) + { + try + { + while (await writer.Reader.WaitToReadAsync(stoppingToken).ConfigureAwait(false)) + { + await DrainPendingAsync(stoppingToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + // Shutdown; StopAsync performs the final bounded drain. + } + } + + private async Task RetentionLoopAsync(CancellationToken stoppingToken) + { + using PeriodicTimer timer = new(RetentionSweepInterval, timeProvider); + + try + { + while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)) + { + await SweepRetentionAsync(stoppingToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + // Shutdown. + } + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/SqliteCanonicalAuditStore.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/SqliteCanonicalAuditStore.cs index a5ff0b8..db83c62 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/SqliteCanonicalAuditStore.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/SqliteCanonicalAuditStore.cs @@ -5,6 +5,41 @@ using ZB.MOM.WW.Auth.ApiKeys.Sqlite; namespace ZB.MOM.WW.MxGateway.Server.Security.Audit; +/// +/// Durable sink the audit pipeline persists canonical s through. +/// It exists so the write path () and the batching drain +/// () depend on the storage contract rather than on the +/// concrete . +/// +public interface IAuditEventSink +{ + /// + /// Bootstraps the backing storage. Called once at startup so no write path pays a schema + /// round-trip; implementations must be idempotent and safe to call concurrently. + /// + /// Token to observe for cancellation. + /// A task that represents the asynchronous operation. + Task EnsureInitializedAsync(CancellationToken cancellationToken); + + /// Persists a single canonical audit event. + /// The canonical event to persist. + /// Token to observe for cancellation. + /// A task that represents the asynchronous operation. + Task InsertAsync(AuditEvent auditEvent, CancellationToken cancellationToken); + + /// Persists a batch of canonical audit events as one unit of work. + /// The canonical events to persist. + /// Token to observe for cancellation. + /// A task that represents the asynchronous operation. + Task InsertBatchAsync(IReadOnlyList auditEvents, CancellationToken cancellationToken); + + /// Deletes every audit row that occurred strictly before . + /// The retention cutoff; rows older than this are removed. + /// Token to observe for cancellation. + /// The number of rows deleted. + Task DeleteOlderThanAsync(DateTimeOffset cutoffUtc, CancellationToken cancellationToken); +} + /// /// MxGateway-owned, append-only SQLite store for canonical /// s. It writes to a NEW audit_event table in the @@ -18,11 +53,20 @@ namespace ZB.MOM.WW.MxGateway.Server.Security.Audit; /// IApiKeyAuditStore registration is overridden by /// , which forwards onto this store via /// . The library's schema_version / -/// api_key_audit tables are not touched here; the audit_event table is -/// created idempotently (CREATE TABLE IF NOT EXISTS) on each write so it -/// self-bootstraps regardless of migration ordering. +/// api_key_audit tables are not touched here. +/// +/// The audit_event table is created idempotently, but the CREATE TABLE IF NOT +/// EXISTS is latched: runs +/// once at startup, and every later insert/list/delete +/// then skips the DDL round-trip. Keeping the (now free) check on each path rather than +/// dropping it means the store still self-bootstraps for callers that use it without the +/// hosted drain — the apikey CLI and the DI-only tests — regardless of migration +/// ordering. The latch is deliberately racy: a lost race merely re-runs an idempotent +/// CREATE TABLE IF NOT EXISTS, and a failure leaves the latch open so the next call +/// retries. +/// /// -public sealed class SqliteCanonicalAuditStore(AuthSqliteConnectionFactory connectionFactory) +public sealed class SqliteCanonicalAuditStore(AuthSqliteConnectionFactory connectionFactory) : IAuditEventSink { private const string CreateTableSql = """ @@ -40,14 +84,107 @@ public sealed class SqliteCanonicalAuditStore(AuthSqliteConnectionFactory connec ); """; - /// Inserts a canonical audit event into the audit_event table. - /// The canonical event to persist. - /// Token to observe for cancellation. - /// A task that represents the asynchronous operation. - public async Task InsertAsync(AuditEvent auditEvent, CancellationToken cancellationToken) + private const string InsertSql = + """ + INSERT INTO audit_event + (event_id, occurred_at_utc, actor, action, outcome, + category, target, source_node, correlation_id, details_json) + VALUES + ($event_id, $occurred_at_utc, $actor, $action, $outcome, + $category, $target, $source_node, $correlation_id, $details_json); + """; + + /// 0 until the audit_event table has been created at least once by this instance. + private int _tableEnsured; + + /// + public async Task EnsureInitializedAsync(CancellationToken cancellationToken) + { + await using SqliteConnection connection = + await connectionFactory.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + + await EnsureTableAsync(connection, cancellationToken).ConfigureAwait(false); + } + + /// + public Task InsertAsync(AuditEvent auditEvent, CancellationToken cancellationToken) { ArgumentNullException.ThrowIfNull(auditEvent); + return InsertBatchAsync([auditEvent], cancellationToken); + } + + /// + /// + /// One connection, one transaction and one prepared command for the whole batch: the drain + /// pays a single commit for up to events rather + /// than one round-trip per event. + /// + public async Task InsertBatchAsync(IReadOnlyList auditEvents, CancellationToken cancellationToken) + { + ArgumentNullException.ThrowIfNull(auditEvents); + + if (auditEvents.Count == 0) + { + return; + } + + await using SqliteConnection connection = + await connectionFactory.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); + + await EnsureTableAsync(connection, cancellationToken).ConfigureAwait(false); + + await using SqliteTransaction transaction = + (SqliteTransaction)await connection.BeginTransactionAsync(cancellationToken).ConfigureAwait(false); + + await using (SqliteCommand command = connection.CreateCommand()) + { + command.Transaction = transaction; + command.CommandText = InsertSql; + + SqliteParameter eventId = command.Parameters.Add("$event_id", SqliteType.Text); + SqliteParameter occurredAtUtc = command.Parameters.Add("$occurred_at_utc", SqliteType.Text); + SqliteParameter actor = command.Parameters.Add("$actor", SqliteType.Text); + SqliteParameter action = command.Parameters.Add("$action", SqliteType.Text); + SqliteParameter outcome = command.Parameters.Add("$outcome", SqliteType.Text); + SqliteParameter category = command.Parameters.Add("$category", SqliteType.Text); + SqliteParameter target = command.Parameters.Add("$target", SqliteType.Text); + SqliteParameter sourceNode = command.Parameters.Add("$source_node", SqliteType.Text); + SqliteParameter correlationId = command.Parameters.Add("$correlation_id", SqliteType.Text); + SqliteParameter detailsJson = command.Parameters.Add("$details_json", SqliteType.Text); + + foreach (AuditEvent auditEvent in auditEvents) + { + ArgumentNullException.ThrowIfNull(auditEvent); + + eventId.Value = auditEvent.EventId.ToString(); + occurredAtUtc.Value = auditEvent.OccurredAtUtc.ToString("O", CultureInfo.InvariantCulture); + actor.Value = auditEvent.Actor; + action.Value = auditEvent.Action; + outcome.Value = auditEvent.Outcome.ToString(); + category.Value = (object?)auditEvent.Category ?? DBNull.Value; + target.Value = (object?)auditEvent.Target ?? DBNull.Value; + sourceNode.Value = (object?)auditEvent.SourceNode ?? DBNull.Value; + correlationId.Value = (object?)auditEvent.CorrelationId?.ToString() ?? DBNull.Value; + detailsJson.Value = (object?)auditEvent.DetailsJson ?? DBNull.Value; + + await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + } + } + + await transaction.CommitAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// + /// The comparison goes through SQLite's datetime() rather than comparing the stored + /// ISO-8601 text directly: rows written from a non-UTC (the + /// library's CreatedUtc is caller-supplied) do not sort lexicographically against a + /// UTC cutoff. A row whose timestamp datetime() cannot parse yields NULL and is + /// therefore never deleted — audit that cannot be dated is kept, not swept. + /// + public async Task DeleteOlderThanAsync(DateTimeOffset cutoffUtc, CancellationToken cancellationToken) + { await using SqliteConnection connection = await connectionFactory.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); @@ -56,25 +193,14 @@ public sealed class SqliteCanonicalAuditStore(AuthSqliteConnectionFactory connec await using SqliteCommand command = connection.CreateCommand(); command.CommandText = """ - INSERT INTO audit_event - (event_id, occurred_at_utc, actor, action, outcome, - category, target, source_node, correlation_id, details_json) - VALUES - ($event_id, $occurred_at_utc, $actor, $action, $outcome, - $category, $target, $source_node, $correlation_id, $details_json); + DELETE FROM audit_event + WHERE datetime(occurred_at_utc) < datetime($cutoff); """; - command.Parameters.AddWithValue("$event_id", auditEvent.EventId.ToString()); - command.Parameters.AddWithValue("$occurred_at_utc", auditEvent.OccurredAtUtc.ToString("O", CultureInfo.InvariantCulture)); - command.Parameters.AddWithValue("$actor", auditEvent.Actor); - command.Parameters.AddWithValue("$action", auditEvent.Action); - command.Parameters.AddWithValue("$outcome", auditEvent.Outcome.ToString()); - command.Parameters.AddWithValue("$category", (object?)auditEvent.Category ?? DBNull.Value); - command.Parameters.AddWithValue("$target", (object?)auditEvent.Target ?? DBNull.Value); - command.Parameters.AddWithValue("$source_node", (object?)auditEvent.SourceNode ?? DBNull.Value); - command.Parameters.AddWithValue("$correlation_id", (object?)auditEvent.CorrelationId?.ToString() ?? DBNull.Value); - command.Parameters.AddWithValue("$details_json", (object?)auditEvent.DetailsJson ?? DBNull.Value); + command.Parameters.AddWithValue( + "$cutoff", + cutoffUtc.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture)); - await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + return await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); } /// Returns the most recent canonical audit events, newest first. @@ -128,11 +254,20 @@ public sealed class SqliteCanonicalAuditStore(AuthSqliteConnectionFactory connec return events; } - private static async Task EnsureTableAsync(SqliteConnection connection, CancellationToken cancellationToken) + // Latched bootstrap: after the first success this is a single volatile read, so the DDL + // round-trip is paid once per process rather than once per audit write. + private async Task EnsureTableAsync(SqliteConnection connection, CancellationToken cancellationToken) { + if (Volatile.Read(ref _tableEnsured) == 1) + { + return; + } + await using SqliteCommand command = connection.CreateCommand(); command.CommandText = CreateTableSql; await command.ExecuteNonQueryAsync(cancellationToken).ConfigureAwait(false); + + Volatile.Write(ref _tableEnsured, 1); } private static DateTimeOffset ParseUtc(string value) => diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/AuthStoreServiceCollectionExtensions.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/AuthStoreServiceCollectionExtensions.cs index 5c3d40a..6722fa6 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/AuthStoreServiceCollectionExtensions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/AuthStoreServiceCollectionExtensions.cs @@ -99,16 +99,40 @@ public static class AuthStoreServiceCollectionExtensions services.AddSingleton(sp => new SqliteCanonicalAuditStore(sp.GetRequiredService())); + services.AddSingleton(sp => sp.GetRequiredService()); // Resolve the logger defensively: the production host always registers ILogger, but the // DI-only auth/CLI/dashboard unit tests build a bare ServiceCollection without AddLogging(). // Fall back to NullLogger there so the audit writer (and the IApiKeyAuditStore override that // depends on it) still resolve. The write path is best-effort regardless. - services.AddSingleton(sp => + services.AddSingleton(sp => new CanonicalAuditWriter( - sp.GetRequiredService(), + sp.GetRequiredService(), sp.GetService>() ?? Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance)); + // The registered IAuditWriter is the bounded, asynchronous one: audit producers — above + // all IConstraintEnforcer.RecordDenialAsync, which fires once per denied tag inside bulk + // RPC loops — enqueue and return instead of awaiting a SQLite insert each. No producer + // signature changes; the seam is entirely here. AuditDrainService batches the buffered + // events onto the sink, owns the one-time schema bootstrap and sweeps expired rows. Where + // no hosted service runs (the `apikey` CLI, the DI-only tests) the channel writer falls + // back to CanonicalAuditWriter's synchronous path, so audit is never silently buffered + // into a channel nobody drains. + services.AddSingleton(sp => + new ChannelAuditWriter( + sp.GetRequiredService(), + sp.GetService>() + ?? Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance)); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => new AuditDrainService( + sp.GetRequiredService(), + sp.GetRequiredService(), + security, + sp.GetService() ?? TimeProvider.System, + sp.GetService>() + ?? Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance)); + services.AddHostedService(sp => sp.GetRequiredService()); + // OVERRIDE the library's IApiKeyAuditStore (AddZbApiKeyAuth registered the library's // SqliteApiKeyAuditStore via TryAddSingleton) with an adapter that canonicalizes every // library-emitted ApiKeyAuditEntry onto AuditEvent and forwards it through IAuditWriter. diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Security/Audit/ChannelAuditWriterTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Security/Audit/ChannelAuditWriterTests.cs new file mode 100644 index 0000000..c96dce5 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/Security/Audit/ChannelAuditWriterTests.cs @@ -0,0 +1,271 @@ +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Time.Testing; +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.Auth.ApiKeys.Sqlite; +using ZB.MOM.WW.MxGateway.Server.Configuration; +using ZB.MOM.WW.MxGateway.Server.Security.Audit; +using ZB.MOM.WW.MxGateway.Tests.Security.Authentication; + +namespace ZB.MOM.WW.MxGateway.Tests.Security.Audit; + +/// +/// Tests the bounded, asynchronous audit path: (enqueue-only +/// once a drain is attached, drops rather than blocks when the channel is full) and +/// (batched drain, one-time table bootstrap, retention sweep). +/// The channel makes the already-documented best-effort audit contract explicit and bounded: +/// a partially denied bulk RPC no longer pays a SQLite round-trip per denied tag. +/// +public sealed class ChannelAuditWriterTests : IDisposable +{ + private readonly List _tempDirectories = []; + + /// + /// With a drain attached, only enqueues — the + /// sink is not touched until the drain runs, and the event arrives once it does. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task WriteAsync_WithDrainAttached_EnqueuesOnlyAndReachesSinkAfterDrain() + { + CountingAuditSink sink = new(); + (ChannelAuditWriter writer, AuditDrainService drain) = CreateWriterAndDrain(sink); + writer.AttachDrain(); + + await writer.WriteAsync(MakeEvent("constraint-denied"), CancellationToken.None); + + // Enqueue-only: nothing has reached the sink yet. + Assert.Equal(0, sink.InsertBatchCalls); + Assert.Empty(sink.Events); + + int drained = await drain.DrainPendingAsync(CancellationToken.None); + + Assert.Equal(1, drained); + Assert.Equal(1, sink.InsertBatchCalls); + Assert.Equal("constraint-denied", Assert.Single(sink.Events).Action); + } + + /// + /// A drain pass batches up to events into one + /// sink call, so a partially denied bulk RPC costs a handful of transactions, not one per tag. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task DrainPendingAsync_BatchesEventsIntoBoundedInsertCalls() + { + CountingAuditSink sink = new(); + (ChannelAuditWriter writer, AuditDrainService drain) = CreateWriterAndDrain(sink); + writer.AttachDrain(); + + const int eventCount = 150; + for (int index = 0; index < eventCount; index++) + { + await writer.WriteAsync(MakeEvent($"denied-{index}"), CancellationToken.None); + } + + int drained = await drain.DrainPendingAsync(CancellationToken.None); + + Assert.Equal(eventCount, drained); + Assert.Equal(eventCount, sink.Events.Count); + // 150 events at a batch size of 64 → 3 transactions (64 + 64 + 22), not 150. + Assert.Equal(3, sink.InsertBatchCalls); + Assert.All(sink.BatchSizes, size => Assert.True(size <= AuditDrainService.MaxBatchSize)); + } + + /// + /// When the bounded channel is full the write is dropped rather than blocking the caller, + /// and the drop is counted. A stalled or slow audit database must never stall an RPC. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task WriteAsync_WhenChannelFull_DropsWriteAndCountsItWithoutBlocking() + { + CountingAuditSink sink = new(); + (ChannelAuditWriter writer, _) = CreateWriterAndDrain(sink); + writer.AttachDrain(); + + const int overflow = 32; + for (int index = 0; index < ChannelAuditWriter.ChannelCapacity + overflow; index++) + { + // Every call must complete synchronously: the channel never blocks a producer. + Task write = writer.WriteAsync(MakeEvent($"denied-{index}"), CancellationToken.None); + Assert.True(write.IsCompletedSuccessfully); + } + + Assert.Equal(overflow, writer.DroppedCount); + // The channel still holds exactly its capacity; nothing reached the sink (no drain ran). + Assert.Equal(0, sink.InsertBatchCalls); + + await Task.CompletedTask; + } + + /// + /// With no drain attached (the apikey CLI, DI-only tests, post-shutdown) the writer + /// writes through to the sink instead of enqueueing into a channel nobody will ever read. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task WriteAsync_WithNoDrainAttached_WritesThroughToSink() + { + CountingAuditSink sink = new(); + (ChannelAuditWriter writer, _) = CreateWriterAndDrain(sink); + + await writer.WriteAsync(MakeEvent("dashboard-create-key"), CancellationToken.None); + + Assert.Equal("dashboard-create-key", Assert.Single(sink.Events).Action); + } + + /// + /// The retention sweep deletes rows older than AuditRetentionDays, measured from the + /// injected clock, and runs once at startup after the one-time table bootstrap. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task StartAsync_BootstrapsTableThenSweepsRetentionAtConfiguredCutoff() + { + CountingAuditSink sink = new(); + FakeTimeProvider clock = new(new DateTimeOffset(2026, 8, 15, 12, 0, 0, TimeSpan.Zero)); + (_, AuditDrainService drain) = CreateWriterAndDrain( + sink, + new SecurityOptions { AuditRetentionDays = 30 }, + clock); + + await drain.StartAsync(CancellationToken.None); + await drain.StopAsync(CancellationToken.None); + + Assert.Equal(1, sink.EnsureInitializedCalls); + Assert.Equal(new DateTimeOffset(2026, 7, 16, 12, 0, 0, TimeSpan.Zero), Assert.Single(sink.DeleteCutoffs)); + } + + /// + /// The real SQLite sink deletes only rows older than the cutoff, and its bootstrap latch + /// means the CREATE TABLE round-trip is not repeated on every subsequent write. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task SqliteStore_DeleteOlderThan_RemovesOnlyRowsBeyondTheRetentionWindow() + { + SqliteCanonicalAuditStore store = CreateStore(); + DateTimeOffset now = new(2026, 8, 15, 12, 0, 0, TimeSpan.Zero); + + await store.InsertBatchAsync( + [ + MakeEvent("stale", now.AddDays(-120)), + // A non-UTC offset must still compare correctly against the cutoff. + MakeEvent("stale-offset", now.AddDays(-100).ToOffset(TimeSpan.FromHours(-5))), + MakeEvent("fresh", now.AddDays(-1)), + ], + CancellationToken.None); + + int deleted = await store.DeleteOlderThanAsync(now.AddDays(-90), CancellationToken.None); + + Assert.Equal(2, deleted); + Assert.Equal("fresh", Assert.Single(await store.ListRecentAsync(10, CancellationToken.None)).Action); + } + + private static (ChannelAuditWriter Writer, AuditDrainService Drain) CreateWriterAndDrain( + IAuditEventSink sink, + SecurityOptions? security = null, + TimeProvider? timeProvider = null) + { + ChannelAuditWriter writer = new( + new CanonicalAuditWriter(sink, NullLogger.Instance), + NullLogger.Instance); + AuditDrainService drain = new( + writer, + sink, + security ?? new SecurityOptions(), + timeProvider ?? TimeProvider.System, + NullLogger.Instance); + return (writer, drain); + } + + private SqliteCanonicalAuditStore CreateStore() + { + TempDatabaseDirectory directory = TempDatabaseDirectory.Create("mxgateway-channel-audit"); + _tempDirectories.Add(directory); + return new SqliteCanonicalAuditStore(new AuthSqliteConnectionFactory(directory.DatabasePath())); + } + + private static AuditEvent MakeEvent(string action, DateTimeOffset? occurredAtUtc = null) => new() + { + EventId = Guid.NewGuid(), + OccurredAtUtc = occurredAtUtc ?? DateTimeOffset.UtcNow, + Actor = "operator01", + Action = action, + Outcome = AuditOutcome.Denied, + Category = "ApiKey", + }; + + /// Clears SQLite pools and deletes every temporary directory created by this test. + public void Dispose() + { + foreach (TempDatabaseDirectory directory in _tempDirectories) + { + directory.Dispose(); + } + + _tempDirectories.Clear(); + } + + /// + /// In-memory that records every call, so a test can prove the + /// write path did NOT touch the sink and that the drain batched what it did write. + /// + private sealed class CountingAuditSink : IAuditEventSink + { + private readonly Lock _gate = new(); + + /// Gets the events handed to the sink, in the order they were written. + public List Events { get; } = []; + + /// Gets the size of each batch the sink was asked to insert. + public List BatchSizes { get; } = []; + + /// Gets the cutoffs the sink was asked to delete below. + public List DeleteCutoffs { get; } = []; + + /// Gets the number of calls. + public int InsertBatchCalls => BatchSizes.Count; + + /// Gets the number of calls. + public int EnsureInitializedCalls { get; private set; } + + /// + public Task EnsureInitializedAsync(CancellationToken cancellationToken) + { + lock (_gate) + { + EnsureInitializedCalls++; + } + + return Task.CompletedTask; + } + + /// + public Task InsertAsync(AuditEvent auditEvent, CancellationToken cancellationToken) => + InsertBatchAsync([auditEvent], cancellationToken); + + /// + public Task InsertBatchAsync(IReadOnlyList auditEvents, CancellationToken cancellationToken) + { + lock (_gate) + { + BatchSizes.Add(auditEvents.Count); + Events.AddRange(auditEvents); + } + + return Task.CompletedTask; + } + + /// + public Task DeleteOlderThanAsync(DateTimeOffset cutoffUtc, CancellationToken cancellationToken) + { + lock (_gate) + { + DeleteCutoffs.Add(cutoffUtc); + } + + return Task.FromResult(0); + } + } +} From ca34a2d65da6452e05a9fc79e9f139f14c73b23d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:17:27 -0400 Subject: [PATCH 05/41] fix(logging): fail-closed bearer redaction; hoist per-request logger creation --- docs/Diagnostics.md | 64 ++++------ .../Diagnostics/GatewayLogRedactor.cs | 118 +++++++++++++----- ...tewayRequestLoggingMiddlewareExtensions.cs | 12 +- .../Diagnostics/GatewayLogRedactorTests.cs | 61 +++++++++ 4 files changed, 182 insertions(+), 73 deletions(-) diff --git a/docs/Diagnostics.md b/docs/Diagnostics.md index 70c24d0..bec160c 100644 --- a/docs/Diagnostics.md +++ b/docs/Diagnostics.md @@ -84,42 +84,32 @@ The names match the MXAccess command list in `gateway.md` exactly. `Write` and ` ### API key redaction -`RedactApiKey` is built around the `mxgw_` API key format issued by the gateway. It preserves the bearer scheme and the key id segment so that operators can correlate a log entry to a specific principal, but always strips the secret tail: +`RedactClientIdentity` is the single redaction path for identity-bearing values; `RedactApiKey` is a +name-preserving alias for it. Redaction **fails closed**: the only value that survives with any of its +content is a gateway-issued `mxgw__` key, whose key id is kept so operators can +correlate a log entry to a specific principal. -```csharp -public static string? RedactApiKey(string? authorizationHeader) -{ - if (string.IsNullOrWhiteSpace(authorizationHeader)) - { - return authorizationHeader; - } +| Input | Output | Why | +|-------|--------|-----| +| `Bearer mxgw_operator01_super-secret` | `Bearer mxgw_operator01_[redacted]` | Recognized gateway key; key id identifies the principal | +| `Bearer eyJhbGciOi…` (any foreign token) | `Bearer [redacted]` | Structure is unknown, so the whole credential goes | +| `Basic dXNlcjpwYXNz` | `Basic [redacted]` | Same, for any recognized scheme | +| `Bearer mxgw_operator01` (no secret separator) | `Bearer mxgw_[redacted]` | No trustworthy key-id boundary | +| `Bearer` (scheme only), `anonymous`, `some junk` | `[redacted]` | No scheme/credential split that can be trusted | +| `null`, `""`, whitespace | unchanged | Nothing to redact | - const string bearerPrefix = "Bearer "; - if (!authorizationHeader.StartsWith(bearerPrefix, StringComparison.OrdinalIgnoreCase)) - { - return RedactedValue; - } +A scheme word survives only when it is one of the recognized authorization schemes (`Bearer`, +`Basic`, `Digest`, `Negotiate`, `NTLM`, `ApiKey`, `Token`). An unrecognized leading word is as likely +to be credential material as it is to be a scheme, so it is dropped along with the rest. The key id is +also dropped when it runs longer than 64 characters, which no issued key id does — a long run before +the first `_` is secret material, not an identifier. - string token = authorizationHeader[bearerPrefix.Length..].Trim(); +The parse is span-based (no regex, no `Split` allocation): the value is split once at the first space, +and the key id is read up to the first `_` of the remainder. - if (!token.StartsWith("mxgw_", StringComparison.OrdinalIgnoreCase)) - { - return $"{bearerPrefix}{RedactedValue}"; - } - - string[] tokenParts = token.Split('_', 3, StringSplitOptions.RemoveEmptyEntries); - if (tokenParts.Length < 2) - { - return $"{bearerPrefix}mxgw_{RedactedValue}"; - } - - return $"{bearerPrefix}mxgw_{tokenParts[1]}_{RedactedValue}"; -} -``` - -The split uses `count: 3` because the secret portion may itself contain underscores; only the first two segments (`mxgw` and the key id) are kept verbatim. Authorization headers that are not bearer tokens are reduced to `[redacted]` rather than passed through, since the gateway cannot reason about their structure. - -`RedactClientIdentity` is the entry point used by `GatewayLogScope` and `DashboardRedactor`. It only invokes `RedactApiKey` when the input contains the `mxgw_` marker, leaving non-key identities (for example, Windows account names) untouched. +The consequence for callers is that a non-key identity (for example a Windows account name) reaching +`RedactClientIdentity` is now replaced rather than passed through. `DashboardRedactor` routes only +values containing the `mxgw_` marker here, so dashboard display names are unaffected. ### Command value redaction @@ -160,12 +150,12 @@ public static IApplicationBuilder UseGatewayRequestLoggingScope(this IApplicatio { ArgumentNullException.ThrowIfNull(app); + ILogger logger = app.ApplicationServices + .GetRequiredService() + .CreateLogger("MxGateway.Request"); + return app.Use(async (context, next) => { - ILogger logger = context.RequestServices - .GetRequiredService() - .CreateLogger("ZB.MOM.WW.MxGateway.Request"); - using IDisposable? scope = logger.BeginGatewayScope(new GatewayLogScope( SessionId: ReadHeader(context, SessionIdHeaderName), WorkerProcessId: ReadInt32Header(context, WorkerProcessIdHeaderName), @@ -190,7 +180,7 @@ The scope is keyed off four custom headers and the standard `authorization` head The numeric headers use `int.TryParse` and `ulong.TryParse`; missing or unparseable values become `null` and are dropped by `GatewayLogScope.ToDictionary`. This keeps the middleware tolerant of clients that do not yet emit every header, which matters because the earliest call in a session (`OpenSession`) has no `SessionId` to send. -The logger category is `ZB.MOM.WW.MxGateway.Request`, which lets operators filter the request scope events independently from per-component categories. +The logger category is `MxGateway.Request`, which lets operators filter the request scope events independently from per-component categories. The logger is resolved once at registration rather than per request: the category is fixed, so a per-request `IServiceProvider` resolve and `ILoggerFactory.CreateLogger` (which takes the factory lock) bought nothing. Scope construction itself stays unconditional — gating it on `ILogger.IsEnabled` would drop scope state for providers and scope consumers registered after startup. ### Pipeline ordering diff --git a/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GatewayLogRedactor.cs b/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GatewayLogRedactor.cs index b22b081..bc517d9 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GatewayLogRedactor.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GatewayLogRedactor.cs @@ -15,6 +15,28 @@ public static class GatewayLogRedactor "WriteSecured2" }; + /// + /// Authorization schemes whose name may survive redaction. Anything outside this list is + /// dropped whole: an unrecognized leading word is as likely to be credential material as it + /// is to be a scheme, so it is not worth the leak. + /// + private static readonly string[] KnownAuthorizationSchemes = + [ + "Bearer", + "Basic", + "Digest", + "Negotiate", + "NTLM", + "ApiKey", + "Token", + ]; + + /// Prefix identifying a gateway-issued API key. + private const string GatewayKeyPrefix = "mxgw_"; + + /// Upper bound on a key id kept in the clear; a longer run is treated as secret material. + private const int MaxKeyIdLength = 64; + /// /// Determines whether a command method bears credentials. /// @@ -27,44 +49,24 @@ public static class GatewayLogRedactor } /// - /// Redacts the API key secret portion of a Bearer authorization header. + /// Redacts the credential portion of an authorization header value. /// /// The authorization header value to redact. - /// The header with the secret portion redacted, or the original value when it is null, blank, or not a Bearer header. + /// The header with the credential redacted, or the original value when it is null or blank. public static string? RedactApiKey(string? authorizationHeader) { - if (string.IsNullOrWhiteSpace(authorizationHeader)) - { - return authorizationHeader; - } - - const string bearerPrefix = "Bearer "; - if (!authorizationHeader.StartsWith(bearerPrefix, StringComparison.OrdinalIgnoreCase)) - { - return RedactedValue; - } - - string token = authorizationHeader[bearerPrefix.Length..].Trim(); - - if (!token.StartsWith("mxgw_", StringComparison.OrdinalIgnoreCase)) - { - return $"{bearerPrefix}{RedactedValue}"; - } - - string[] tokenParts = token.Split('_', 3, StringSplitOptions.RemoveEmptyEntries); - if (tokenParts.Length < 2) - { - return $"{bearerPrefix}mxgw_{RedactedValue}"; - } - - return $"{bearerPrefix}mxgw_{tokenParts[1]}_{RedactedValue}"; + return RedactClientIdentity(authorizationHeader); } /// - /// Redacts the client identity if it contains an API key. + /// Redacts the credential carried by a client identity. Redaction fails closed: only a + /// gateway-issued API key keeps its mxgw_<key-id>_ shape (so operators can tell keys + /// apart in logs), and only a recognized scheme keeps its name. Every other value — a foreign + /// bearer token, a scheme-less string, junk — is replaced whole, because nothing that reaches + /// this method is known to be safe to log. /// /// The client identity string to redact. - /// The redacted client identity, or the original value when it contains no API key. + /// The redacted client identity, or the original value when it is null or blank. public static string? RedactClientIdentity(string? clientIdentity) { if (string.IsNullOrWhiteSpace(clientIdentity)) @@ -72,9 +74,61 @@ public static class GatewayLogRedactor return clientIdentity; } - return clientIdentity.Contains("mxgw_", StringComparison.OrdinalIgnoreCase) - ? RedactApiKey(clientIdentity) - : clientIdentity; + ReadOnlySpan value = clientIdentity.AsSpan().Trim(); + int separatorIndex = value.IndexOf(' '); + + if (separatorIndex < 0) + { + // A single token carries no scheme, so the token itself is the credential. + return RedactedValue; + } + + ReadOnlySpan scheme = value[..separatorIndex]; + ReadOnlySpan credential = value[(separatorIndex + 1)..].Trim(); + + if (credential.IsEmpty || !IsKnownAuthorizationScheme(scheme)) + { + return RedactedValue; + } + + return credential.StartsWith(GatewayKeyPrefix, StringComparison.OrdinalIgnoreCase) + ? $"{scheme} {GatewayKeyPrefix}{RedactKeyId(credential)}" + : $"{scheme} {RedactedValue}"; + } + + /// + /// Renders the trailing portion of a gateway API key: the key id when the key is well formed, + /// otherwise nothing but the placeholder. + /// + /// The credential, known to start with the gateway key prefix. + /// The <key-id>_[redacted] tail, or just the placeholder. + private static string RedactKeyId(ReadOnlySpan credential) + { + ReadOnlySpan remainder = credential[GatewayKeyPrefix.Length..]; + int secretIndex = remainder.IndexOf('_'); + + // No separator means no secret boundary to trust, so the whole remainder is treated as secret. + return secretIndex is <= 0 or > MaxKeyIdLength + ? RedactedValue + : $"{remainder[..secretIndex]}_{RedactedValue}"; + } + + /// + /// Determines whether a leading word is a recognized authorization scheme. + /// + /// The candidate scheme word. + /// when the word may survive redaction; otherwise . + private static bool IsKnownAuthorizationScheme(ReadOnlySpan scheme) + { + foreach (string knownScheme in KnownAuthorizationSchemes) + { + if (scheme.Equals(knownScheme, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; } /// diff --git a/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GatewayRequestLoggingMiddlewareExtensions.cs b/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GatewayRequestLoggingMiddlewareExtensions.cs index 2b0e11b..2ae0de3 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GatewayRequestLoggingMiddlewareExtensions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Diagnostics/GatewayRequestLoggingMiddlewareExtensions.cs @@ -24,12 +24,16 @@ public static class GatewayRequestLoggingMiddlewareExtensions { ArgumentNullException.ThrowIfNull(app); + // Resolved once at registration: the logger is keyed by category, not by request, so the + // per-request DI resolve and logger-factory lock bought nothing. + ILogger logger = app.ApplicationServices + .GetRequiredService() + .CreateLogger("MxGateway.Request"); + return app.Use(async (context, next) => { - ILogger logger = context.RequestServices - .GetRequiredService() - .CreateLogger("MxGateway.Request"); - + // Scope construction is deliberately unconditional: gating it on IsEnabled would drop + // scope state for providers (and scope consumers) registered after startup. using IDisposable? scope = logger.BeginGatewayScope(new GatewayLogScope( SessionId: ReadHeader(context, SessionIdHeaderName), WorkerProcessId: ReadInt32Header(context, WorkerProcessIdHeaderName), diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/GatewayLogRedactorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/GatewayLogRedactorTests.cs index eedd129..70df716 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/GatewayLogRedactorTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/GatewayLogRedactorTests.cs @@ -24,6 +24,67 @@ public sealed class GatewayLogRedactorTests Assert.DoesNotContain("super_secret_value", redacted); } + /// + /// Verifies that a bearer credential the gateway does not issue is redacted too. A client that + /// pastes a JWT (or any other token) into the authorization header must not have it logged. + /// + [Fact] + public void RedactClientIdentity_RedactsNonGatewayBearerCredential() + { + const string token = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiIxIn0.c2lnbmF0dXJl"; + + string? redacted = GatewayLogRedactor.RedactClientIdentity($"Bearer {token}"); + + Assert.Equal("Bearer [redacted]", redacted); + Assert.DoesNotContain(token, redacted, StringComparison.Ordinal); + Assert.DoesNotContain("eyJ", redacted, StringComparison.Ordinal); + } + + /// Verifies that a gateway API key keeps its key-id shape so an operator can still tell keys apart. + [Fact] + public void RedactClientIdentity_PreservesGatewayKeyIdShape() + { + string? redacted = GatewayLogRedactor.RedactClientIdentity("Bearer mxgw_operator01_super-secret"); + + Assert.Equal("Bearer mxgw_operator01_[redacted]", redacted); + Assert.DoesNotContain("super-secret", redacted, StringComparison.Ordinal); + } + + /// + /// Verifies that anything not recognized as a gateway API key fails closed: the scheme word + /// survives only when it looks like an auth scheme, and the credential never does. + /// + /// The raw client identity value. + /// The expected redacted value. + [Theory] + [InlineData("Bearer", "[redacted]")] + [InlineData("Bearer ", "[redacted]")] + [InlineData("Basic dXNlcjpwYXNzd29yZA==", "Basic [redacted]")] + [InlineData("Negotiate YIIJvwYGKwYBBQUCoIIJ", "Negotiate [redacted]")] + [InlineData("mxgw_operator01_super-secret", "[redacted]")] + [InlineData("mxgw_operator01_super-secret trailing", "[redacted]")] + [InlineData("Bearer mxgw_operator01", "Bearer mxgw_[redacted]")] + [InlineData("Bearer mxgw_", "Bearer mxgw_[redacted]")] + [InlineData("Bearer mxgw__super-secret", "Bearer mxgw_[redacted]")] + [InlineData("bearer mxgw_operator01_super-secret", "bearer mxgw_operator01_[redacted]")] + [InlineData("anonymous", "[redacted]")] + [InlineData("some random junk", "[redacted]")] + public void RedactClientIdentity_FailsClosedForUnrecognizedCredentials(string clientIdentity, string expected) + { + Assert.Equal(expected, GatewayLogRedactor.RedactClientIdentity(clientIdentity)); + } + + /// Verifies that a blank client identity is passed through — there is nothing to redact. + /// The raw client identity value. + [Theory] + [InlineData(null)] + [InlineData("")] + [InlineData(" ")] + public void RedactClientIdentity_PassesThroughBlankValues(string? clientIdentity) + { + Assert.Equal(clientIdentity, GatewayLogRedactor.RedactClientIdentity(clientIdentity)); + } + /// Verifies that IsCredentialBearingCommand identifies credential-bearing MXAccess commands. /// Name of the MXAccess command method. [Theory] From f1e26fed4f22ee676f460dd546ad1332cfe72c10 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:20:51 -0400 Subject: [PATCH 06/41] perf(alarms): memoize CurrentAlarms projection, invalidate on mutation --- .../Alarms/GatewayAlarmMonitor.cs | 23 ++++++++- .../GatewayAlarmMonitorAttachOrderTests.cs | 49 +++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Alarms/GatewayAlarmMonitor.cs b/src/ZB.MOM.WW.MxGateway.Server/Alarms/GatewayAlarmMonitor.cs index 59bb783..b29d00a 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Alarms/GatewayAlarmMonitor.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Alarms/GatewayAlarmMonitor.cs @@ -34,6 +34,13 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic private readonly Dictionary _alarms = new(StringComparer.Ordinal); private readonly List _subscribers = []; + // Memoized CurrentAlarms projection, guarded by _sync: the cloned, read-only view of _alarms + // handed to the dashboard and the QueryActiveAlarms RPC. Cloning the whole set per read held + // _sync — the broadcast lock — for the length of the copy, so a polled dashboard stalled every + // ApplyTransition/Broadcast behind it. Null means "not built for the current generation": + // every path that writes _alarms must null this under _sync, or readers keep a stale set. + private ActiveAlarmSnapshot[]? _currentAlarmsProjection; + // NEXT-03 dedup tombstones, guarded by _sync: alarm instances whose Clear was synthesized by // the most recent reconcile pass, keyed by reference with the instance's original raise // timestamp as the identity marker. A buffered live Clear for the same instance is a duplicate @@ -93,7 +100,12 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic { lock (_sync) { - return _alarms.Values.Select(alarm => alarm.Clone()).ToArray(); + // Same clone semantics as an uncached read — callers still get instances no + // mutation can leak back into the cache — but built once per alarm-set + // generation instead of once per caller. + return _currentAlarmsProjection ??= _alarms.Values + .Select(alarm => alarm.Clone()) + .ToArray(); } } } @@ -422,6 +434,11 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic if (transition.TransitionKind == AlarmTransitionKind.Clear) { bool wasKnown = _alarms.Remove(reference); + if (wasKnown) + { + _currentAlarmsProjection = null; + } + if (!wasKnown && IsDuplicateOfReconcileClear(reference, transition)) { return; @@ -433,6 +450,7 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic bool duplicate = _alarms.TryGetValue(reference, out ActiveAlarmSnapshot? existing) && IsDuplicateOfCachedState(existing, snapshot); _alarms[reference] = snapshot; + _currentAlarmsProjection = null; if (duplicate) { return; @@ -650,6 +668,8 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic { _alarms[incoming.Key] = incoming.Value; } + + _currentAlarmsProjection = null; } } @@ -696,6 +716,7 @@ public sealed class GatewayAlarmMonitor : BackgroundService, IGatewayAlarmServic lock (_sync) { _alarms.Clear(); + _currentAlarmsProjection = null; } } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs index d262154..969645b 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Alarms/GatewayAlarmMonitorAttachOrderTests.cs @@ -282,6 +282,55 @@ public sealed class GatewayAlarmMonitorAttachOrderTests await monitor.StopAsync(CancellationToken.None); } + /// + /// clones the whole active-alarm set under + /// the broadcast lock, so rebuilding it per read stalls every transition and broadcast + /// behind the copy once the dashboard polls a large alarm set. The projection is memoized + /// for as long as the set is unchanged, and every mutation must invalidate it — a stale + /// projection would hide live transitions from the dashboard and the QueryActiveAlarms RPC. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task CurrentAlarmsProjectionIsMemoizedUntilTheAlarmSetChanges() + { + using GatewayMetrics metrics = new(); + await using FakeSessionManager sessions = new(); + using GatewayAlarmMonitor monitor = CreateMonitor(sessions, metrics); + + using CancellationTokenSource cts = new(); + await monitor.StartAsync(cts.Token); + await sessions.WaitForSubscribeStartAsync(WaitTimeout); + + // Seed through a reconcile (forced by a provider-mode probe) so the cache holds one + // unacked alarm and no further mutation is in flight. + sessions.SetReconcileSnapshot(Snapshot(AlarmConditionState.Active)); + sessions.EmitEvent(ProviderModeProbe(1)); + await WaitUntilAsync( + () => monitor.CurrentAlarms.Any(alarm => alarm.AlarmFullReference == AlarmReference + && alarm.CurrentState == AlarmConditionState.Active), + WaitTimeout); + + IReadOnlyList first = monitor.CurrentAlarms; + Assert.Same(first, monitor.CurrentAlarms); + + // A live Acknowledge replaces the cached snapshot, so the next read must rebuild. + sessions.EmitEvent(Transition(2, AlarmTransitionKind.Acknowledge)); + await WaitUntilAsync( + () => monitor.CurrentAlarms.Any(alarm => alarm.AlarmFullReference == AlarmReference + && alarm.CurrentState == AlarmConditionState.ActiveAcked), + WaitTimeout); + + IReadOnlyList second = monitor.CurrentAlarms; + Assert.NotSame(first, second); + Assert.Same(second, monitor.CurrentAlarms); + + // The pre-transition projection is a snapshot of the old generation, not a live view. + Assert.Equal(AlarmConditionState.Active, Assert.Single(first).CurrentState); + + await cts.CancelAsync(); + await monitor.StopAsync(CancellationToken.None); + } + private static GatewayAlarmMonitor CreateMonitor(FakeSessionManager sessions, GatewayMetrics metrics) { AlarmsOptions options = new() From 75e3dc27942e1ff35831fccae6b52df4ca8471fa Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:21:13 -0400 Subject: [PATCH 07/41] perf(dashboard): LRU cap on the shared live-read session's advised set --- docs/GatewayDashboardDesign.md | 46 ++- .../Dashboard/DashboardLiveDataService.cs | 160 +++++++- .../DashboardLiveDataServiceTests.cs | 344 ++++++++++++++++++ 3 files changed, 540 insertions(+), 10 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardLiveDataServiceTests.cs diff --git a/docs/GatewayDashboardDesign.md b/docs/GatewayDashboardDesign.md index 5267fdc..d2327fa 100644 --- a/docs/GatewayDashboardDesign.md +++ b/docs/GatewayDashboardDesign.md @@ -165,7 +165,7 @@ bearer). Each hub class is `[Authorize(Policy = HubClientsPolicy)]`. | Hub | Path | Producer | Payload | Routing | |---|---|---|---|---| -| `DashboardSnapshotHub` | `/hubs/snapshot` | `DashboardSnapshotPublisher` (BackgroundService consuming `IDashboardSnapshotService.WatchSnapshotsAsync`) | `DashboardSnapshot` | Sent to all connected clients on every snapshot tick; new connections receive the current snapshot synchronously in `OnConnectedAsync`. | +| `DashboardSnapshotHub` | `/hubs/snapshot` | `DashboardSnapshotPublisher` (BackgroundService consuming `IDashboardSnapshotService.WatchSnapshotsAsync`) | `DashboardSnapshot` | Sent to all connected clients on every snapshot tick, but only while at least one client is connected (see "Idle gating" below); new connections receive the current snapshot synchronously in `OnConnectedAsync`. | | `AlarmsHub` | `/hubs/alarms` | `AlarmsHubPublisher` (BackgroundService consuming `IGatewayAlarmService.StreamAsync(filter: null)`) | `AlarmFeedMessage` (`active_alarm` / `snapshot_complete` / `transition`) | Connected clients auto-join `__alarms__`; all clients receive every message. Publisher auto-reconnects every 5s on stream faults. | | `EventsHub` | `/hubs/events` | `DashboardEventBroadcaster` invoked by each session's internal dashboard-mirror subscriber on its `SessionEventDistributor` (registered when the session becomes Ready) | `MxEvent` | Clients call `SubscribeSession(sessionId)` to join `session:{id}`, which also registers them in `EventsHubViewerRegistry` — the mirror is gated on that registry (see "Mirror gating" below). The dashboard is a first-class distributor subscriber, so it receives the session's events whether or not a gRPC client is streaming. It sees RAW session events — not the per-gRPC-subscriber `AfterWorkerSequence` filtering that `EventStreamService` applies at its own boundary — because the dashboard is a separate LDAP-authenticated monitoring view meant to show the session's full event activity. Tag values are stripped from the mirrored `MxEvent` copy by `DashboardEventBroadcaster` when `Dashboard:ShowTagValues` is false (the default) — event metadata (tag reference, quality, status, timestamps) still renders, but the value fields are blanked, so no value leaks through this seam. The per-session hub ACL that would scope a Viewer to specific sessions is still outstanding (SEC-25 / remediation roadmap item 12); the value redaction is the near-term hardening that closes the value-leak seam independently of that ACL. | @@ -187,6 +187,38 @@ Default cadences: - event publisher emits per event fanned by the session's `SessionEventDistributor` to its internal dashboard-mirror subscriber (independent of any gRPC `StreamEvents`). +### Idle gating and snapshot cost + +A snapshot is not free: each one takes a session-registry snapshot and sorts it, +copies the metrics dictionaries under the global metrics lock, and projects +sessions, workers, faults, and the Galaxy summary. Without gating that work ran +once a second for the life of the process even when no browser was connected. + +`DashboardSnapshotHub` counts live connections into the singleton +`DashboardSnapshotHubConnectionCounter` (`OnConnectedAsync` / `OnDisconnectedAsync`, +clamped at zero). `DashboardSnapshotPublisher` reads that count before advancing the +snapshot enumerator: while it is zero the publisher does not call `MoveNextAsync` at +all, so the producing iterator stays suspended at its `yield` and builds nothing — +the gate removes the snapshot *build*, not just the broadcast. The publisher +re-checks once a second while idle, so the first viewer to connect resumes the tick +within roughly one snapshot interval. That viewer does not wait for it either: +`DashboardPageBase` seeds its first render synchronously from +`IDashboardSnapshotService.GetSnapshot()`, and `OnConnectedAsync` pushes a snapshot +to the new connection immediately. + +Two per-tick costs inside the snapshot itself are bounded independently of the gate: + +- the effective configuration (`EffectiveGatewayConfiguration`) is built once and + cached. It is a projection of `IOptions`, which the gateway binds + at startup and never reloads, so rebuilding the whole option tree every tick + produced an identical object; +- the API key summaries are refreshed at most once every 15 seconds + (`ApiKeySummaryRefreshInterval`) instead of on every tick. The list is a SQLite + read whose content changes only when an operator creates, rotates, or revokes a + key, so a key change reaches the dashboard within that interval. Only a + *successful* refresh restarts the interval, so a failed or timed-out read is + retried on the next tick and the previous summaries stay on screen. + Avoid pushing every MXAccess data-change event into a wider broadcast group. The current design routes events strictly through `session:{id}` groups; the snapshot hub continues to carry aggregate event counters and rates. @@ -362,6 +394,18 @@ its lease expires. One session means one worker process backs every dashboard circuit; all access is serialised so the worker sees one in-flight command at a time. Tag reads go through `GatewaySession.SubscribeBulkAsync` / `ReadBulkAsync`. +The advise set that backs those reads is capped at 256 tags (one browse page plus +headroom) and evicted least-recently-read-first. Without the cap every tag any +viewer ever inspected stayed advised on the single dashboard worker until the +session faulted, so browsing a large galaxy accreted unbounded live MXAccess +subscriptions — and the event churn they feed — on one x86 process. Reading a tag +already in the set marks it most-recently-read; subscribing past the cap unadvises +the oldest entries with `GatewaySession.UnsubscribeBulkAsync` in one batch before +the new ones are advised. Tags read in the same call are never evicted to make +room for each other. A failed unadvise does not fail the read: the tags are +dropped from tracking anyway (they re-subscribe if read again), because the +session-invalidation path already handles gateway/worker drift. + The Alarms page does **not** use the dashboard session: alarm data comes from the gateway's always-on central monitor. `QueryAlarmsAsync` reads `IGatewayAlarmService.CurrentAlarms` — the monitor's in-process cache — so the diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardLiveDataService.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardLiveDataService.cs index 55e9c56..d28a44c 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardLiveDataService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardLiveDataService.cs @@ -15,13 +15,26 @@ public sealed class DashboardLiveDataService : IDashboardLiveDataService, IAsync { private const string BackendName = "Galaxy"; private const string ClientName = "mxgateway-dashboard"; + + // One browse page of tags plus headroom. Bounds the standing advise load the + // single dashboard worker carries — and the event churn that advise set feeds — + // however much of a galaxy an operator browses through in one sitting. + private const int MaxSubscribedTags = 256; + private static readonly TimeSpan ReadTimeout = TimeSpan.FromSeconds(5); private readonly ISessionManager _sessionManager; private readonly IGatewayAlarmService _alarmService; private readonly ILogger _logger; private readonly SemaphoreSlim _gate = new(1, 1); - private readonly HashSet _subscribed = new(StringComparer.OrdinalIgnoreCase); + + // Least-recently-read-last advise set: the list holds every currently advised + // tag ordered most- to least-recently read, the dictionary indexes into it. + // Both are only ever touched under _gate, which already serialises all viewers. + private readonly Dictionary> _subscribed = + new(StringComparer.OrdinalIgnoreCase); + + private readonly LinkedList _recency = new(); private GatewaySession? _session; private int _serverHandle; @@ -58,15 +71,15 @@ public sealed class DashboardLiveDataService : IDashboardLiveDataService, IAsync (GatewaySession session, int serverHandle) = await EnsureReadyAsync(cancellationToken) .ConfigureAwait(false); - string[] toSubscribe = tagAddresses.Where(tag => !_subscribed.Contains(tag)).ToArray(); + string[] toSubscribe = TouchAndCollectNewTags(tagAddresses, out int justReadCount); if (toSubscribe.Length > 0) { - await session.SubscribeBulkAsync(serverHandle, toSubscribe, cancellationToken) + await EvictForAsync(session, serverHandle, toSubscribe.Length, justReadCount, cancellationToken) .ConfigureAwait(false); - foreach (string tag in toSubscribe) - { - _subscribed.Add(tag); - } + IReadOnlyList subscribeResults = await session + .SubscribeBulkAsync(serverHandle, toSubscribe, cancellationToken) + .ConfigureAwait(false); + TrackSubscribed(toSubscribe, subscribeResults); } IReadOnlyList results = await session @@ -107,6 +120,131 @@ public sealed class DashboardLiveDataService : IDashboardLiveDataService, IAsync return Task.FromResult(new DashboardAlarmQueryResult(alarms, error, _alarmService.WorkerProcessId)); } + // Promotes every already-advised tag in this read to the front of the recency + // list and returns the tags that still need subscribing (distinct, in request + // order). `justReadCount` is how many distinct tags of this read were already + // advised — they now occupy the front of the list and must never be evicted to + // make room for the same read's new tags. Callers must hold _gate. + private string[] TouchAndCollectNewTags(IReadOnlyCollection tagAddresses, out int justReadCount) + { + int touched = 0; + List toSubscribe = []; + HashSet seen = new(StringComparer.OrdinalIgnoreCase); + + foreach (string tag in tagAddresses) + { + if (_subscribed.TryGetValue(tag, out LinkedListNode? node)) + { + if (!ReferenceEquals(node, _recency.First)) + { + _recency.Remove(node); + _recency.AddFirst(node); + } + + if (seen.Add(tag)) + { + touched++; + } + } + else if (seen.Add(tag)) + { + toSubscribe.Add(tag); + } + } + + justReadCount = touched; + return [.. toSubscribe]; + } + + // Drops least-recently-read tags off the back of the advise set until the + // incoming tags fit under MaxSubscribedTags, unadvising them on the worker in + // one batch. A failed unadvise must not fail the read: the tags are dropped + // from tracking regardless, and the session-invalidation path already handles + // gateway/worker drift. Callers must hold _gate. + private async Task EvictForAsync( + GatewaySession session, + int serverHandle, + int incomingCount, + int justReadCount, + CancellationToken cancellationToken) + { + int overflow = _subscribed.Count + incomingCount - MaxSubscribedTags; + int evictable = _subscribed.Count - justReadCount; + int evictCount = Math.Min(overflow, evictable); + if (evictCount <= 0) + { + return; + } + + List evictedHandles = new(evictCount); + for (int i = 0; i < evictCount && _recency.Last is { } oldest; i++) + { + _recency.RemoveLast(); + _subscribed.Remove(oldest.Value.TagAddress); + if (oldest.Value.ItemHandle != 0) + { + evictedHandles.Add(oldest.Value.ItemHandle); + } + } + + _logger.LogDebug( + "Dashboard advise set hit its cap of {Cap}; evicted {EvictedCount} least-recently-read tags.", + MaxSubscribedTags, + evictCount); + + if (evictedHandles.Count == 0) + { + return; + } + + try + { + await session.UnsubscribeBulkAsync(serverHandle, evictedHandles, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception exception) when (exception is not OperationCanceledException) + { + _logger.LogDebug( + exception, + "Unadvising {EvictedCount} evicted dashboard tags failed; they stay dropped from tracking.", + evictedHandles.Count); + } + } + + // Records the freshly advised tags as the most recently read, keeping each + // tag's item handle so eviction can unadvise it. Tags the worker failed to + // advise are still tracked (matching the pre-cap behaviour of not retrying + // them on every read) but carry no handle, so eviction just forgets them. + // Callers must hold _gate. + private void TrackSubscribed(IReadOnlyList tagAddresses, IReadOnlyList results) + { + Dictionary handles = new(results.Count, StringComparer.OrdinalIgnoreCase); + foreach (SubscribeResult result in results) + { + if (result.WasSuccessful && !string.IsNullOrEmpty(result.TagAddress)) + { + handles[result.TagAddress] = result.ItemHandle; + } + } + + // Inserted back-to-front so the read's first tag ends up most recent. + for (int i = tagAddresses.Count - 1; i >= 0; i--) + { + string tag = tagAddresses[i]; + handles.TryGetValue(tag, out int itemHandle); + _subscribed[tag] = _recency.AddFirst(new SubscribedTag(tag, itemHandle)); + } + } + + // Forgets the whole advise set without unadvising: every call site is one where + // the backing session (and with it every item handle) is already gone. + // Callers must hold _gate. + private void ClearSubscriptions() + { + _subscribed.Clear(); + _recency.Clear(); + } + // Returns a Ready session + its Register server handle, opening a fresh // session when none exists or the current one is no longer usable. Callers // must hold _gate. @@ -132,7 +270,7 @@ public sealed class DashboardLiveDataService : IDashboardLiveDataService, IAsync await CloseQuietlyAsync(existing.SessionId).ConfigureAwait(false); } - _subscribed.Clear(); + ClearSubscriptions(); _session = null; GatewaySession session = await _sessionManager.OpenSessionAsync( @@ -178,7 +316,7 @@ public sealed class DashboardLiveDataService : IDashboardLiveDataService, IAsync { _session = null; _serverHandle = 0; - _subscribed.Clear(); + ClearSubscriptions(); } private async Task CloseQuietlyAsync(string sessionId) @@ -212,4 +350,8 @@ public sealed class DashboardLiveDataService : IDashboardLiveDataService, IAsync _gate.Dispose(); } + + // One entry of the advise set. ItemHandle is the handle the worker bound for + // the tag, or 0 when the subscribe failed and there is nothing to unadvise. + private readonly record struct SubscribedTag(string TagAddress, int ItemHandle); } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardLiveDataServiceTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardLiveDataServiceTests.cs new file mode 100644 index 0000000..dcb12ca --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardLiveDataServiceTests.cs @@ -0,0 +1,344 @@ +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; +using Microsoft.Extensions.Logging.Abstractions; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.MxGateway.Server.Dashboard; +using ZB.MOM.WW.MxGateway.Server.Sessions; +using ZB.MOM.WW.MxGateway.Server.Workers; +using ZB.MOM.WW.MxGateway.Tests.TestSupport; + +namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard; + +public sealed class DashboardLiveDataServiceTests +{ + // Mirrors DashboardLiveDataService.MaxSubscribedTags — the cap is private, so the + // tests drive it through the public read surface at exactly its documented size. + private const int MaxSubscribedTags = 256; + + /// + /// Verifies a tag already in the advise set is not subscribed again on a later read. + /// + [Fact] + public async Task ReadAsync_WhenTagAlreadySubscribed_DoesNotResubscribe() + { + RecordingWorkerClient worker = new(); + await using FakeSessionManager sessionManager = new(worker); + await using DashboardLiveDataService service = CreateService(sessionManager); + + await service.ReadAsync(["Tank_001.PV", "Tank_002.PV"], CancellationToken.None); + DashboardLiveReadResult second = await service.ReadAsync( + ["Tank_001.PV", "Tank_002.PV"], + CancellationToken.None); + + Assert.Null(second.Error); + Assert.Equal(["Tank_001.PV", "Tank_002.PV"], worker.SubscribedTags); + Assert.Empty(worker.UnsubscribedHandles); + } + + /// + /// Verifies the advise set is capped: subscribing past the cap unadvises the + /// least-recently-read tag on the worker and leaves the re-read tag advised. + /// + [Fact] + public async Task ReadAsync_PastCap_EvictsLeastRecentlyReadTag() + { + RecordingWorkerClient worker = new(); + await using FakeSessionManager sessionManager = new(worker); + await using DashboardLiveDataService service = CreateService(sessionManager); + + string[] filler = CreateTagAddresses(MaxSubscribedTags); + await service.ReadAsync(filler, CancellationToken.None); + + // Re-read the tag that is currently least recent, making it most recent: the + // tag read before it is now the eviction candidate. + await service.ReadAsync([filler[^1]], CancellationToken.None); + Assert.Equal(MaxSubscribedTags, worker.SubscribedTags.Count); + + DashboardLiveReadResult overflow = await service.ReadAsync( + ["Overflow.PV"], + CancellationToken.None); + + Assert.Null(overflow.Error); + Assert.Equal([worker.HandleFor(filler[^2])], worker.UnsubscribedHandles); + Assert.Equal("Overflow.PV", worker.SubscribedTags[^1]); + Assert.Equal(MaxSubscribedTags + 1, worker.SubscribedTags.Count); + + // The evicted tag is no longer tracked and re-subscribes; the re-read one does not. + await service.ReadAsync([filler[^1], filler[^2]], CancellationToken.None); + Assert.Equal(filler[^2], worker.SubscribedTags[^1]); + Assert.Equal(MaxSubscribedTags + 2, worker.SubscribedTags.Count); + } + + /// + /// Verifies a failed unadvise of an evicted tag does not fail the read, and the + /// evicted tag is dropped from tracking anyway. + /// + [Fact] + public async Task ReadAsync_WhenEvictionUnsubscribeFails_StillCompletesRead() + { + RecordingWorkerClient worker = new() { FailUnsubscribe = true }; + await using FakeSessionManager sessionManager = new(worker); + await using DashboardLiveDataService service = CreateService(sessionManager); + + string[] filler = CreateTagAddresses(MaxSubscribedTags); + await service.ReadAsync(filler, CancellationToken.None); + + DashboardLiveReadResult overflow = await service.ReadAsync( + ["Overflow.PV"], + CancellationToken.None); + + Assert.Null(overflow.Error); + Assert.Equal("Overflow.PV", Assert.Single(overflow.Values).TagAddress); + Assert.Equal(1, sessionManager.OpenCount); + + // The evicted tag was dropped from tracking despite the failed unadvise. + await service.ReadAsync([filler[^1]], CancellationToken.None); + Assert.Equal(filler[^1], worker.SubscribedTags[^1]); + } + + private static DashboardLiveDataService CreateService(ISessionManager sessionManager) + { + return new DashboardLiveDataService( + sessionManager, + new FakeGatewayAlarmService(), + NullLogger.Instance); + } + + private static string[] CreateTagAddresses(int count) + { + string[] addresses = new string[count]; + for (int i = 0; i < count; i++) + { + addresses[i] = $"Tank_{i:D4}.PV"; + } + + return addresses; + } + + // Serves the dashboard service a single Ready session backed by the recording + // worker, so reads exercise the real GatewaySession bulk command path. + private sealed class FakeSessionManager(RecordingWorkerClient workerClient) : ISessionManager, IAsyncDisposable + { + private readonly List _sessions = []; + + /// Gets the number of sessions the dashboard service opened. + public int OpenCount { get; private set; } + + /// + public Task OpenSessionAsync( + SessionOpenRequest request, + string? clientIdentity, + string? ownerKeyId, + CancellationToken cancellationToken) + { + OpenCount++; + string sessionId = $"dashboard-session-{OpenCount}"; + GatewaySession session = new( + sessionId, + "Galaxy", + $"mxgw-1-{sessionId}", + "nonce", + clientIdentity, + request.ClientSessionName, + request.ClientCorrelationId, + TimeSpan.FromSeconds(30), + TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(5), + DateTimeOffset.UnixEpoch); + session.AttachWorkerClient(workerClient); + session.MarkReady(); + _sessions.Add(session); + return Task.FromResult(session); + } + + /// + public bool TryGetSession(string sessionId, [MaybeNullWhen(false)] out GatewaySession session) + { + session = _sessions.Find(candidate => candidate.SessionId == sessionId); + return session is not null; + } + + /// + public Task CloseSessionAsync(string sessionId, CancellationToken cancellationToken) + { + return Task.FromResult(new SessionCloseResult(sessionId, SessionState.Closed, AlreadyClosed: false)); + } + + /// + public Task InvokeAsync( + string sessionId, + WorkerCommand command, + CancellationToken cancellationToken) => + throw new NotSupportedException(); + + /// + public IAsyncEnumerable ReadEventsAsync(string sessionId, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + /// + public Task KillWorkerAsync( + string sessionId, + string reason, + CancellationToken cancellationToken) => + throw new NotSupportedException(); + + /// + public Task CloseExpiredLeasesAsync(DateTimeOffset now, CancellationToken cancellationToken) => + throw new NotSupportedException(); + + /// + public Task ShutdownAsync(CancellationToken cancellationToken) => throw new NotSupportedException(); + + /// Disposes every session handed to the dashboard service. + /// A task that represents the asynchronous operation. + public async ValueTask DisposeAsync() + { + foreach (GatewaySession session in _sessions) + { + await session.DisposeAsync().ConfigureAwait(false); + } + } + } + + // Answers Register / SubscribeBulk / UnsubscribeBulk / ReadBulk with successful + // replies and records what the dashboard advised and unadvised. + private sealed class RecordingWorkerClient : IWorkerClient + { + private const int RegisteredServerHandle = 77; + + private readonly Dictionary _itemHandles = new(StringComparer.OrdinalIgnoreCase); + private int _nextItemHandle = 1000; + + /// + public string SessionId => "dashboard-session-1"; + + /// + public int? ProcessId => 4242; + + /// + public WorkerClientState State => WorkerClientState.Ready; + + /// + public DateTimeOffset LastHeartbeatAt => DateTimeOffset.UnixEpoch; + + /// Gets the tag addresses subscribed, in the order the dashboard asked for them. + public List SubscribedTags { get; } = []; + + /// Gets the item handles the dashboard unsubscribed, in order. + public List UnsubscribedHandles { get; } = []; + + /// Gets or sets a value indicating whether unsubscribe commands throw. + public bool FailUnsubscribe { get; set; } + + /// Gets the item handle bound for a previously subscribed tag. + /// Tag address to look up. + /// The bound item handle. + public int HandleFor(string tagAddress) => _itemHandles[tagAddress]; + + /// + public Task StartAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + /// + public Task InvokeAsync( + WorkerCommand command, + TimeSpan timeout, + CancellationToken cancellationToken) + { + MxCommand mxCommand = command.Command + ?? throw new InvalidOperationException("The dashboard sent a command with no payload."); + + MxCommandReply reply = new() + { + Kind = mxCommand.Kind, + ProtocolStatus = new ProtocolStatus { Code = ProtocolStatusCode.Ok }, + }; + + switch (mxCommand.Kind) + { + case MxCommandKind.Register: + reply.Register = new RegisterReply { ServerHandle = RegisteredServerHandle }; + break; + case MxCommandKind.SubscribeBulk: + reply.SubscribeBulk = Subscribe(mxCommand.SubscribeBulk.TagAddresses); + break; + case MxCommandKind.UnsubscribeBulk: + if (FailUnsubscribe) + { + throw new InvalidOperationException("Simulated worker unsubscribe failure."); + } + + UnsubscribedHandles.AddRange(mxCommand.UnsubscribeBulk.ItemHandles); + reply.UnsubscribeBulk = new BulkSubscribeReply(); + break; + case MxCommandKind.ReadBulk: + reply.ReadBulk = Read(mxCommand.ReadBulk.TagAddresses); + break; + default: + throw new NotSupportedException($"Unexpected dashboard command {mxCommand.Kind}."); + } + + return Task.FromResult(new WorkerCommandReply { Reply = reply }); + } + + /// + public async IAsyncEnumerable ReadEventsAsync( + [EnumeratorCancellation] CancellationToken cancellationToken) + { + await Task.CompletedTask.ConfigureAwait(false); + yield break; + } + + /// + public Task ShutdownAsync(TimeSpan timeout, CancellationToken cancellationToken) => Task.CompletedTask; + + /// + public void Kill(string reason) + { + } + + /// + public ValueTask DisposeAsync() => ValueTask.CompletedTask; + + private BulkSubscribeReply Subscribe(IEnumerable tagAddresses) + { + BulkSubscribeReply subscribeReply = new(); + foreach (string tagAddress in tagAddresses) + { + SubscribedTags.Add(tagAddress); + if (!_itemHandles.TryGetValue(tagAddress, out int itemHandle)) + { + itemHandle = _nextItemHandle++; + _itemHandles[tagAddress] = itemHandle; + } + + subscribeReply.Results.Add(new SubscribeResult + { + ServerHandle = RegisteredServerHandle, + TagAddress = tagAddress, + ItemHandle = itemHandle, + WasSuccessful = true, + }); + } + + return subscribeReply; + } + + private BulkReadReply Read(IEnumerable tagAddresses) + { + BulkReadReply readReply = new(); + foreach (string tagAddress in tagAddresses) + { + readReply.Results.Add(new BulkReadResult + { + ServerHandle = RegisteredServerHandle, + TagAddress = tagAddress, + ItemHandle = _itemHandles.TryGetValue(tagAddress, out int itemHandle) ? itemHandle : 0, + WasSuccessful = true, + Quality = 192, + }); + } + + return readReply; + } + } +} From e04b1c91993fcab8728498f913b50f72652df3c4 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:21:42 -0400 Subject: [PATCH 08/41] perf(events): copy-on-write subscriber snapshot in fan-out pump --- .../Sessions/SessionEventDistributor.cs | 124 ++++++++++++++---- .../Sessions/SessionEventDistributorTests.cs | 57 ++++++++ 2 files changed, 155 insertions(+), 26 deletions(-) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs index 797122b..441ce1c 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs @@ -67,11 +67,22 @@ public delegate void SubscriberOverflowHandler(bool isOnlySubscriber, bool isInt /// /// /// Concurrency. The subscriber set is a -/// keyed by a monotonic id. -/// The pump iterates it with a snapshot-free enumerator (which never throws on -/// concurrent add/remove), and / lease disposal mutate it -/// without any lock held across an await. Each subscriber channel has a -/// single writer — the pump — so per-channel writes never race. MXAccess parity: +/// keyed by a monotonic id, used +/// for keyed add/remove only. Fan-out does NOT enumerate the dictionary: every +/// mutation (, , lease +/// disposal, overflow disconnect) happens inside the _lifecycleLock critical +/// section and rebuilds an immutable copy-on-write Subscriber[] snapshot, +/// which the pump reads once per event. This matters because +/// ConcurrentDictionary.Values is a PROPERTY that acquires every internal +/// lock and materializes a fresh List plus a read-only wrapper on each call +/// — per event, on the hot fan-out path. The subscriber set is tiny (one to a +/// handful) and mutates rarely, so paying a full array rebuild per registration to +/// make fan-out a bare array walk is the right trade. No lock is held across an +/// await. Each subscriber channel has a single writer — the pump — so +/// per-channel writes never race. A subscriber registered after the pump captured +/// the array for the in-flight event misses that event, which matches "late +/// subscribers see events after they register"; the reconnect path closes that +/// window deliberately (see ). MXAccess parity: /// events are fanned in the order received; the pump never reorders or /// synthesizes events. /// @@ -97,6 +108,14 @@ public sealed class SessionEventDistributor : IAsyncDisposable private readonly CancellationTokenSource _shutdownCts = new(); private readonly object _lifecycleLock = new(); + // Copy-on-write fan-out snapshot of _subscribers.Values. Rebuilt (a whole new array) + // inside the _lifecycleLock section of every register/unregister; never mutated in + // place, so the pump can walk the array it captured with no lock and no allocation. + // Written with Volatile.Write / read with Volatile.Read so a reader on another core + // cannot observe a stale reference after the publishing store. See the type remarks + // for why fan-out must not touch ConcurrentDictionary.Values. + private Subscriber[] _subscriberSnapshot = []; + // Replay ring buffer. Appended on the pump thread and queried from arbitrary // threads via TryGetReplayFrom, so every access is under _replayLock. Backed by a // fixed-size circular array preallocated to the capacity so appending a retained @@ -268,7 +287,12 @@ public sealed class SessionEventDistributor : IAsyncDisposable /// , which tracks only external /// (gRPC) subscribers and excludes the internal dashboard subscriber. /// - public int SubscriberCount => _subscribers.Count; + /// + /// Read from the copy-on-write snapshot rather than ConcurrentDictionary.Count + /// (which acquires every internal lock). The snapshot is rebuilt in the same + /// _lifecycleLock section that mutates the dictionary, so the two never diverge. + /// + public int SubscriberCount => Volatile.Read(ref _subscriberSnapshot).Length; /// /// Starts the background pump. Idempotent — a second call is a no-op. @@ -332,6 +356,7 @@ public sealed class SessionEventDistributor : IAsyncDisposable { ObjectDisposedException.ThrowIf(_disposed, this); _subscribers[subscriber.Id] = subscriber; + RebuildSubscriberSnapshot(); // Close the register-after-pump-completion window: if the pump already ran its // final CompleteAllSubscribers (source completed/faulted) but the distributor is @@ -416,24 +441,26 @@ public sealed class SessionEventDistributor : IAsyncDisposable /// /// Why this is atomic and the handoff is correct. The replay snapshot and the /// subscriber registration both run inside the SAME _replayLock critical - /// section. The pump appends each event to the replay buffer under _replayLock - /// before fanning it to subscribers (outside the lock). Therefore, relative - /// to this method's critical section, for every event E: + /// section. The pump appends each event to the replay buffer AND captures the + /// copy-on-write subscriber array in one _replayLock section, then fans the + /// event to that captured array outside the lock. Mutual exclusion therefore places + /// every event E strictly on one side of this method's critical section: /// /// /// /// If the pump appended E before this critical section, E is in /// (when newer than - /// ). The pump's fan-out of E may race the - /// registration: if it writes E to this new channel too, E's sequence is - /// <= liveResumeSequence, so the caller's live filter DROPS it — no - /// duplicate. + /// ). The pump captured its subscriber array in + /// that same earlier section, so it cannot fan E into this not-yet-registered + /// channel. Even if it could, E's sequence is <= liveResumeSequence, so + /// the caller's live filter DROPS it — no duplicate either way. /// /// /// If the pump appends E after this critical section, E is NOT in the snapshot, - /// but this subscriber is already registered, so the pump fans E into the live - /// channel with sequence > liveResumeSequence — delivered as live, no - /// gap. + /// but this subscriber was registered — and the snapshot array republished — + /// before that section began, so the pump's capture includes it and E is fanned + /// into the live channel with sequence > liveResumeSequence — delivered + /// as live, no gap. /// /// /// @@ -508,6 +535,7 @@ public sealed class SessionEventDistributor : IAsyncDisposable { ObjectDisposedException.ThrowIf(_disposed, this); _subscribers[id] = subscriber; + RebuildSubscriberSnapshot(); // Same register-after-pump-completion guard as Register: a resume that races in // after the source already ended still gets its retained replay batch (snapshot @@ -591,13 +619,19 @@ public sealed class SessionEventDistributor : IAsyncDisposable // Retain for replay BEFORE fan-out so a reconnecting subscriber that // queries between fan-out and its own read still sees this event. Order // is preserved: the pump is the single appender and events arrive in - // source order. - AppendToReplayBuffer(mxEvent); + // source order. The same call returns the subscriber array to fan to, + // captured under _replayLock — see the method for why the capture must + // share the append's critical section. + Subscriber[] subscribers = AppendToReplayBufferAndCaptureSubscribers(mxEvent); - // Enumerating a ConcurrentDictionary's Values never throws on concurrent - // add/remove; a subscriber registered mid-iteration may miss this event, - // which matches "late subscribers see events after they register". - foreach (Subscriber subscriber in _subscribers.Values) + // Walk the captured copy-on-write array: no dictionary enumeration, no + // per-event allocation. A subscriber registered after this capture misses + // this event, which matches "late subscribers see events after they + // register". A subscriber unregistered after the capture is still written + // to — the identical window the previous ConcurrentDictionary.Values + // enumeration had (that property also materialized its list up front), so + // the race and its outcome are unchanged by the copy-on-write array. + foreach (Subscriber subscriber in subscribers) { // Non-blocking write: TryWrite never blocks the pump on a slow reader. // A false return means this subscriber's bounded channel is full — the @@ -669,7 +703,11 @@ public sealed class SessionEventDistributor : IAsyncDisposable // remove it from the fan-out set. Its gRPC reader's MoveNextAsync then throws the // SessionManagerException, which EventStreamService surfaces to the client exactly as // the pre-epic per-RPC overflow did. The pump and every other subscriber are untouched. - if (_subscribers.TryRemove(subscriber.Id, out _)) + // + // The removal takes _lifecycleLock because it must republish the fan-out snapshot; the + // pump holds no other lock here (fan-out runs outside _replayLock), so this cannot + // invert the _replayLock-then-_lifecycleLock order RegisterWithReplay uses. + if (RemoveSubscriber(subscriber)) { subscriber.Channel.Writer.TryComplete(new SessionManagerException( SessionManagerErrorCode.EventQueueOverflow, @@ -699,12 +737,36 @@ public sealed class SessionEventDistributor : IAsyncDisposable private void Unregister(Subscriber subscriber) { - if (_subscribers.TryRemove(subscriber.Id, out _)) + if (RemoveSubscriber(subscriber)) { subscriber.Channel.Writer.TryComplete(); } } + // Removes a subscriber from the fan-out set and republishes the copy-on-write snapshot. + // Returns true only for the caller that actually removed it, so the channel is completed + // exactly once however many disposal/overflow paths race. Completing the channel is left to + // that caller and happens OUTSIDE the lock: this lock guards set membership only. + private bool RemoveSubscriber(Subscriber subscriber) + { + lock (_lifecycleLock) + { + if (!_subscribers.TryRemove(subscriber.Id, out _)) + { + return false; + } + + RebuildSubscriberSnapshot(); + return true; + } + } + + // Republishes the fan-out array from the current dictionary contents. MUST be called with + // _lifecycleLock held — holding that lock across the dictionary mutation and this rebuild is + // what keeps the array and the dictionary from diverging. + private void RebuildSubscriberSnapshot() + => Volatile.Write(ref _subscriberSnapshot, [.. _subscribers.Values]); + /// /// Returns the retained events with strictly /// greater than , in ascending sequence order, so a @@ -791,7 +853,16 @@ public sealed class SessionEventDistributor : IAsyncDisposable } } - private void AppendToReplayBuffer(MxEvent mxEvent) + // Appends an event to the replay ring AND captures the fan-out array the pump will write it + // to, in ONE _replayLock section. The capture must share the append's critical section, not + // follow it: RegisterWithReplay snapshots the ring and registers under the same _replayLock, + // so mutual exclusion is what puts each event strictly before or strictly after a resume — + // "replayed, not fanned" or "fanned, not replayed", never neither. Capturing after the lock + // released would let a resume interleave between the append and the capture, replaying + // nothing for the event and fanning it to a stale array that omits the new subscriber: a + // silently dropped event. Returns the array; the pump fans OUTSIDE the lock so a slow + // reader can never stall replay. + private Subscriber[] AppendToReplayBufferAndCaptureSubscribers(MxEvent mxEvent) { lock (_replayLock) { @@ -805,7 +876,7 @@ public sealed class SessionEventDistributor : IAsyncDisposable // can still report a gap) but keep no events. if (_replayBufferCapacity == 0) { - return; + return Volatile.Read(ref _subscriberSnapshot); } // Append at the logical tail. When the ring is full the oldest entry is @@ -824,6 +895,7 @@ public sealed class SessionEventDistributor : IAsyncDisposable } EvictAged(); + return Volatile.Read(ref _subscriberSnapshot); } } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs index e910549..75a74ad 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs @@ -1057,6 +1057,63 @@ public sealed class SessionEventDistributorTests Assert.False(lateCts.IsCancellationRequested); } + /// + /// Guards the copy-on-write fan-out snapshot: registrations and unregistrations churn on + /// another thread while the pump is actively fanning events, and the stable subscriber + /// must still receive every event exactly once and in order. The pump captures the + /// subscriber array once per event instead of enumerating the dictionary, so a mutation + /// racing the fan-out must never drop, duplicate, or reorder an event for a subscriber + /// registered throughout — nor leave the array and the dictionary disagreeing on the + /// subscriber count once the churn stops. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task RegistrationChurnDuringFanOut_StableSubscriberStillReceivesEveryEventInOrder() + { + // Below the 64-event per-subscriber queue capacity, so the stable subscriber cannot + // overflow and be disconnected while the writes race the churn — the assertion stays + // deterministic no matter how the threads interleave. + const int EventCount = 50; + + Channel source = Channel.CreateUnbounded(); + await using SessionEventDistributor distributor = CreateDistributor(source.Reader); + await distributor.StartAsync(CancellationToken.None); + + using IEventSubscriberLease stable = distributor.Register(); + + using CancellationTokenSource churnCts = new(); + Task churn = Task.Run(async () => + { + while (!churnCts.IsCancellationRequested) + { + // Register then immediately unregister: every iteration rebuilds the fan-out + // snapshot twice, maximizing the chance of landing inside a fan-out pass. + distributor.Register().Dispose(); + await Task.Yield(); + } + }); + + for (ulong sequence = 1; sequence <= EventCount; sequence++) + { + source.Writer.TryWrite(Event(sequence)); + } + + List received = []; + for (int i = 0; i < EventCount; i++) + { + received.Add((await ReadOneAsync(stable.Reader)).WorkerSequence); + } + + await churnCts.CancelAsync(); + await churn.WaitAsync(ReadTimeout); + + Assert.Equal(Enumerable.Range(1, EventCount).Select(sequence => (ulong)sequence), received); + + // Only the stable subscriber remains: the snapshot the count is read from tracked every + // add and remove the churn performed. + Assert.Equal(1, distributor.SubscriberCount); + } + private static SessionEventDistributor CreateDistributor(ChannelReader source) => CreateDistributor(source, replayBufferCapacity: 1024, replayRetentionSeconds: 300); From 77c5731b7b59f0b927f093b27d3fc8631ca376f2 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:21:49 -0400 Subject: [PATCH 09/41] perf(dashboard): idle-gate the snapshot tick; cache static config; bound key-list refresh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The snapshot publisher broadcast to Clients.All on every ~1s tick forever, with zero viewers. Each tick cost a session-registry snapshot and sort, a metrics snapshot that copies dictionaries under the global metrics lock, a full rebuild of EffectiveGatewayConfiguration, and a SQLite read of the API key table. DashboardSnapshotHub now counts live connections into the singleton DashboardSnapshotHubConnectionCounter (clamped at zero, since SignalR can call OnDisconnectedAsync for a connection whose OnConnectedAsync faulted). The publisher drives the snapshot enumerator by hand instead of await foreach: with no connections it does not call MoveNextAsync at all, so the producing iterator stays suspended at its yield and no snapshot is built — the gate removes the build, not just the broadcast. It re-checks once a second, so the first viewer resumes the tick within about one interval; that viewer is seeded immediately by DashboardPageBase's synchronous GetSnapshot() and by the hub's OnConnectedAsync. Two per-tick costs are bounded independently of the gate: the effective configuration is startup-static (options are bound once at boot and never reloaded), so it is built once and cached; and the API key summaries refresh at most every 15s, since the list only changes when an operator creates, rotates, or revokes a key. Only a successful refresh restarts the interval, so a failed or timed-out read is still retried on the next tick with the previous summaries left on screen. --- docs/GatewayDashboardDesign.md | 12 -- .../DashboardServiceCollectionExtensions.cs | 1 + .../Dashboard/DashboardSnapshotService.cs | 45 ++++++- .../Dashboard/Hubs/DashboardSnapshotHub.cs | 59 ++++++++- .../Hubs/DashboardSnapshotPublisher.cs | 54 +++++++-- .../DashboardSnapshotPublisherTests.cs | 85 ++++++++++++- .../DashboardSnapshotServiceTests.cs | 112 ++++++++++++++++-- 7 files changed, 338 insertions(+), 30 deletions(-) diff --git a/docs/GatewayDashboardDesign.md b/docs/GatewayDashboardDesign.md index d2327fa..0beb73a 100644 --- a/docs/GatewayDashboardDesign.md +++ b/docs/GatewayDashboardDesign.md @@ -394,18 +394,6 @@ its lease expires. One session means one worker process backs every dashboard circuit; all access is serialised so the worker sees one in-flight command at a time. Tag reads go through `GatewaySession.SubscribeBulkAsync` / `ReadBulkAsync`. -The advise set that backs those reads is capped at 256 tags (one browse page plus -headroom) and evicted least-recently-read-first. Without the cap every tag any -viewer ever inspected stayed advised on the single dashboard worker until the -session faulted, so browsing a large galaxy accreted unbounded live MXAccess -subscriptions — and the event churn they feed — on one x86 process. Reading a tag -already in the set marks it most-recently-read; subscribing past the cap unadvises -the oldest entries with `GatewaySession.UnsubscribeBulkAsync` in one batch before -the new ones are advised. Tags read in the same call are never evicted to make -room for each other. A failed unadvise does not fail the read: the tags are -dropped from tracking anyway (they re-subscribe if read again), because the -session-invalidation path already handles gateway/worker drift. - The Alarms page does **not** use the dashboard session: alarm data comes from the gateway's always-on central monitor. `QueryAlarmsAsync` reads `IGatewayAlarmService.CurrentAlarms` — the monitor's in-process cache — so the diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardServiceCollectionExtensions.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardServiceCollectionExtensions.cs index c28f859..42e014f 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardServiceCollectionExtensions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardServiceCollectionExtensions.cs @@ -51,6 +51,7 @@ public static class DashboardServiceCollectionExtensions // subscriber bookkeeping they share with the broadcaster must outlive them. services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddHostedService(); services.AddHostedService(); services.AddHttpContextAccessor(); diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs index 7039f47..49dc7be 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs @@ -16,6 +16,17 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService { private const string HealthyStatus = "Healthy"; + /// + /// Minimum spacing between API key list reads. The list is a SQLite query whose + /// content only changes when an operator creates, rotates, or revokes a key, so + /// refreshing it on every ~1s snapshot tick buys nothing; the dashboard still sees + /// a key change within this interval. + /// + private static readonly TimeSpan ApiKeySummaryRefreshInterval = TimeSpan.FromSeconds(15); + + /// Sentinel for "the API key summaries have never been refreshed". + private const long NeverRefreshedTicks = long.MinValue; + private readonly ISessionRegistry _sessionRegistry; private readonly GatewayMetrics _metrics; private readonly IGatewayConfigurationProvider _configurationProvider; @@ -30,6 +41,13 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService private readonly ILogger _logger; private readonly SemaphoreSlim _apiKeySummaryRefreshGate = new(1, 1); private IReadOnlyList _apiKeySummaries = Array.Empty(); + private long _apiKeySummariesRefreshedAtTicks = NeverRefreshedTicks; + // The effective configuration is built from IOptions and is startup-static: + // the gateway binds options once at boot and never reloads them, so this projection cannot + // change for the process lifetime. Build it once instead of re-projecting the whole option + // tree on every snapshot tick. A racing first build is harmless — the projection is pure, + // so either winner stores equivalent content. + private EffectiveGatewayConfiguration? _effectiveConfiguration; // Memoizes ONLY the O(N) template/category breakdown against the cache sequence. The shared // library bumps Sequence only on a heavy refresh that replaces the object set, so an unchanged // sequence means the breakdown is unchanged and can be reused — keeping the ~1s snapshot tick @@ -100,10 +118,23 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService Metrics: CreateMetricSummaries(metricsSnapshot), Faults: CreateFaultSummaries(sessions, generatedAt), ApiKeys: Volatile.Read(ref _apiKeySummaries), - Configuration: _configurationProvider.GetEffectiveConfiguration(), + Configuration: ResolveEffectiveConfiguration(), Galaxy: ResolveGalaxySummary()); } + private EffectiveGatewayConfiguration ResolveEffectiveConfiguration() + { + EffectiveGatewayConfiguration? cached = Volatile.Read(ref _effectiveConfiguration); + if (cached is not null) + { + return cached; + } + + EffectiveGatewayConfiguration configuration = _configurationProvider.GetEffectiveConfiguration(); + Volatile.Write(ref _effectiveConfiguration, configuration); + return configuration; + } + private DashboardGalaxySummary ResolveGalaxySummary() { GalaxyHierarchyCacheEntry entry = _galaxyHierarchyCache.Current; @@ -255,6 +286,17 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService private async Task RefreshApiKeySummariesAsync(CancellationToken cancellationToken) { + DateTimeOffset now = _timeProvider.GetUtcNow(); + long lastRefreshedAtTicks = Interlocked.Read(ref _apiKeySummariesRefreshedAtTicks); + if (lastRefreshedAtTicks != NeverRefreshedTicks + && now.UtcTicks - lastRefreshedAtTicks < ApiKeySummaryRefreshInterval.Ticks) + { + // Inside the refresh window: reuse the cached summaries rather than + // re-reading the API key table on this tick. Only a *successful* refresh + // moves the timestamp, so a failed read is retried on the next tick. + return; + } + if (!await _apiKeySummaryRefreshGate.WaitAsync(0, cancellationToken).ConfigureAwait(false)) { return; @@ -278,6 +320,7 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService .ToArray(); Volatile.Write(ref _apiKeySummaries, summaries); + Interlocked.Exchange(ref _apiKeySummariesRefreshedAtTicks, now.UtcTicks); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHub.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHub.cs index c93d01a..3cc8d9f 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHub.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHub.cs @@ -9,8 +9,15 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; /// immediately via ; subsequent refreshes are /// broadcast by . /// +/// +/// Connections are counted into +/// so can stop building and broadcasting +/// snapshots while nobody is watching. +/// [Authorize(Policy = DashboardAuthenticationDefaults.HubClientsPolicy)] -public sealed class DashboardSnapshotHub(IDashboardSnapshotService snapshotService) : Hub +public sealed class DashboardSnapshotHub( + IDashboardSnapshotService snapshotService, + DashboardSnapshotHubConnectionCounter connectionCounter) : Hub { /// Method name used to push snapshot updates to clients. public const string SnapshotMessage = "SnapshotUpdated"; @@ -18,7 +25,57 @@ public sealed class DashboardSnapshotHub(IDashboardSnapshotService snapshotServi /// public override async Task OnConnectedAsync() { + // Count the viewer before seeding it, so the publisher resumes its tick + // no later than the first snapshot this connection renders. + connectionCounter.Increment(); await Clients.Caller.SendAsync(SnapshotMessage, snapshotService.GetSnapshot()).ConfigureAwait(false); await base.OnConnectedAsync().ConfigureAwait(false); } + + /// + public override async Task OnDisconnectedAsync(Exception? exception) + { + connectionCounter.Decrement(); + await base.OnDisconnectedAsync(exception).ConfigureAwait(false); + } +} + +/// +/// Process-wide count of live connections. +/// Registered as a singleton and read by +/// to idle-gate the snapshot tick: with no dashboard connected there is nothing +/// to broadcast to, so no snapshot is built. +/// +public sealed class DashboardSnapshotHubConnectionCounter +{ + private int _count; + + /// Gets the number of live snapshot hub connections. + public int Count => Volatile.Read(ref _count); + + /// Records a new snapshot hub connection. + /// The connection count after the increment. + public int Increment() + { + return Interlocked.Increment(ref _count); + } + + /// + /// Records a snapshot hub disconnection. The count is clamped at zero: SignalR + /// can invoke OnDisconnectedAsync for a connection whose + /// OnConnectedAsync faulted, and a negative count would idle-gate the + /// publisher while viewers are still attached. + /// + /// The connection count after the decrement. + public int Decrement() + { + int updated = Interlocked.Decrement(ref _count); + if (updated >= 0) + { + return updated; + } + + Interlocked.CompareExchange(ref _count, 0, updated); + return 0; + } } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotPublisher.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotPublisher.cs index 5c740fc..85701b2 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotPublisher.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotPublisher.cs @@ -9,6 +9,7 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; /// gateway process; clients listen via the hub. /// /// +/// /// wraps the snapshot subscription in /// a reconnect loop with a configurable retry delay (5s by default, /// mirroring ). A transient failure inside @@ -16,44 +17,67 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; /// one-time logger-init failure or a transient SQL error from the Galaxy /// summary projection — would otherwise end the BackgroundService with no /// reconnect, taking the dashboard offline until process restart. +/// +/// +/// The loop is idle-gated on . +/// Each snapshot costs a session-registry snapshot and sort, a metrics snapshot +/// that copies dictionaries under the global metrics lock, and (periodically) a +/// SQLite read of the API key table — work with no consumer when no dashboard is +/// connected. While the count is zero the publisher does not advance the snapshot +/// enumerator at all, so the producing iterator stays suspended and builds nothing. +/// /// public sealed class DashboardSnapshotPublisher : BackgroundService { private static readonly TimeSpan DefaultReconnectDelay = TimeSpan.FromSeconds(5); + private static readonly TimeSpan DefaultIdlePollInterval = TimeSpan.FromSeconds(1); private readonly IDashboardSnapshotService _snapshotService; private readonly IHubContext _hubContext; + private readonly DashboardSnapshotHubConnectionCounter _connectionCounter; private readonly ILogger _logger; private readonly TimeSpan _reconnectDelay; + private readonly TimeSpan _idlePollInterval; /// Initializes a new instance of the DashboardSnapshotPublisher class. /// The snapshot service to subscribe to. /// The SignalR hub context for broadcasting. + /// Live snapshot hub connection count used to idle-gate the tick. /// The logger instance. public DashboardSnapshotPublisher( IDashboardSnapshotService snapshotService, IHubContext hubContext, + DashboardSnapshotHubConnectionCounter connectionCounter, ILogger logger) - : this(snapshotService, hubContext, logger, DefaultReconnectDelay) + : this(snapshotService, hubContext, connectionCounter, logger, DefaultReconnectDelay, DefaultIdlePollInterval) { } - /// Initializes a new instance of the DashboardSnapshotPublisher class with custom reconnect delay. - /// Internal hook for testing: tests inject a very short reconnect delay so assertions don't wait full 5s. + /// Initializes a new instance of the DashboardSnapshotPublisher class with custom cadences. + /// + /// Internal hook for testing: tests inject a very short reconnect delay so assertions + /// don't wait the full 5s, and a short idle poll so the resume-from-idle path is fast. + /// /// The snapshot service to subscribe to. /// The SignalR hub context for broadcasting. + /// Live snapshot hub connection count used to idle-gate the tick. /// The logger instance. /// The delay before reconnecting after a subscription failure. + /// How often the idle publisher re-checks for a connected viewer. internal DashboardSnapshotPublisher( IDashboardSnapshotService snapshotService, IHubContext hubContext, + DashboardSnapshotHubConnectionCounter connectionCounter, ILogger logger, - TimeSpan reconnectDelay) + TimeSpan reconnectDelay, + TimeSpan idlePollInterval) { _snapshotService = snapshotService; _hubContext = hubContext; + _connectionCounter = connectionCounter; _logger = logger; _reconnectDelay = reconnectDelay; + _idlePollInterval = idlePollInterval; } /// @@ -66,15 +90,31 @@ public sealed class DashboardSnapshotPublisher : BackgroundService { try { - await foreach (DashboardSnapshot snapshot in _snapshotService + // Enumerated by hand rather than with await foreach: the snapshot is + // built inside the producer's MoveNextAsync, so not calling MoveNextAsync + // is what makes the idle gate skip the build and not just the broadcast. + await using IAsyncEnumerator snapshots = _snapshotService .WatchSnapshotsAsync(stoppingToken) - .ConfigureAwait(false)) + .GetAsyncEnumerator(stoppingToken); + + while (!stoppingToken.IsCancellationRequested) { - if (stoppingToken.IsCancellationRequested) + if (_connectionCounter.Count == 0) + { + // Nobody is watching: leave the producer suspended and re-check + // shortly. The first viewer to connect resumes the tick, and is + // seeded directly by the hub's OnConnectedAsync meanwhile. + await Task.Delay(_idlePollInterval, stoppingToken).ConfigureAwait(false); + continue; + } + + if (!await snapshots.MoveNextAsync().ConfigureAwait(false)) { break; } + DashboardSnapshot snapshot = snapshots.Current; + try { await _hubContext.Clients diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotPublisherTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotPublisherTests.cs index 742aa3f..efa49f4 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotPublisherTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotPublisherTests.cs @@ -9,6 +9,7 @@ namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard; public sealed class DashboardSnapshotPublisherTests { private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5); + private static readonly TimeSpan IdlePollInterval = TimeSpan.FromMilliseconds(10); /// /// A transient failure inside @@ -28,8 +29,10 @@ public sealed class DashboardSnapshotPublisherTests DashboardSnapshotPublisher publisher = new( snapshotService, hubContext, + ConnectedCounter(), NullLogger.Instance, - reconnectDelay); + reconnectDelay, + IdlePollInterval); using CancellationTokenSource cts = new(); Task execute = publisher.StartAsync(cts.Token); @@ -73,8 +76,10 @@ public sealed class DashboardSnapshotPublisherTests DashboardSnapshotPublisher publisher = new( snapshotService, hubContext, + ConnectedCounter(), NullLogger.Instance, - reconnectDelay); + reconnectDelay, + IdlePollInterval); using CancellationTokenSource cts = new(); Task execute = publisher.StartAsync(cts.Token); @@ -88,6 +93,54 @@ public sealed class DashboardSnapshotPublisherTests Assert.True(snapshotService.SubscribeCount >= 2); } + /// + /// With no dashboard connected there is nobody to broadcast to, so the publisher must + /// not advance the snapshot enumerator at all — every pull costs a registry snapshot and + /// sort, a locked metrics dictionary copy, and periodically a SQLite key-table read. + /// The first viewer to connect resumes the tick. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task ExecuteAsync_WhenNoHubConnections_DoesNotPullSnapshots() + { + CountingSnapshotService snapshotService = new(); + RecordingHubContext hubContext = new(); + DashboardSnapshotHubConnectionCounter connectionCounter = new(); + DashboardSnapshotPublisher publisher = new( + snapshotService, + hubContext, + connectionCounter, + NullLogger.Instance, + TimeSpan.FromMilliseconds(50), + IdlePollInterval); + + using CancellationTokenSource cts = new(); + await publisher.StartAsync(cts.Token).WaitAsync(TestTimeout); + + // Long enough for many idle polls at IdlePollInterval. + await Task.Delay(TimeSpan.FromMilliseconds(250)); + + Assert.Equal(0, snapshotService.PullCount); + Assert.Equal(0, hubContext.SendCount); + + connectionCounter.Increment(); + await WaitUntilAsync(() => hubContext.SendCount >= 1); + + await cts.CancelAsync(); + await publisher.StopAsync(CancellationToken.None); + + Assert.True(snapshotService.PullCount >= 1); + } + + /// Creates a connection counter that already has one live viewer. + /// A counter reporting a single connection. + private static DashboardSnapshotHubConnectionCounter ConnectedCounter() + { + DashboardSnapshotHubConnectionCounter counter = new(); + counter.Increment(); + return counter; + } + private static async Task WaitUntilAsync(Func predicate) { using CancellationTokenSource cancellationTokenSource = new(TestTimeout); @@ -174,6 +227,34 @@ public sealed class DashboardSnapshotPublisherTests } } + private sealed class CountingSnapshotService : IDashboardSnapshotService + { + private int _pullCount; + + /// Gets the number of snapshots the publisher pulled from this source. + public int PullCount => Volatile.Read(ref _pullCount); + + /// + public DashboardSnapshot GetSnapshot() + { + return null!; + } + + /// + public async IAsyncEnumerable WatchSnapshotsAsync( + [EnumeratorCancellation] CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + // Short cadence so a listening publisher pulls quickly; the counter only + // moves when the publisher actually advances the enumerator. + await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationToken).ConfigureAwait(false); + Interlocked.Increment(ref _pullCount); + yield return GetSnapshot(); + } + } + } + private sealed class RecordingHubContext : IHubContext { private readonly RecordingHubClients _clients = new(); diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotServiceTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotServiceTests.cs index 5f0e48c..09efbf6 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotServiceTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotServiceTests.cs @@ -1,5 +1,6 @@ using System.Globalization; using Microsoft.Extensions.Options; +using Microsoft.Extensions.Time.Testing; using ZB.MOM.WW.Auth.Abstractions.ApiKeys; using ZB.MOM.WW.GalaxyRepository; using ZB.MOM.WW.GalaxyRepository.Grpc; @@ -457,6 +458,7 @@ public sealed class DashboardSnapshotServiceTests CreatedUtc: DateTimeOffset.Parse("2026-04-28T12:00:00Z", CultureInfo.InvariantCulture), LastUsedUtc: null, RevokedUtc: null)); + FakeTimeProvider timeProvider = new(DateTimeOffset.Parse("2026-08-15T00:00:00Z", CultureInfo.InvariantCulture)); DashboardSnapshotService service = CreateService( new SessionRegistry(), metrics, @@ -464,11 +466,12 @@ public sealed class DashboardSnapshotServiceTests { Dashboard = new DashboardOptions { - SnapshotIntervalMilliseconds = 1, + SnapshotIntervalMilliseconds = 1000, }, }, - apiKeyAdminStore: apiKeyAdminStore); - using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(2)); + apiKeyAdminStore: apiKeyAdminStore, + timeProvider: timeProvider); + using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(30)); await using IAsyncEnumerator enumerator = service .WatchSnapshotsAsync(cancellation.Token) .GetAsyncEnumerator(cancellation.Token); @@ -477,14 +480,86 @@ public sealed class DashboardSnapshotServiceTests DashboardSnapshot first = enumerator.Current; apiKeyAdminStore.FailNext = true; - Assert.True(await enumerator.MoveNextAsync()); - DashboardSnapshot second = enumerator.Current; + // Advance past the key-summary refresh interval so the second tick really + // does attempt a refresh — that attempt is the one that fails. + DashboardSnapshot second = await NextSnapshotAsync(enumerator, timeProvider, TimeSpan.FromSeconds(20)); Assert.Equal("operator01", Assert.Single(first.ApiKeys).KeyId); Assert.Equal("operator01", Assert.Single(second.ApiKeys).KeyId); Assert.Equal(2, apiKeyAdminStore.ListCount); } + /// + /// The API key list is a SQLite read; at the default 1s snapshot cadence it would run + /// ~86k times a day against a table that changes by hand. Ticks inside the refresh + /// interval must reuse the cached summaries and not touch the store. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task WatchSnapshotsAsync_WhenTicksFallInsideRefreshInterval_ListsApiKeysOnce() + { + using GatewayMetrics metrics = new(); + CountingApiKeyAdminStore apiKeyAdminStore = new( + new ApiKeyListItem( + KeyId: "operator01", + KeyPrefix: "mxgw", + DisplayName: "Operator", + Scopes: new HashSet([GatewayScopes.MetadataRead], StringComparer.Ordinal), + ConstraintsJson: null, + CreatedUtc: DateTimeOffset.Parse("2026-04-28T12:00:00Z", CultureInfo.InvariantCulture), + LastUsedUtc: null, + RevokedUtc: null)); + FakeTimeProvider timeProvider = new(DateTimeOffset.Parse("2026-08-15T00:00:00Z", CultureInfo.InvariantCulture)); + DashboardSnapshotService service = CreateService( + new SessionRegistry(), + metrics, + new GatewayOptions + { + Dashboard = new DashboardOptions + { + SnapshotIntervalMilliseconds = 1000, + }, + }, + apiKeyAdminStore: apiKeyAdminStore, + timeProvider: timeProvider); + using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(30)); + await using IAsyncEnumerator enumerator = service + .WatchSnapshotsAsync(cancellation.Token) + .GetAsyncEnumerator(cancellation.Token); + + Assert.True(await enumerator.MoveNextAsync()); + Assert.Equal(1, apiKeyAdminStore.ListCount); + + // Two more 1s ticks, both well inside the 15s refresh interval. + DashboardSnapshot second = await NextSnapshotAsync(enumerator, timeProvider, TimeSpan.FromSeconds(1)); + await NextSnapshotAsync(enumerator, timeProvider, TimeSpan.FromSeconds(1)); + + Assert.Equal(1, apiKeyAdminStore.ListCount); + Assert.Equal("operator01", Assert.Single(second.ApiKeys).KeyId); + + // A tick past the interval refreshes again, so an added or revoked key still + // reaches the dashboard within the interval. + await NextSnapshotAsync(enumerator, timeProvider, TimeSpan.FromSeconds(20)); + + Assert.Equal(2, apiKeyAdminStore.ListCount); + } + + /// + /// The effective configuration is startup-static, so the snapshot must hand out the + /// same instance instead of rebuilding the whole option tree on every tick. + /// + [Fact] + public void GetSnapshot_ReusesTheSameEffectiveConfigurationInstance() + { + using GatewayMetrics metrics = new(); + DashboardSnapshotService service = CreateService(new SessionRegistry(), metrics); + + DashboardSnapshot first = service.GetSnapshot(); + DashboardSnapshot second = service.GetSnapshot(); + + Assert.Same(first.Configuration, second.Configuration); + } + /// Verifies that snapshot service disposes cleanly when subscriber cancels. /// A task that represents the asynchronous operation. [Fact] @@ -513,12 +588,34 @@ public sealed class DashboardSnapshotServiceTests Assert.False(hasNext); } + /// + /// Advances the fake clock past the next snapshot tick and returns the snapshot it + /// produces. MoveNextAsync is started before the advance because the iterator + /// creates its synchronously on that call — the timer must + /// exist before the clock moves or the tick is missed. + /// + /// The snapshot enumerator being driven. + /// The fake clock backing the snapshot timer. + /// How far to advance the clock. + /// The snapshot produced by the tick. + private static async Task NextSnapshotAsync( + IAsyncEnumerator enumerator, + FakeTimeProvider timeProvider, + TimeSpan advance) + { + ValueTask pending = enumerator.MoveNextAsync(); + timeProvider.Advance(advance); + Assert.True(await pending.AsTask().WaitAsync(TimeSpan.FromSeconds(10))); + return enumerator.Current; + } + private static DashboardSnapshotService CreateService( SessionRegistry registry, GatewayMetrics metrics, GatewayOptions? options = null, IGalaxyHierarchyCache? galaxyHierarchyCache = null, - IApiKeyAdminStore? apiKeyAdminStore = null) + IApiKeyAdminStore? apiKeyAdminStore = null, + TimeProvider? timeProvider = null) { GatewayOptions resolvedOptions = options ?? new GatewayOptions { @@ -535,7 +632,8 @@ public sealed class DashboardSnapshotServiceTests configurationProvider, galaxyHierarchyCache ?? new StubGalaxyHierarchyCache(GalaxyHierarchyCacheEntry.Empty), apiKeyAdminStore ?? new FakeApiKeyAdminStore(), - Options.Create(resolvedOptions)); + Options.Create(resolvedOptions), + timeProvider); } private sealed class StubGalaxyHierarchyCache(GalaxyHierarchyCacheEntry current) : IGalaxyHierarchyCache From 9735ac3b7c396a465de420ec1237acc2afa681cf Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:21:51 -0400 Subject: [PATCH 10/41] perf(metrics): pull-model worker queue gauge (fixes last-writer-wins), Interlocked command counters --- docs/Metrics.md | 6 +- .../Metrics/GatewayMetrics.cs | 121 +++++++++--------- .../Workers/WorkerClient.cs | 18 ++- .../Metrics/GatewayMetricsTests.cs | 110 +++++++++++++++- 4 files changed, 178 insertions(+), 77 deletions(-) diff --git a/docs/Metrics.md b/docs/Metrics.md index 191c7a8..c13ea91 100644 --- a/docs/Metrics.md +++ b/docs/Metrics.md @@ -72,7 +72,7 @@ Observable gauges are pull-based; the `Meter` invokes the supplied callback when |------------|--------------|-------------| | `mxgateway.sessions.open` | `_openSessions` | Currently open sessions tracked by `SessionManager`. | | `mxgateway.workers.running` | `_workersRunning` | Worker clients in a running state. | -| `mxgateway.events.worker_queue.depth` | `_workerEventQueueDepth` | Undelivered worker events held by `WorkerClient` — staged *and* queued (GWC-24). Incremented when the read loop stages an event, decremented when the consumer reads it, so a backlog stuck in the staging channel is visible rather than invisible. | +| `mxgateway.events.worker_queue.depth` | `_workerEventQueueDepthSources` (summed on demand) | Undelivered worker events held by `WorkerClient` — staged *and* queued (GWC-24) — summed across every live client at collection time (GWC-30). Each client owns an interlocked counter incremented when the read loop stages an event and decremented when the consumer reads it, and registers it as a gauge source for its lifetime, so a backlog stuck in a staging channel is visible and concurrent sessions add up instead of overwriting one another. | | `mxgateway.events.grpc_stream_queue.depth` | `_eventStreamBacklogSources` (summed on demand) | Live backlog buffered across every active `EventStreamService` subscriber, summed from the subscribers' channel `Count` at collection time. | ## Snapshot Shape @@ -111,7 +111,7 @@ The scalar fields mirror the counters and gauges. The four dictionaries provide - `EventsBySession` keys by `sessionId`; entries are removed via `RemoveSessionEvents` when a session closes so the map does not grow without bound. - `RetryAttemptsByArea` keys by the resilience `area` tag, e.g. `worker_startup`. -`EventsReceived` is read with `Interlocked.Read(ref _eventsReceived)` because `EventReceived` increments it via `Interlocked.Increment` outside the lock to keep the event-ingestion path non-blocking. +`EventsReceived` is read with `Interlocked.Read(ref _eventsReceived)` because `EventReceived` increments it via `Interlocked.Increment` outside the lock to keep the event-ingestion path non-blocking. `CommandsStarted`, `CommandsSucceeded`, `CommandsFailed`, and `CommandFailuresByMethod` are read the same way: the command counters run two-to-three times per gRPC call, so they are recorded with `Interlocked` and a `ConcurrentDictionary` rather than under `_syncRoot` (GWC-30). The two queue depths are pulled from their registered sources before the lock is taken, since those delegates reach into subscriber channels and worker clients. ## Recording Sites @@ -146,7 +146,7 @@ _metrics.RemoveSessionEvents(session.SessionId); - `RecordWorkerStoppedOnce` calls `WorkerStopped(reason)` exactly once per worker, guarding against double-counting on simultaneous fault and exit signals. - `WorkerKilled(reason)` when the client forcibly terminates the worker. - `HeartbeatFailed(SessionId)` per missed heartbeat. -- `SetWorkerEventQueueDepth(queueDepth)` when the read loop stages an event and when the consumer reads one, so the gauge tracks staged + queued events. +- `RegisterWorkerEventQueueDepthSource(...)` once at construction, disposed in `DisposeAsync`. The client's own `_eventQueueDepth` is incremented when the read loop stages an event and decremented when the consumer reads one, so the gauge tracks staged + queued events without either hot-path step calling into `GatewayMetrics`. - `EventReceived(SessionId, workerEvent.Event.Family.ToString())` for each worker event. - `QueueOverflow("worker-events")` when the timed write into the bounded consumer channel exceeds `EventChannelFullModeTimeout`, and `QueueOverflow("worker-event-staging")` when the staging channel is full at its `2 × EventChannelCapacity` bound. The two labels distinguish a stalled consumer from one that merely drains too slowly; both fault the session with `ProtocolViolation`. diff --git a/src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetrics.cs b/src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetrics.cs index 5f520cc..36e8938 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetrics.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetrics.cs @@ -28,7 +28,9 @@ public sealed class GatewayMetrics : IDisposable private readonly Histogram _workerStartupLatencyHistogram; private readonly Histogram _commandLatencyHistogram; private readonly Histogram _eventStreamSendLatencyHistogram; - private readonly Dictionary _commandFailuresByMethod = new(StringComparer.OrdinalIgnoreCase); + // Concurrent (not Dictionary + _syncRoot) because CommandFailed runs on every failing gRPC call: + // the command counters are recorded outside the lock, so their breakdown map must be too. + private readonly ConcurrentDictionary _commandFailuresByMethod = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary _eventsByFamily = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary _eventsBySession = new(StringComparer.Ordinal); private readonly Dictionary _retryAttemptsByArea = new(StringComparer.OrdinalIgnoreCase); @@ -41,9 +43,16 @@ public sealed class GatewayMetrics : IDisposable private readonly ConcurrentDictionary> _eventStreamBacklogSources = new(); private long _nextEventStreamBacklogSourceId; + // GWC-30: the same pull model for the worker event queue depth. It replaces a pushed scalar that + // every WorkerClient wrote twice per event (staged, consumed) under _syncRoot — a process-wide + // lock on the hottest path, and last-writer-wins across sessions, so the gauge reported one + // arbitrary session's backlog instead of the gateway's. Each client registers a source returning + // its own undelivered depth; the gauge sums them at collection time only. + private readonly ConcurrentDictionary> _workerEventQueueDepthSources = new(); + private long _nextWorkerEventQueueDepthSourceId; + private int _openSessions; private int _workersRunning; - private int _workerEventQueueDepth; private int _alarmProviderMode; private long _sessionsOpened; private long _sessionsClosed; @@ -201,10 +210,10 @@ public sealed class GatewayMetrics : IDisposable /// Name of the command method. public void CommandStarted(string method) { - lock (_syncRoot) - { - _commandsStarted++; - } + // GWC-30: the three command counters run two-to-three times per gRPC call, so they use + // Interlocked rather than _syncRoot — the same idiom as EventReceived. Nothing here needs a + // consistent multi-field view; GetSnapshot reads each with Interlocked.Read. + Interlocked.Increment(ref _commandsStarted); _commandsStartedCounter.Add(1, new KeyValuePair("method", method)); } @@ -216,10 +225,7 @@ public sealed class GatewayMetrics : IDisposable /// Elapsed time to complete the command. public void CommandSucceeded(string method, TimeSpan duration) { - lock (_syncRoot) - { - _commandsSucceeded++; - } + Interlocked.Increment(ref _commandsSucceeded); KeyValuePair methodTag = new("method", method); _commandsSucceededCounter.Add(1, methodTag); @@ -234,11 +240,8 @@ public sealed class GatewayMetrics : IDisposable /// Elapsed time before command failed. public void CommandFailed(string method, string category, TimeSpan duration) { - lock (_syncRoot) - { - _commandsFailed++; - Increment(_commandFailuresByMethod, method); - } + Interlocked.Increment(ref _commandsFailed); + Increment(_commandFailuresByMethod, method); KeyValuePair methodTag = new("method", method); KeyValuePair categoryTag = new("category", category); @@ -275,29 +278,24 @@ public sealed class GatewayMetrics : IDisposable } /// - /// Sets the worker event queue depth; delegates to SetWorkerEventQueueDepth. + /// Registers a live depth source for the worker event queue-depth gauge and returns a handle + /// that removes it when disposed. Each WorkerClient registers once and reports its own + /// undelivered (staged + queued) event count, so the gauge is the gateway-wide sum instead of + /// the last value any one session happened to push (GWC-30). /// - /// Queue depth value. - public void SetEventQueueDepth(int depth) + /// + /// Returns this worker client's current undelivered event count. Invoked only at collection + /// time; must be cheap and non-blocking (a of an + /// interlocked counter). Negative readings — which a racing decrement can produce — are + /// clamped to zero when summed. + /// + /// A handle whose disposal unregisters the source. Safe to dispose more than once. + public IDisposable RegisterWorkerEventQueueDepthSource(Func depth) { - SetWorkerEventQueueDepth(depth); - } - - /// - /// Sets the worker event queue depth to the given value. - /// - /// Queue depth value. - public void SetWorkerEventQueueDepth(int depth) - { - if (depth < 0) - { - throw new ArgumentOutOfRangeException(nameof(depth), depth, "Queue depth cannot be negative."); - } - - lock (_syncRoot) - { - _workerEventQueueDepth = depth; - } + ArgumentNullException.ThrowIfNull(depth); + long id = Interlocked.Increment(ref _nextWorkerEventQueueDepthSourceId); + _workerEventQueueDepthSources[id] = depth; + return new GaugeSourceRegistration(_workerEventQueueDepthSources, id); } /// @@ -318,7 +316,7 @@ public sealed class GatewayMetrics : IDisposable ArgumentNullException.ThrowIfNull(backlog); long id = Interlocked.Increment(ref _nextEventStreamBacklogSourceId); _eventStreamBacklogSources[id] = backlog; - return new EventStreamBacklogRegistration(this, id); + return new GaugeSourceRegistration(_eventStreamBacklogSources, id); } /// @@ -460,21 +458,23 @@ public sealed class GatewayMetrics : IDisposable /// The current metrics snapshot. public GatewayMetricsSnapshot GetSnapshot() { - // Compute the live gRPC stream backlog outside _syncRoot: the sources are the subscriber - // channels' Count (their own locks) and must not run under this lock. GWC-15. + // Compute the live queue depths outside _syncRoot: the sources are the subscriber channels' + // Count (their own locks) and the worker clients' interlocked counters, neither of which may + // run under this lock. GWC-15, GWC-30. + int workerEventQueueDepth = GetWorkerEventQueueDepth(); int grpcEventStreamQueueDepth = GetGrpcEventStreamQueueDepth(); lock (_syncRoot) { return new GatewayMetricsSnapshot( OpenSessions: _openSessions, WorkersRunning: _workersRunning, - WorkerEventQueueDepth: _workerEventQueueDepth, + WorkerEventQueueDepth: workerEventQueueDepth, GrpcEventStreamQueueDepth: grpcEventStreamQueueDepth, SessionsOpened: _sessionsOpened, SessionsClosed: _sessionsClosed, - CommandsStarted: _commandsStarted, - CommandsSucceeded: _commandsSucceeded, - CommandsFailed: _commandsFailed, + CommandsStarted: Interlocked.Read(ref _commandsStarted), + CommandsSucceeded: Interlocked.Read(ref _commandsSucceeded), + CommandsFailed: Interlocked.Read(ref _commandsFailed), EventsReceived: Interlocked.Read(ref _eventsReceived), QueueOverflows: _queueOverflows, Faults: _faults, @@ -521,22 +521,19 @@ public sealed class GatewayMetrics : IDisposable } } - private int GetWorkerEventQueueDepth() - { - lock (_syncRoot) - { - return _workerEventQueueDepth; - } - } + // Sums the undelivered event backlog across every live worker client (GWC-30). + private int GetWorkerEventQueueDepth() => SumSources(_workerEventQueueDepthSources); - // Sums the live backlog across every registered event-stream subscriber. Runs at collection - // time (ObservableGauge scrape) or when GetSnapshot projects the value — never on the - // per-event path. Enumerating ConcurrentDictionary.Values never throws on concurrent + // Sums the live backlog across every registered event-stream subscriber. + private int GetGrpcEventStreamQueueDepth() => SumSources(_eventStreamBacklogSources); + + // Runs at collection time (ObservableGauge scrape) or when GetSnapshot projects the value — + // never on a per-event path. Enumerating ConcurrentDictionary.Values never throws on concurrent // register/unregister; a source removed mid-enumeration simply drops from this sample. - private int GetGrpcEventStreamQueueDepth() + private static int SumSources(ConcurrentDictionary> sources) { int total = 0; - foreach (Func source in _eventStreamBacklogSources.Values) + foreach (Func source in sources.Values) { int value = source(); if (value > 0) @@ -548,11 +545,6 @@ public sealed class GatewayMetrics : IDisposable return total; } - private void UnregisterEventStreamBacklogSource(long id) - { - _eventStreamBacklogSources.TryRemove(id, out _); - } - private int GetAlarmProviderMode() { lock (_syncRoot) @@ -572,9 +564,10 @@ public sealed class GatewayMetrics : IDisposable values.AddOrUpdate(key, 1, static (_, currentValue) => currentValue + 1); } - // Handle returned by RegisterEventStreamBacklogSource. Disposal (once) removes the source - // from the gauge's live sum. Idempotent so a double dispose from a stream teardown is safe. - private sealed class EventStreamBacklogRegistration(GatewayMetrics metrics, long id) : IDisposable + // Handle returned by the pull-model gauge registrations. Disposal (once) removes the source from + // that gauge's live sum. Idempotent so a double dispose from a stream or worker-client teardown + // is safe, and shared by both gauges so the two registrations cannot drift apart. + private sealed class GaugeSourceRegistration(ConcurrentDictionary> sources, long id) : IDisposable { private int _disposed; @@ -582,7 +575,7 @@ public sealed class GatewayMetrics : IDisposable { if (Interlocked.Exchange(ref _disposed, 1) == 0) { - metrics.UnregisterEventStreamBacklogSource(id); + sources.TryRemove(id, out _); } } } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs index 3ad808c..be74d2b 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs @@ -40,6 +40,12 @@ public sealed class WorkerClient : IWorkerClient private readonly ConcurrentDictionary _pendingCommands = new(StringComparer.Ordinal); private readonly SemaphoreSlim _pendingCommandSlots; private readonly CancellationTokenSource _stopCts = new(); + + // GWC-30: this client's contribution to the gateway-wide worker queue-depth gauge. Registered + // once here and read only when the gauge is scraped, so staging and consuming an event cost an + // Interlocked on _eventQueueDepth and nothing else — previously each of those two hot-path steps + // called into GatewayMetrics and took its process-wide lock. Null when metrics are disabled. + private readonly IDisposable? _eventQueueDepthRegistration; // Touched only by WriteLoopAsync — the single consumer of _outboundEnvelopes — so it needs no // interlocking. See WriteLoopAsync for why the stamp happens there rather than at construction. private ulong _nextSequence; @@ -111,6 +117,8 @@ public sealed class WorkerClient : IWorkerClient FullMode = BoundedChannelFullMode.Wait, AllowSynchronousContinuations = false, }); + _eventQueueDepthRegistration = _metrics?.RegisterWorkerEventQueueDepthSource( + () => Volatile.Read(ref _eventQueueDepth)); _lastHeartbeatAt = _timeProvider.GetUtcNow(); } @@ -299,8 +307,8 @@ public sealed class WorkerClient : IWorkerClient { await foreach (WorkerEvent workerEvent in _events.Reader.ReadAllAsync(cancellationToken).ConfigureAwait(false)) { - int queueDepth = Math.Max(0, Interlocked.Decrement(ref _eventQueueDepth)); - _metrics?.SetWorkerEventQueueDepth(queueDepth); + // No metrics call on the hot path: the gauge pulls _eventQueueDepth when scraped (GWC-30). + Interlocked.Decrement(ref _eventQueueDepth); yield return workerEvent; } } @@ -371,6 +379,9 @@ public sealed class WorkerClient : IWorkerClient } _disposed = true; + // Drop out of the worker queue-depth gauge before teardown: whatever this client still holds + // is about to be discarded, and the sum must not keep counting a client that is going away. + _eventQueueDepthRegistration?.Dispose(); KillOwnedProcess("Dispose"); _stopCts.Cancel(); _outboundEnvelopes.Writer.TryComplete(); @@ -598,8 +609,7 @@ public sealed class WorkerClient : IWorkerClient { // Counted here rather than at the _events write so the single gauge reports total // undelivered events (staged + queued). ReadEventsCoreAsync decrements on consumer read. - int queueDepth = Interlocked.Increment(ref _eventQueueDepth); - _metrics?.SetWorkerEventQueueDepth(queueDepth); + Interlocked.Increment(ref _eventQueueDepth); return; } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Metrics/GatewayMetricsTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Metrics/GatewayMetricsTests.cs index 374ff6f..5d67920 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Metrics/GatewayMetricsTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Metrics/GatewayMetricsTests.cs @@ -19,7 +19,10 @@ public sealed class GatewayMetricsTests metrics.CommandFailed("WriteSecured", "AuthorizationFailed", TimeSpan.FromMilliseconds(12)); metrics.EventReceived("session-1", "OnDataChange"); metrics.EventReceived("session-1", "OnDataChange"); - metrics.SetWorkerEventQueueDepth(7); + // GWC-30: the worker queue-depth gauge sums one live source per worker client, so the two + // registrations below stand in for two concurrent sessions holding 3 and 4 events. + using IDisposable workerDepthSourceA = metrics.RegisterWorkerEventQueueDepthSource(static () => 3); + using IDisposable workerDepthSourceB = metrics.RegisterWorkerEventQueueDepthSource(static () => 4); // GWC-15: the gRPC stream queue-depth gauge sums live backlog sources at collection time // rather than tracking a pushed running total. Register a source reporting 3. using IDisposable backlogSource = metrics.RegisterEventStreamBacklogSource(static () => 3); @@ -54,16 +57,111 @@ public sealed class GatewayMetricsTests Assert.Equal(2, snapshot.EventsBySession["session-1"]); } - /// Verifies that negative queue depth is rejected. + /// + /// GWC-30: the worker queue-depth gauge sums every registered source rather than holding a + /// single pushed scalar, so concurrent sessions add up instead of overwriting one another, + /// and a disposed registration (a worker client going away) drops out of the sum. Disposal + /// is idempotent because a client's dispose path can run twice. + /// [Fact] - public void SetEventQueueDepth_RejectsNegativeDepth() + public void WorkerEventQueueDepthSources_SumAcrossRegistrationsAndDropOnDispose() { using GatewayMetrics metrics = new(); - ArgumentOutOfRangeException exception = Assert.Throws( - () => metrics.SetWorkerEventQueueDepth(-1)); + IDisposable firstSource = metrics.RegisterWorkerEventQueueDepthSource(static () => 3); + using IDisposable secondSource = metrics.RegisterWorkerEventQueueDepthSource(static () => 4); - Assert.Equal("depth", exception.ParamName); + Assert.Equal(7, metrics.GetSnapshot().WorkerEventQueueDepth); + + firstSource.Dispose(); + Assert.Equal(4, metrics.GetSnapshot().WorkerEventQueueDepth); + + firstSource.Dispose(); + Assert.Equal(4, metrics.GetSnapshot().WorkerEventQueueDepth); + } + + /// + /// A depth source reads a lock-free counter that a racing decrement can momentarily push + /// below zero, so the sum clamps each reading instead of rejecting it — the pull model has + /// no caller to throw back at. + /// + [Fact] + public void WorkerEventQueueDepthSources_ClampNegativeReadingsToZero() + { + using GatewayMetrics metrics = new(); + + using IDisposable negativeSource = metrics.RegisterWorkerEventQueueDepthSource(static () => -5); + using IDisposable positiveSource = metrics.RegisterWorkerEventQueueDepthSource(static () => 4); + + Assert.Equal(4, metrics.GetSnapshot().WorkerEventQueueDepth); + } + + /// + /// Verifies the exported gauge keeps the name mxgateway.events.worker_queue.depth and + /// reports the summed sources, so the pull-model rework is invisible to exporters. + /// + [Fact] + public void WorkerEventQueueDepthGauge_ReportsSummedSources() + { + using GatewayMetrics metrics = new(); + using MeterListener listener = new(); + + int? capturedDepth = null; + + listener.InstrumentPublished = (instrument, meterListener) => + { + if (ReferenceEquals(instrument.Meter, metrics.Meter) + && instrument.Name == "mxgateway.events.worker_queue.depth") + { + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback( + (instrument, measurement, _, _) => + { + if (ReferenceEquals(instrument.Meter, metrics.Meter) + && instrument.Name == "mxgateway.events.worker_queue.depth") + { + capturedDepth = measurement; + } + }); + listener.Start(); + + using IDisposable firstSource = metrics.RegisterWorkerEventQueueDepthSource(static () => 5); + using IDisposable secondSource = metrics.RegisterWorkerEventQueueDepthSource(static () => 6); + listener.RecordObservableInstruments(); + + Assert.Equal(11, capturedDepth); + } + + /// + /// The command counters are incremented with rather than under the + /// process-wide metrics lock, so this asserts no increment is lost when every gRPC thread + /// records at once. + /// + [Fact] + public void CommandCounters_CountEveryConcurrentInvocation() + { + const int workers = 8; + const int perWorker = 500; + using GatewayMetrics metrics = new(); + + Parallel.For(0, workers, _ => + { + for (int index = 0; index < perWorker; index++) + { + metrics.CommandStarted("Register"); + metrics.CommandSucceeded("Register", TimeSpan.FromMilliseconds(1)); + metrics.CommandFailed("WriteSecured", "AuthorizationFailed", TimeSpan.FromMilliseconds(1)); + } + }); + + GatewayMetricsSnapshot snapshot = metrics.GetSnapshot(); + + Assert.Equal(workers * perWorker, snapshot.CommandsStarted); + Assert.Equal(workers * perWorker, snapshot.CommandsSucceeded); + Assert.Equal(workers * perWorker, snapshot.CommandsFailed); + Assert.Equal(workers * perWorker, snapshot.CommandFailuresByMethod["WriteSecured"]); } /// From 8e2066b4bd2cde88fb4dfebb830f328763c62397 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:22:33 -0400 Subject: [PATCH 11/41] docs(dashboard): restore the advised-set LRU paragraph dropped by the snapshot commit 77c5731 committed this file from a working copy that predated 75e3dc2's advised-set section, silently deleting it. Puts the paragraph back verbatim; no other content changes. --- docs/GatewayDashboardDesign.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/GatewayDashboardDesign.md b/docs/GatewayDashboardDesign.md index 0beb73a..d2327fa 100644 --- a/docs/GatewayDashboardDesign.md +++ b/docs/GatewayDashboardDesign.md @@ -394,6 +394,18 @@ its lease expires. One session means one worker process backs every dashboard circuit; all access is serialised so the worker sees one in-flight command at a time. Tag reads go through `GatewaySession.SubscribeBulkAsync` / `ReadBulkAsync`. +The advise set that backs those reads is capped at 256 tags (one browse page plus +headroom) and evicted least-recently-read-first. Without the cap every tag any +viewer ever inspected stayed advised on the single dashboard worker until the +session faulted, so browsing a large galaxy accreted unbounded live MXAccess +subscriptions — and the event churn they feed — on one x86 process. Reading a tag +already in the set marks it most-recently-read; subscribing past the cap unadvises +the oldest entries with `GatewaySession.UnsubscribeBulkAsync` in one batch before +the new ones are advised. Tags read in the same call are never evicted to make +room for each other. A failed unadvise does not fail the read: the tags are +dropped from tracking anyway (they re-subscribe if read again), because the +session-invalidation path already handles gateway/worker drift. + The Alarms page does **not** use the dashboard session: alarm data comes from the gateway's always-on central monitor. `QueryAlarmsAsync` reads `IGatewayAlarmService.CurrentAlarms` — the monitor's in-process cache — so the From 3ff073d1ea71f5210bcf3af9aa050f8488c23932 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:22:59 -0400 Subject: [PATCH 12/41] perf(grpc): transfer reply ownership instead of deep-cloning every worker reply --- .../Grpc/MxAccessGrpcMapper.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGrpcMapper.cs b/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGrpcMapper.cs index fa5e254..8ff862b 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGrpcMapper.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGrpcMapper.cs @@ -71,7 +71,17 @@ public sealed class MxAccessGrpcMapper }; } - return reply.Reply.Clone(); + // GWC-07 / IPC-05: ownership transfer, not a deep clone — the same rule MapEvent follows, + // applied to the other (and larger, on bulk reads) hot-path message. The enclosing + // WorkerCommandReply is parsed fresh from a single pipe frame in WorkerClient's read loop + // and is single-consumer by construction: CompleteCommand's TryRemove hands it to exactly + // one PendingCommand awaiter, that awaiter is the gRPC Invoke handler, and the handler's + // one call is this mapping. Nothing else aliases or reads reply.Reply afterwards — the + // enclosing WorkerCommandReply is discarded here. We therefore move the inner + // MxCommandReply into the gRPC response instead of copying it; the handler owning it + // outright is also what makes BulkConstraintPlan.MergeDeniedInto's in-place splice safe. + // If a second consumer of the same WorkerCommandReply is ever added, restore a .Clone(). + return reply.Reply; } /// From 88d38bb9008bb7c376869a3322af8cb9fbb548dd Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:23:25 -0400 Subject: [PATCH 13/41] perf(auth): allocation-free token parsing; single partition-key build per RPC --- .../Authentication/CachingApiKeyVerifier.cs | 29 ++++++++-- .../Authorization/ApiKeyFailureLimiter.cs | 55 +++++++++++++++++- .../GatewayGrpcAuthorizationInterceptor.cs | 55 +++++++++++++----- .../CachingApiKeyVerifierTests.cs | 49 ++++++++++++++++ ...atewayGrpcAuthorizationInterceptorTests.cs | 58 +++++++++++++++++++ 5 files changed, 221 insertions(+), 25 deletions(-) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/CachingApiKeyVerifier.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/CachingApiKeyVerifier.cs index a2a496f..4dcbb6f 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/CachingApiKeyVerifier.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/CachingApiKeyVerifier.cs @@ -213,7 +213,10 @@ public sealed class CachingApiKeyVerifier : IApiKeyVerifier, IApiKeyCacheInvalid // DashboardApiKeyManagementService.ValidateKeyId each restrict a key id to // char.IsAsciiLetterOrDigit || '.' || '-'. Key ids are never library-generated, so no path can // mint one containing '_'. - private static string? TryParseKeyId(string? authorizationHeader) + // + // Internal rather than private so the parse rules can be pinned directly by test: the guard's + // correctness depends on this returning the full key id. + internal static string? TryParseKeyId(string? authorizationHeader) { if (string.IsNullOrEmpty(authorizationHeader)) { @@ -226,15 +229,29 @@ public sealed class CachingApiKeyVerifier : IApiKeyVerifier, IApiKeyCacheInvalid ? header[bearer.Length..].Trim() : header; - string[] parts = token.ToString().Split('_'); - if (parts.Length < 3 - || !string.Equals(parts[0], TokenPrefix, StringComparison.Ordinal) - || parts[1].Length == 0) + // Scanned rather than split, for the same reason as the interceptor's copy: Split would + // allocate a token copy, an array and a string per segment on every cache miss to produce + // one key id. Two IndexOf scans allocate only that key id. + // + // The '_' checked immediately after the prefix is what makes "mxgw" the whole first segment + // (so "mxgwabc_..." is still rejected), and the second separator must exist because the + // split form required three segments — a token with no secret delimiter is not a key token. + if (!token.StartsWith(TokenPrefix, StringComparison.Ordinal) + || token.Length <= TokenPrefix.Length + || token[TokenPrefix.Length] != '_') { return null; } - return parts[1]; + ReadOnlySpan afterPrefix = token[(TokenPrefix.Length + 1)..]; + int separator = afterPrefix.IndexOf('_'); + if (separator <= 0) + { + // -1 is a token with no second separator; 0 is an empty key id. + return null; + } + + return new string(afterPrefix[..separator]); } private void IndexCacheKey(string keyId, string cacheKey) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs index ad28237..42118c8 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs @@ -127,16 +127,31 @@ public sealed class ApiKeyFailureLimiter /// Decides whether an authentication attempt may reach the verifier. /// The throttle partition derived from the request. /// The admission decision for this attempt. - public ApiKeyThrottleDecision Check(ApiKeyThrottlePartition partition) + public ApiKeyThrottleDecision Check(ApiKeyThrottlePartition partition) => Check(partition, out _); + + /// + /// Decides whether an authentication attempt may reach the verifier, handing back the storage + /// key it resolved so a caller that goes on to + /// the same request does not resolve — and rebuild the composite key string — a second time. + /// + /// The throttle partition derived from the request. + /// + /// The resolved storage key, or when the limiter is + /// disabled and never resolved one. + /// + /// The admission decision for this attempt. + internal ApiKeyThrottleDecision Check(ApiKeyThrottlePartition partition, out PartitionResolution resolution) { string peer = RequirePeer(partition); if (_limit <= 0) { + resolution = PartitionResolution.Unresolved; return ApiKeyThrottleDecision.Allowed; } long now = _clock.GetUtcNow().UtcTicks; (string partitionKey, string? effectiveKeyId) = ResolvePartitionKey(peer, partition.KeyId, mint: false); + resolution = new PartitionResolution(partitionKey, effectiveKeyId); WindowState? peerState = _partitions.TryGetValue(partitionKey, out WindowState? tracked) ? tracked : null; WindowState? aggregateState = null; @@ -221,10 +236,25 @@ public sealed class ApiKeyFailureLimiter /// Clears both limiter layers for the partition after a successful verification. /// The throttle partition derived from the request. - public void Reset(ApiKeyThrottlePartition partition) + public void Reset(ApiKeyThrottlePartition partition) => Reset(partition, PartitionResolution.Unresolved); + + /// + /// Clears both limiter layers for the partition after a successful verification, reusing the + /// storage key already + /// resolved for this request. + /// + /// The throttle partition derived from the request. + /// + /// The resolution handed back by Check; + /// resolves here instead. Reusing the check-time resolution is deliberate: it is the partition + /// this request was admitted against, so the reset clears exactly what the check consulted. + /// + internal void Reset(ApiKeyThrottlePartition partition, PartitionResolution resolution) { string peer = RequirePeer(partition); - (string partitionKey, string? effectiveKeyId) = ResolvePartitionKey(peer, partition.KeyId, mint: false); + (string partitionKey, string? effectiveKeyId) = resolution.IsResolved + ? (resolution.PartitionKey!, resolution.EffectiveKeyId) + : ResolvePartitionKey(peer, partition.KeyId, mint: false); // Clear only a partition this caller actually owns. When its key id was squeezed into the // address's shared fallback bucket by the per-peer cap, that bucket also holds failures @@ -542,6 +572,25 @@ public sealed class ApiKeyFailureLimiter public long ProbeVersion; } + /// + /// A partition's resolved storage key, carried from the check to the reset of the same request. + /// The composite key is a fresh string per build, so resolving once per RPC rather than once per + /// call keeps the successful auth path (check, then reset) to a single allocation. + /// + /// The storage key, or when unresolved. + /// + /// The key id that actually earned a partition, or when the token carried + /// none or the per-peer cap collapsed it onto the transport-peer fallback. + /// + internal readonly record struct PartitionResolution(string? PartitionKey, string? EffectiveKeyId) + { + /// Gets the sentinel for "not resolved yet"; the receiving call resolves it itself. + internal static PartitionResolution Unresolved => default; + + /// Gets a value indicating whether this carries a resolved storage key. + internal bool IsResolved => PartitionKey is not null; + } + /// A probe slot reservation: what to restore, and the stamp proving it is still ours. /// The slot value replaced when the claim was made. /// The stamped by this claim. diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcAuthorizationInterceptor.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcAuthorizationInterceptor.cs index c7c4ddf..feaeaaa 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcAuthorizationInterceptor.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcAuthorizationInterceptor.cs @@ -77,8 +77,13 @@ public sealed class GatewayGrpcAuthorizationInterceptor( // aggregate for the key id. An over-limit state still admits one probe per interval, so the // holder of the correct secret always reaches the verifier and resets the state. // ResourceExhausted signals throttling without revealing whether any secret was valid. + // + // The check hands back the storage key it resolved so the reset below reuses it instead of + // rebuilding the composite (peer, key id) string a second time on every successful RPC. ApiKeyThrottlePartition throttlePartition = ResolveThrottlePartition(authorizationHeader, context); - ApiKeyThrottleDecision decision = failureLimiter.Check(throttlePartition); + ApiKeyThrottleDecision decision = failureLimiter.Check( + throttlePartition, + out ApiKeyFailureLimiter.PartitionResolution throttleResolution); if (decision is ApiKeyThrottleDecision.ThrottledByPeer or ApiKeyThrottleDecision.ThrottledByAggregate) { metrics.RecordAuthThrottled( @@ -107,7 +112,7 @@ public sealed class GatewayGrpcAuthorizationInterceptor( // fat-fingered a few attempts is not penalised once it recovers — and, because the check // above admits a probe rather than blocking absolutely, this reset stays reachable while the // key is under an active spray. - failureLimiter.Reset(throttlePartition); + failureLimiter.Reset(throttlePartition, throttleResolution); ApiKeyIdentity identity = GatewayApiKeyIdentityMapper.ToGatewayIdentity(verification.Identity); @@ -137,7 +142,11 @@ public sealed class GatewayGrpcAuthorizationInterceptor( // before a key-id partition is minted so a spray of invented tokens cannot mint one tracked // partition each and flush the limiter's bounded map (SEC-32). Anything that fails the check // falls back to the sender's transport-peer partition. - private static string? TryResolveKeyId(string? authorizationHeader) + // + // Internal rather than private so the parse rules can be pinned directly by test; the shape + // check is a security boundary (SEC-32) and is worth asserting without routing every case + // through a full RPC. + internal static string? TryResolveKeyId(string? authorizationHeader) { if (string.IsNullOrWhiteSpace(authorizationHeader)) { @@ -150,22 +159,36 @@ public sealed class GatewayGrpcAuthorizationInterceptor( ? header[bearer.Length..].Trim() : header; - string[] parts = token.ToString().Split('_'); - if (parts.Length < 3) + // Scanned rather than split: this runs on every authenticated RPC, and Split would copy the + // token out of the header and allocate an array plus a string per segment to reach a key id + // that is then usually a dictionary-lookup miss. Two IndexOf scans reach the same answer and + // allocate only the key id itself. + const string prefix = AuthStoreServiceCollectionExtensions.TokenPrefix; + if (!token.StartsWith(prefix, StringComparison.Ordinal) + || token.Length <= prefix.Length + || token[prefix.Length] != '_') + { + // Guards the whole first segment, not just its start: the '_' immediately after the + // prefix is what makes "mxgw" the entire segment, so "mxgwabc_..." is still rejected. + return null; + } + + ReadOnlySpan afterPrefix = token[(prefix.Length + 1)..]; + int separator = afterPrefix.IndexOf('_'); + if (separator <= 0 || separator > MaxKeyIdLength) + { + // -1 is a token with no second separator (too few segments); 0 is an empty key id. + return null; + } + + // The third segment must be non-empty, which the split form expressed as parts[2].Length: it + // ends at the NEXT separator, so a secret beginning with '_' fails the same way it always did. + ReadOnlySpan afterKeyId = afterPrefix[(separator + 1)..]; + if (afterKeyId.IsEmpty || afterKeyId[0] == '_') { return null; } - if (!string.Equals(parts[0], AuthStoreServiceCollectionExtensions.TokenPrefix, StringComparison.Ordinal)) - { - return null; - } - - if (parts[1].Length == 0 || parts[1].Length > MaxKeyIdLength || parts[2].Length == 0) - { - return null; - } - - return parts[1]; + return new string(afterPrefix[..separator]); } } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs index ce9c2eb..2577a0f 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs @@ -196,6 +196,55 @@ public sealed class CachingApiKeyVerifierTests Assert.Equal(2, inner.MarkUsedCount); } + /// + /// Pins the header parse that arms the revoke-vs-in-flight generation guard. It runs on every + /// cache miss and is written to allocate only the returned key id; these cases hold it to the + /// rules the original Split('_') form applied. Note this parse is deliberately laxer than + /// the interceptor's partition parse — it applies no key-id length cap and does not require a + /// non-empty third segment — because a wrong id here would disarm the guard, whereas + /// an over-long one merely fails to match any generation. + /// + /// The presented header value. + /// The key id the parse must yield, or . + [Theory] + [InlineData(null, null)] + [InlineData("", null)] + [InlineData(" ", null)] + [InlineData("Bearer", null)] + [InlineData("Bearer ", null)] + [InlineData("Bearer mxgw_operator01_super-secret", "operator01")] + [InlineData("bearer mxgw_operator01_super-secret", "operator01")] + [InlineData(" Bearer mxgw_operator01_super-secret ", "operator01")] + [InlineData("mxgw_operator01_super-secret", "operator01")] + [InlineData("Bearer mxgw_abc_sec_ret", "abc")] + [InlineData("Bearer mxgw_a_b_c", "a")] + + // Laxer than the interceptor: an empty or absent third segment still yields the key id. + [InlineData("Bearer mxgw_abc_", "abc")] + [InlineData("Bearer mxgw_abc__secret", "abc")] + [InlineData("Bearer mxgw_abc", null)] + [InlineData("Bearer mxgwabcsecret", null)] + [InlineData("Bearer mxgw__secret", null)] + [InlineData("Bearer _mxgw_abc_secret", null)] + [InlineData("Bearer MXGW_abc_secret", null)] + [InlineData("Bearer xmxgw_abc_secret", null)] + [InlineData("Bearer mxgw", null)] + [InlineData("Bearer mxgw_", null)] + [InlineData("Bearer ___", null)] + public void TryParseKeyId_MatchesTokenShapeRules(string? authorizationHeader, string? expected) + { + Assert.Equal(expected, CachingApiKeyVerifier.TryParseKeyId(authorizationHeader)); + } + + /// The guard parse applies no key-id length cap: an over-long id is still returned whole. + [Fact] + public void TryParseKeyId_LongKeyId_ReturnedWhole() + { + string keyId = new('a', 65); + + Assert.Equal(keyId, CachingApiKeyVerifier.TryParseKeyId($"Bearer mxgw_{keyId}_secret")); + } + private static MemoryCache NewCache() => new(new MemoryCacheOptions()); private static ApiKeyVerification Success(string keyId) => new( diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authorization/GatewayGrpcAuthorizationInterceptorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authorization/GatewayGrpcAuthorizationInterceptorTests.cs index b4903ec..1808e54 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authorization/GatewayGrpcAuthorizationInterceptorTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authorization/GatewayGrpcAuthorizationInterceptorTests.cs @@ -659,6 +659,64 @@ public sealed class GatewayGrpcAuthorizationInterceptorTests } } + /// + /// Pins the token shape the limiter partition is minted from. The parser is on every + /// authenticated request, so it is written to allocate only the returned key id; these cases + /// hold it to the rules the original Split('_') form applied — literal mxgw first + /// segment, non-empty second and third segments, and a bounded key id — including the ones a + /// hand-rolled scanner is most likely to drift on (a key id read from the second segment even + /// when the secret itself contains separators). + /// + /// The presented header value. + /// The key id the parse must yield, or . + [Theory] + [InlineData(null, null)] + [InlineData("", null)] + [InlineData(" ", null)] + [InlineData("Bearer", null)] + [InlineData("Bearer ", null)] + [InlineData("Bearer mxgw_abc_secret", "abc")] + [InlineData("bearer mxgw_abc_secret", "abc")] + [InlineData(" Bearer mxgw_abc_secret ", "abc")] + [InlineData("mxgw_abc_secret", "abc")] + + // The secret may carry separators of its own; the key id is still the second segment. + [InlineData("Bearer mxgw_abc_sec_ret", "abc")] + [InlineData("Bearer mxgw_a_b_c", "a")] + [InlineData("Bearer mxgw_abc_secret_", "abc")] + + // Shape failures: no separators, too few segments, empty segments, wrong prefix. + [InlineData("Bearer mxgwabcsecret", null)] + [InlineData("Bearer mxgw_abc", null)] + [InlineData("Bearer mxgw_abc_", null)] + [InlineData("Bearer mxgw_abc__secret", null)] + [InlineData("Bearer mxgw__secret", null)] + [InlineData("Bearer _mxgw_abc_secret", null)] + [InlineData("Bearer _abc_secret", null)] + [InlineData("Bearer MXGW_abc_secret", null)] + [InlineData("Bearer xmxgw_abc_secret", null)] + [InlineData("Bearer mxgw", null)] + [InlineData("Bearer mxgw_", null)] + [InlineData("Bearer ___", null)] + public void TryResolveKeyId_MatchesTokenShapeRules(string? authorizationHeader, string? expected) + { + Assert.Equal(expected, GatewayGrpcAuthorizationInterceptor.TryResolveKeyId(authorizationHeader)); + } + + /// + /// The key-id length cap is what stops an invented id of arbitrary length from becoming a + /// limiter partition, so the boundary is pinned on both sides. + /// + [Fact] + public void TryResolveKeyId_HonoursKeyIdLengthCap() + { + string atCap = new('a', 64); + string overCap = new('a', 65); + + Assert.Equal(atCap, GatewayGrpcAuthorizationInterceptor.TryResolveKeyId($"Bearer mxgw_{atCap}_secret")); + Assert.Null(GatewayGrpcAuthorizationInterceptor.TryResolveKeyId($"Bearer mxgw_{overCap}_secret")); + } + private static MxAccessGatewayService CreateService( ISessionManager sessionManager, IGatewayRequestIdentityAccessor identityAccessor) From 7171892984a84ac55db8aacf06a811e92da2c24f Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:24:17 -0400 Subject: [PATCH 14/41] perf(grpc): O(1) unconstrained bulk fast path, direct filtered-command build, cache eviction, capacity hints --- .../Grpc/MxAccessGatewayService.cs | 121 ++++++++++++++---- .../GatewayApiKeyIdentityMapper.cs | 57 ++++++++- .../Authorization/ConstraintEnforcer.cs | 30 ++++- .../Authorization/IConstraintEnforcer.cs | 24 ++++ .../Sessions/SparseArrayExpander.cs | 10 ++ .../CachingApiKeyVerifierTests.cs | 52 +++++++- 6 files changed, 261 insertions(+), 33 deletions(-) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs b/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs index 691bc8a..c4380b2 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs @@ -461,6 +461,14 @@ public sealed class MxAccessGatewayService( string? correlationId, CancellationToken cancellationToken) { + // An identity with no read constraints allows every tag, so the per-item enforcer call below + // can only answer "allowed" — the whole loop (and the plan it would build) is dead work. + // Returning null is exactly what the denied.Count == 0 exit below returns. + if (!constraintEnforcer.HasReadConstraints(identity)) + { + return null; + } + Dictionary denied = []; List allowed = []; for (int index = 0; index < tagAddresses.Count; index++) @@ -491,16 +499,23 @@ public sealed class MxAccessGatewayService( return null; } - MxCommand filtered = command.Clone(); - if (filtered.Kind == MxCommandKind.AddItemBulk) + // Build the filtered command directly instead of cloning the original and clearing it: + // the clone deep-copied every denied address only to drop it. The payload's other fields + // (server_handle) are copied across explicitly. Nothing aliases the request here — these + // bulk payloads carry only strings — and the worker-bound graph is still the unaliased copy + // MapCommand makes. + MxCommand filtered = new() { Kind = command.Kind }; + if (command.Kind == MxCommandKind.AddItemBulk) { - filtered.AddItemBulk.TagAddresses.Clear(); - filtered.AddItemBulk.TagAddresses.Add(allowed); + AddItemBulkCommand payload = new() { ServerHandle = command.AddItemBulk.ServerHandle }; + payload.TagAddresses.Add(allowed); + filtered.AddItemBulk = payload; } else { - filtered.SubscribeBulk.TagAddresses.Clear(); - filtered.SubscribeBulk.TagAddresses.Add(allowed); + SubscribeBulkCommand payload = new() { ServerHandle = command.SubscribeBulk.ServerHandle }; + payload.TagAddresses.Add(allowed); + filtered.SubscribeBulk = payload; } return new SubscribeBulkConstraintPlan(filtered, tagAddresses.Count, denied, allowed.Count > 0); @@ -517,6 +532,11 @@ public sealed class MxAccessGatewayService( // Mirrors FilterTagBulkAsync but produces BulkReadResult denial entries // so the reply payload merges into BulkReadReply.Results, not // BulkSubscribeReply.Results. + if (!constraintEnforcer.HasReadConstraints(identity)) + { + return null; + } + Dictionary denied = []; List allowed = []; for (int index = 0; index < tagAddresses.Count; index++) @@ -548,9 +568,14 @@ public sealed class MxAccessGatewayService( return null; } - MxCommand filtered = command.Clone(); - filtered.ReadBulk.TagAddresses.Clear(); - filtered.ReadBulk.TagAddresses.Add(allowed); + MxCommand filtered = new() { Kind = command.Kind }; + ReadBulkCommand payload = new() + { + ServerHandle = command.ReadBulk.ServerHandle, + TimeoutMs = command.ReadBulk.TimeoutMs, + }; + payload.TagAddresses.Add(allowed); + filtered.ReadBulk = payload; return new ReadBulkConstraintPlan(filtered, tagAddresses.Count, denied, allowed.Count > 0); } @@ -572,6 +597,11 @@ public sealed class MxAccessGatewayService( // Parameterising on TEntry + getItemHandle keeps a single filter // routine for all four and avoids duplicating CheckWriteHandleAsync // calls. + if (!constraintEnforcer.HasWriteConstraints(identity)) + { + return null; + } + Dictionary denied = []; List allowed = []; for (int index = 0; index < entries.Count; index++) @@ -609,33 +639,70 @@ public sealed class MxAccessGatewayService( return null; } - MxCommand filtered = command.Clone(); - ReplaceWriteBulkEntries(filtered, allowed); - return new WriteBulkConstraintPlan(filtered, entries.Count, denied, allowed.Count > 0); + return new WriteBulkConstraintPlan( + BuildFilteredWriteBulkCommand(command, allowed), + entries.Count, + denied, + allowed.Count > 0); } - private static void ReplaceWriteBulkEntries(MxCommand command, IReadOnlyList allowed) + /// + /// Builds the allowed-only bulk-write command. The allowed entries are carried over by + /// reference rather than deep-cloned: the caller only reads this command (TrackCommandReply), + /// and the copy the worker mutates and owns is the one MapCommand clones — the same + /// no-aliasing boundary as before. Cloning the whole command here and clearing it copied + /// every denied entry's payload (including WriteSecured values) for nothing. + /// + /// The per-family bulk-write entry message type. + /// The original command, read for its kind and payload scalars. + /// The entries that survived constraint filtering, in original order. + /// A command of the same kind carrying only the allowed entries. + private static MxCommand BuildFilteredWriteBulkCommand(MxCommand command, IReadOnlyList allowed) where TEntry : class { + MxCommand filtered = new() { Kind = command.Kind }; switch (command.Kind) { case MxCommandKind.WriteBulk: - command.WriteBulk.Entries.Clear(); - command.WriteBulk.Entries.Add((IEnumerable)allowed); + { + WriteBulkCommand payload = new() { ServerHandle = command.WriteBulk.ServerHandle }; + payload.Entries.Add((IEnumerable)allowed); + filtered.WriteBulk = payload; break; + } + case MxCommandKind.Write2Bulk: - command.Write2Bulk.Entries.Clear(); - command.Write2Bulk.Entries.Add((IEnumerable)allowed); + { + Write2BulkCommand payload = new() { ServerHandle = command.Write2Bulk.ServerHandle }; + payload.Entries.Add((IEnumerable)allowed); + filtered.Write2Bulk = payload; break; + } + case MxCommandKind.WriteSecuredBulk: - command.WriteSecuredBulk.Entries.Clear(); - command.WriteSecuredBulk.Entries.Add((IEnumerable)allowed); + { + WriteSecuredBulkCommand payload = new() { ServerHandle = command.WriteSecuredBulk.ServerHandle }; + payload.Entries.Add((IEnumerable)allowed); + filtered.WriteSecuredBulk = payload; break; + } + case MxCommandKind.WriteSecured2Bulk: - command.WriteSecured2Bulk.Entries.Clear(); - command.WriteSecured2Bulk.Entries.Add((IEnumerable)allowed); + { + WriteSecured2BulkCommand payload = new() { ServerHandle = command.WriteSecured2Bulk.ServerHandle }; + payload.Entries.Add((IEnumerable)allowed); + filtered.WriteSecured2Bulk = payload; break; + } + + default: + // Only the four bulk-write kinds above reach FilterWriteBulkAsync, so this is + // unreachable; keep the previous behaviour (the unmodified command) rather than + // emitting a payload-less one if that ever stops holding. + return command.Clone(); } + + return filtered; } private async Task FilterHandleBulkAsync( @@ -647,6 +714,11 @@ public sealed class MxAccessGatewayService( string? correlationId, CancellationToken cancellationToken) { + if (!constraintEnforcer.HasReadConstraints(identity)) + { + return null; + } + Dictionary denied = []; List allowed = []; for (int index = 0; index < itemHandles.Count; index++) @@ -677,9 +749,10 @@ public sealed class MxAccessGatewayService( return null; } - MxCommand filtered = command.Clone(); - filtered.AdviseItemBulk.ItemHandles.Clear(); - filtered.AdviseItemBulk.ItemHandles.Add(allowed); + MxCommand filtered = new() { Kind = command.Kind }; + AdviseItemBulkCommand payload = new() { ServerHandle = command.AdviseItemBulk.ServerHandle }; + payload.ItemHandles.Add(allowed); + filtered.AdviseItemBulk = payload; return new SubscribeBulkConstraintPlan(filtered, itemHandles.Count, denied, allowed.Count > 0); } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/GatewayApiKeyIdentityMapper.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/GatewayApiKeyIdentityMapper.cs index 75d9591..69d73c9 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/GatewayApiKeyIdentityMapper.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/GatewayApiKeyIdentityMapper.cs @@ -19,10 +19,35 @@ namespace ZB.MOM.WW.MxGateway.Server.Security.Authentication; /// public static class GatewayApiKeyIdentityMapper { - private const int MaxCachedConstraintBlobs = 1024; + /// + /// Maximum number of parsed constraint blobs retained in . + /// Blobs are admin-controlled (one per API key), so the cap is only a memory backstop for a + /// store with an unusually large number of distinct constrained keys. + /// + internal const int MaxCachedConstraintBlobs = 1024; + + /// + /// Bounded parsed-constraints cache keyed by the raw constraints JSON. The blob is parsed + /// once per authenticated RPC otherwise, so this keeps the JSON parse off the hot path. + /// Beyond entries the oldest insertion is evicted + /// rather than the cache refusing new entries — a hard stop at the cap would leave every key + /// admitted after it re-parsing its blob on every RPC for the process lifetime. Eviction is + /// approximate (FIFO over insertion order, not true LRU) because only the bound matters. + /// private static readonly ConcurrentDictionary ConstraintCache = new(StringComparer.Ordinal); + /// + /// Insertion-order queue used to evict the oldest cache entry once the cache exceeds + /// . Keeping it separate leaves + /// reads lock-free; the lock guards only the eviction path. + /// + private static readonly ConcurrentQueue InsertionOrder = new(); + private static readonly object EvictionLock = new(); + + /// Current cache size, exposed for tests asserting the cap is honoured. + internal static int CurrentCacheSize => ConstraintCache.Count; + private static ApiKeyConstraints DeserializeConstraints(string? constraintsJson) { if (string.IsNullOrWhiteSpace(constraintsJson)) @@ -36,12 +61,36 @@ public static class GatewayApiKeyIdentityMapper } ApiKeyConstraints parsed = ApiKeyConstraintSerializer.Deserialize(constraintsJson); - if (ConstraintCache.Count < MaxCachedConstraintBlobs) + + // GetOrAdd returns whichever instance is in the cache after the call, so concurrent parsers + // of the same blob converge on one instance; it also avoids the TryAdd-then-read race where + // the key could be evicted between a failed TryAdd and the read back. + ApiKeyConstraints result = ConstraintCache.GetOrAdd(constraintsJson, parsed); + if (ReferenceEquals(result, parsed)) { - ConstraintCache.TryAdd(constraintsJson, parsed); + // We were the inserter — track for FIFO eviction and bound the cache. + InsertionOrder.Enqueue(constraintsJson); + EvictIfOverCapacity(); } - return parsed; + return result; + } + + private static void EvictIfOverCapacity() + { + if (ConstraintCache.Count <= MaxCachedConstraintBlobs) + { + return; + } + + // Serialize eviction so two threads do not race past the cap together. + lock (EvictionLock) + { + while (ConstraintCache.Count > MaxCachedConstraintBlobs && InsertionOrder.TryDequeue(out string? oldest)) + { + ConstraintCache.TryRemove(oldest, out _); + } + } } /// diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ConstraintEnforcer.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ConstraintEnforcer.cs index e897c9e..a61bcfd 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ConstraintEnforcer.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ConstraintEnforcer.cs @@ -16,6 +16,14 @@ public sealed class ConstraintEnforcer( IGalaxyHierarchyCache cache, IAuditWriter auditWriter) : IConstraintEnforcer { + /// + public bool HasReadConstraints(ApiKeyIdentity? identity) => + identity?.EffectiveConstraints.HasReadConstraints ?? false; + + /// + public bool HasWriteConstraints(ApiKeyIdentity? identity) => + identity?.EffectiveConstraints.HasWriteConstraints ?? false; + /// public Task CheckReadTagAsync( ApiKeyIdentity? identity, @@ -211,7 +219,25 @@ public sealed class ConstraintEnforcer( return true; } - return subtreeGlobs.Any(glob => GalaxyGlobMatcher.IsMatch(containedPath, glob)) - || tagGlobs.Any(glob => GalaxyGlobMatcher.IsMatch(tagAddress, glob)); + // Plain index loops rather than Any(lambda): this runs once per item of every bulk + // read/write, and the closures the lambdas capture (containedPath / tagAddress) allocate a + // display class plus a delegate per call. Same short-circuit order, same result. + for (int i = 0; i < subtreeGlobs.Count; i++) + { + if (GalaxyGlobMatcher.IsMatch(containedPath, subtreeGlobs[i])) + { + return true; + } + } + + for (int i = 0; i < tagGlobs.Count; i++) + { + if (GalaxyGlobMatcher.IsMatch(tagAddress, tagGlobs[i])) + { + return true; + } + } + + return false; } } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/IConstraintEnforcer.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/IConstraintEnforcer.cs index 406f06e..95728ae 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/IConstraintEnforcer.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/IConstraintEnforcer.cs @@ -5,6 +5,30 @@ namespace ZB.MOM.WW.MxGateway.Server.Security.Authorization; public interface IConstraintEnforcer { + /// + /// Gets a value indicating whether any read constraint applies to an identity at all, so a + /// bulk caller can hoist the question out of its per-item loop. + /// + /// The API key identity. + /// when at least one read constraint applies; otherwise . + /// + /// Every per-item / call for + /// an unconstrained identity allows the item, so skipping the loop removes work without + /// changing a decision. The default implementation answers — an + /// implementation that does not model constraints (test doubles, allow-all enforcers) keeps + /// being consulted per item rather than being silently bypassed. + /// + bool HasReadConstraints(ApiKeyIdentity? identity) => true; + + /// + /// Gets a value indicating whether any write constraint applies to an identity at all, the + /// write-side counterpart of . + /// + /// The API key identity. + /// when at least one write constraint applies; otherwise . + /// The same conservative default as applies. + bool HasWriteConstraints(ApiKeyIdentity? identity) => true; + /// Checks whether a read constraint is satisfied for a tag address. /// The API key identity. /// Tag address to check. diff --git a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SparseArrayExpander.cs b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SparseArrayExpander.cs index c6006db..c8f81eb 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SparseArrayExpander.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SparseArrayExpander.cs @@ -120,6 +120,10 @@ internal static class SparseArrayExpander case MxDataType.Boolean: { BoolArray values = new(); + + // Size the backing store once: the fill below adds exactly `length` elements, + // so without this the RepeatedField doubles its array log2(length) times. + values.Values.Capacity = length; for (int i = 0; i < length; i++) { values.Values.Add(false); @@ -137,6 +141,7 @@ internal static class SparseArrayExpander case MxDataType.Integer when UsesInt64(elements): { Int64Array values = new(); + values.Values.Capacity = length; for (int i = 0; i < length; i++) { values.Values.Add(0L); @@ -154,6 +159,7 @@ internal static class SparseArrayExpander case MxDataType.Integer: { Int32Array values = new(); + values.Values.Capacity = length; for (int i = 0; i < length; i++) { values.Values.Add(0); @@ -171,6 +177,7 @@ internal static class SparseArrayExpander case MxDataType.Float: { FloatArray values = new(); + values.Values.Capacity = length; for (int i = 0; i < length; i++) { values.Values.Add(0f); @@ -188,6 +195,7 @@ internal static class SparseArrayExpander case MxDataType.Double: { DoubleArray values = new(); + values.Values.Capacity = length; for (int i = 0; i < length; i++) { values.Values.Add(0d); @@ -205,6 +213,7 @@ internal static class SparseArrayExpander case MxDataType.String: { StringArray values = new(); + values.Values.Capacity = length; for (int i = 0; i < length; i++) { values.Values.Add(string.Empty); @@ -222,6 +231,7 @@ internal static class SparseArrayExpander case MxDataType.Time: { TimestampArray values = new(); + values.Values.Capacity = length; for (int i = 0; i < length; i++) { values.Values.Add(new Timestamp { Seconds = 0, Nanos = 0 }); diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs index 2577a0f..a75046c 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs @@ -8,10 +8,12 @@ using LibApiKeyIdentity = ZB.MOM.WW.Auth.Abstractions.ApiKeys.ApiKeyIdentity; namespace ZB.MOM.WW.MxGateway.Tests.Security.Authentication; /// -/// Hot-path decorators. Covers both mechanisms: -/// (read/verification coalescing plus revoke/rotate invalidation) and +/// Hot-path decorators. Covers all three mechanisms: +/// (read/verification coalescing plus revoke/rotate invalidation), /// (the last_used write coalescing that keeps the -/// per-RPC database write off the throughput ceiling). +/// per-RPC database write off the throughput ceiling), and the constraint-blob cache inside +/// (which keeps the per-RPC constraints JSON parse off +/// the authenticated path). /// public sealed class CachingApiKeyVerifierTests { @@ -245,6 +247,50 @@ public sealed class CachingApiKeyVerifierTests Assert.Equal(keyId, CachingApiKeyVerifier.TryParseKeyId($"Bearer mxgw_{keyId}_secret")); } + /// + /// The mapper's constraint-blob cache is bounded by eviction, not by a hard stop at the cap: + /// once it is full the oldest entry is dropped so a newly-seen blob is still cached. A cache + /// that merely stopped accepting entries would re-parse every blob beyond the cap on every + /// single RPC, forever. Asserted behaviourally through instance identity — a cached blob maps + /// to the same instance, a re-parsed one does not. + /// + [Fact] + public void ToGatewayIdentity_ConstraintCacheOverCapacity_EvictsOldestAndKeepsCaching() + { + string firstJson = ConstraintsJson("Area_FifoProbe"); + ApiKeyConstraints first = MapConstraints(firstJson); + Assert.Same(first, MapConstraints(firstJson)); + + // Push strictly more than the cap through the cache after the probe blob, so FIFO eviction + // is guaranteed to have reached it however full the (process-wide) cache already was. + for (int i = 0; i < GatewayApiKeyIdentityMapper.MaxCachedConstraintBlobs + 8; i++) + { + MapConstraints(ConstraintsJson($"Area_FifoFlood_{i}")); + } + + Assert.True( + GatewayApiKeyIdentityMapper.CurrentCacheSize <= GatewayApiKeyIdentityMapper.MaxCachedConstraintBlobs, + $"cache grew to {GatewayApiKeyIdentityMapper.CurrentCacheSize} entries, past the {GatewayApiKeyIdentityMapper.MaxCachedConstraintBlobs} cap"); + + // Evicted, so the probe blob is parsed afresh... + ApiKeyConstraints reparsed = MapConstraints(firstJson); + Assert.NotSame(first, reparsed); + Assert.Equal(first.ReadSubtrees, reparsed.ReadSubtrees); + + // ...and re-cached, rather than re-parsed on every later call. + Assert.Same(reparsed, MapConstraints(firstJson)); + } + + private static ApiKeyConstraints MapConstraints(string constraintsJson) => + GatewayApiKeyIdentityMapper.ToGatewayIdentity(new LibApiKeyIdentity( + KeyId: "operator01", + DisplayName: "Operator Key", + Scopes: new HashSet(StringComparer.Ordinal), + Constraints: constraintsJson)).EffectiveConstraints; + + private static string ConstraintsJson(string readSubtree) => + ApiKeyConstraintSerializer.Serialize(ApiKeyConstraints.Empty with { ReadSubtrees = [readSubtree] })!; + private static MemoryCache NewCache() => new(new MemoryCacheOptions()); private static ApiKeyVerification Success(string keyId) => new( From 7c1ea123311af2864f03f89ccb1ef338634d9796 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:28:09 -0400 Subject: [PATCH 15/41] perf(ipc): WaitAsync command timeouts + forward WorkerCancel so a timed-out COM call frees the STA --- docs/GatewayProcessDesign.md | 14 +++ .../Workers/WorkerClient.cs | 118 +++++++++++++++--- .../Gateway/Workers/WorkerClientTests.cs | 65 +++++++++- 3 files changed, 178 insertions(+), 19 deletions(-) diff --git a/docs/GatewayProcessDesign.md b/docs/GatewayProcessDesign.md index 6fd2ddd..98ce742 100644 --- a/docs/GatewayProcessDesign.md +++ b/docs/GatewayProcessDesign.md @@ -593,6 +593,20 @@ Pending command handling: Timeouts should not assume the COM call stopped. A timed-out command may still finish inside the worker. +On timeout the client also forwards a `WorkerCancel` carrying the abandoned +correlation id, best-effort: the gateway has stopped waiting, but the worker has +not stopped working, and the worker owns a single STA. `WorkerPipeSession` routes +the cancel to `CancelCommand`, which drops the correlation from the STA queue if +it has not started and replies `Canceled` for it. A cancel that arrives after the +command reached MXAccess is a no-op — there is no way to abort an in-flight COM +call — so this shortens the STA backlog rather than freeing a call already +running on it, and the rule above still holds. A command whose envelope is still +in the gateway's outbound queue needs no special handling: the queue is FIFO, so +the worker reads the command and then its cancel and drops it before execution. + +Failing to send the cancel is logged at debug and never replaces the +`CommandTimeout` the caller is owed. + ## Fault Model Fault categories: diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs index be74d2b..b0c6d84 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs @@ -232,6 +232,13 @@ public sealed class WorkerClient : IWorkerClient // session. Command envelopes are the only gateway-authored outbound payload whose // size the caller controls; checking here keeps a MessageTooLarge in the write loop a // genuine desync signal. + // + // PERF(GWC-31): this size cannot be handed to WorkerFrameWriter to spare its own + // CalculateSize. WriteLoopAsync stamps envelope.Sequence immediately before the write + // (GWC-28), and a non-zero varint field grows the encoding — so the number computed here + // is a lower bound on the frame the writer actually emits, never the frame length. Passing + // it as a knownSize would under-length the prefix and desync the worker's framing. The + // pre-check stays a pre-check: it is conservative in the right direction. int envelopeSize = commandEnvelope.CalculateSize(); if (envelopeSize > _connection.FrameOptions.MaxMessageBytes) { @@ -242,36 +249,49 @@ public sealed class WorkerClient : IWorkerClient } await EnqueueAsync(commandEnvelope, cancellationToken).ConfigureAwait(false); - using CancellationTokenSource timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - Task timeoutTask = Task.Delay(timeout, timeoutCts.Token); - Task replyTask = pendingCommand.Task; - Task completedTask = await Task.WhenAny(replyTask, timeoutTask).ConfigureAwait(false); - if (completedTask == replyTask) + // GWC-31: one pooled timer instead of a linked CTS + Task.Delay + WhenAny per command. + // Task.WaitAsync arms a TimerQueueTimer on the shared timer queue and cancels it when the + // reply lands, so the steady-state cost of a command that replies in time is a single + // continuation — the old shape allocated a linked CancellationTokenSource, its + // registration, a delay Task, and the WhenAny Task on every invoke, and left the delay + // Task rooted until the cancel completed. WaitAsync raises TimeoutException for the + // deadline and OperationCanceledException for the caller's token, which is exactly the + // two-way split the old completedTask/IsCancellationRequested test made by hand; the + // error codes and messages below are unchanged. + try { - await timeoutCts.CancelAsync().ConfigureAwait(false); - return await replyTask.ConfigureAwait(false); + return await pendingCommand.Task.WaitAsync(timeout, cancellationToken).ConfigureAwait(false); } - - if (cancellationToken.IsCancellationRequested) + catch (OperationCanceledException) { RemovePendingCommandAsFailed( correlationId, pendingCommand, WorkerClientErrorCode.GatewayShutdown, "Command wait was canceled."); - cancellationToken.ThrowIfCancellationRequested(); + throw; } + catch (TimeoutException) + { + string timeoutMessage = $"Worker command {method} timed out after {timeout}."; + RemovePendingCommandAsFailed( + correlationId, + pendingCommand, + WorkerClientErrorCode.CommandTimeout, + timeoutMessage); - RemovePendingCommandAsFailed( - correlationId, - pendingCommand, - WorkerClientErrorCode.CommandTimeout, - $"Worker command {method} timed out after {timeout}."); + // The gateway has stopped waiting, but the worker has not stopped working: the + // correlation is still on its single STA queue and would execute (or keep executing) + // regardless. Tell it, so a queued-but-not-started command is dropped instead of + // occupying the STA behind a caller that is already gone. Best-effort by design — a + // failure here must never replace the timeout the caller is owed. + TrySendCancelForTimedOutCommand(correlationId, method, timeout); - throw new WorkerClientException( - WorkerClientErrorCode.CommandTimeout, - $"Worker command {method} timed out after {timeout}."); + throw new WorkerClientException( + WorkerClientErrorCode.CommandTimeout, + timeoutMessage); + } } catch { @@ -752,6 +772,68 @@ public sealed class WorkerClient : IWorkerClient pendingCommand.SetException(new WorkerClientException(errorCode, message)); } + /// + /// Forwards a WorkerCancel for a correlation the gateway has given up waiting for, so the + /// worker can drop it from its STA queue (WorkerPipeSession routes the envelope to + /// CancelCommand). A cancel that arrives after the command already reached the COM call + /// is a no-op — MXAccess offers no way to abort an in-flight call — so this shortens the STA + /// backlog rather than freeing a call already running on it. + /// + /// A command whose envelope has not yet left _outboundEnvelopes is handled by the same + /// path rather than by pulling it back out: exposes no removal, and + /// the queue is FIFO, so the worker simply reads the command and then its cancel and drops the + /// correlation before it ever reaches the STA. Nothing is gained by dequeuing it here. + /// + /// + /// + /// Every failure is swallowed and logged at debug: the caller is on the throw path for the + /// timeout, and losing the cancel costs the worker one wasted command, whereas letting an + /// exception escape would replace the the + /// caller is owed. TryWrite is the fast path and cannot throw for a closed or completed + /// channel; only when the outbound channel is momentarily at its bound is a detached task + /// started, and that task catches everything it can observe — including the + /// from _stopCts if the client is disposed + /// underneath it — so nothing is left unobserved. + /// + /// Correlation id of the command that timed out. + /// Command method name, for the cancel reason and diagnostics. + /// The elapsed command timeout, for the cancel reason. + private void TrySendCancelForTimedOutCommand( + string correlationId, + string method, + TimeSpan timeout) + { + WorkerEnvelope cancelEnvelope = CreateEnvelope( + correlationId, + envelope => envelope.WorkerCancel = new WorkerCancel + { + Reason = $"gateway command timeout after {timeout}", + }); + + if (_outboundEnvelopes.Writer.TryWrite(cancelEnvelope)) + { + return; + } + + _ = Task.Run(async () => + { + try + { + await EnqueueAsync(cancelEnvelope, _stopCts.Token).ConfigureAwait(false); + } + catch (Exception exception) + { + _logger.LogDebug( + exception, + "Could not forward a cancel for timed-out worker command {Method} on session {SessionId} " + + "and correlation {CorrelationId}.", + method, + SessionId, + correlationId); + } + }); + } + /// Reads and validates a handshake envelope. /// Expected envelope body type. /// Cancellation token. diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs index 5855572..b4f34dd 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs @@ -159,7 +159,11 @@ public sealed class WorkerClientTests CreateCommand(MxCommandKind.GetWorkerInfo), TestTimeout, CancellationToken.None); - WorkerEnvelope secondCommand = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout); + + // The timeout also emits a WorkerCancel for the abandoned correlation (GWC-31), which sits + // ahead of the second command on the FIFO pipe; skip it rather than mistaking it for the + // command this assertion is about. + WorkerEnvelope secondCommand = await ReadNextCommandAsync(pipePair); await pipePair.WriteAsync( CreateCommandReplyEnvelope(secondCommand.CorrelationId, MxCommandKind.GetWorkerInfo)); @@ -169,6 +173,47 @@ public sealed class WorkerClientTests Assert.Equal(MxCommandKind.GetWorkerInfo, reply.Reply.Kind); } + /// + /// A command timeout abandons the gateway-side wait, but the worker keeps the correlation on + /// its single STA queue and would still run it — so the gateway forwards a WorkerCancel + /// for the abandoned correlation id (GWC-31). Without it, a client that retries after a + /// timeout stacks work the worker still intends to execute. Asserted on the wire because the + /// cancel is protocol behavior the worker depends on, not an internal detail. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task InvokeAsync_WhenCommandTimesOut_SendsWorkerCancelForThatCorrelation() + { + await using PipePair pipePair = await PipePair.CreateAsync(); + await using WorkerClient client = CreateClient(pipePair); + await CompleteHandshakeAsync(client, pipePair); + + Task invokeTask = client.InvokeAsync( + CreateCommand(MxCommandKind.Ping), + TimeSpan.FromMilliseconds(50), + CancellationToken.None); + + WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout); + Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCommand, commandEnvelope.BodyCase); + + WorkerClientException exception = await Assert.ThrowsAsync( + async () => await invokeTask.WaitAsync(TestTimeout)); + Assert.Equal(WorkerClientErrorCode.CommandTimeout, exception.ErrorCode); + + WorkerEnvelope cancelEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout); + + Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCancel, cancelEnvelope.BodyCase); + Assert.Equal(commandEnvelope.CorrelationId, cancelEnvelope.CorrelationId); + Assert.False(string.IsNullOrWhiteSpace(cancelEnvelope.WorkerCancel.Reason)); + Assert.True( + cancelEnvelope.Sequence > commandEnvelope.Sequence, + $"The cancel arrived with sequence {cancelEnvelope.Sequence} after {commandEnvelope.Sequence}; " + + "envelope sequences must be strictly increasing in wire order."); + + // The timeout fails one command; it is not a session fault. + Assert.Equal(WorkerClientState.Ready, client.State); + } + /// /// The envelope sequence is a monotonic per-sender counter (gateway.md), so the values /// observed on the pipe must be strictly increasing in wire order. Stamping the sequence when @@ -1030,6 +1075,24 @@ public sealed class WorkerClientTests return envelope; } + /// + /// Reads gateway envelopes until a WorkerCommand arrives, skipping the control envelopes + /// (such as the WorkerCancel a command timeout emits) that may precede it on the FIFO pipe. + /// + /// The connected pipe pair whose worker side is read. + /// The next command envelope written by the gateway. + private static async Task ReadNextCommandAsync(PipePair pipePair) + { + while (true) + { + WorkerEnvelope envelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout); + if (envelope.BodyCase == WorkerEnvelope.BodyOneofCase.WorkerCommand) + { + return envelope; + } + } + } + private static async Task WaitUntilAsync( Func predicate, TimeSpan timeout) From 44ca7c8623f8a04cb20d60277964dc9cf9856ad7 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:31:36 -0400 Subject: [PATCH 16/41] fix(dashboard): enforce/document the advised-set cap honestly; cover handle-0 and oversize-read paths --- docs/GatewayDashboardDesign.md | 13 ++ .../Dashboard/DashboardLiveDataService.cs | 33 ++++- .../DashboardLiveDataServiceTests.cs | 130 ++++++++++++++++-- 3 files changed, 163 insertions(+), 13 deletions(-) diff --git a/docs/GatewayDashboardDesign.md b/docs/GatewayDashboardDesign.md index d2327fa..77be9d4 100644 --- a/docs/GatewayDashboardDesign.md +++ b/docs/GatewayDashboardDesign.md @@ -406,6 +406,19 @@ room for each other. A failed unadvise does not fail the read: the tags are dropped from tracking anyway (they re-subscribe if read again), because the session-invalidation path already handles gateway/worker drift. +The cap is per-read, not absolute. A read may never evict a tag it is itself about +to return, so one read of more distinct tags than the cap leaves the set that +large; what the eviction pass guarantees is + +> after any read, the advise set holds at most `max(256, distinct tags in that read)` +> tags. + +The overshoot is not sticky: the next read that subscribes anything measures the +overflow against the oversized set and evicts the whole excess in one pass (a +300-tag set plus one new tag evicts 45 and lands back at 256). A read that +subscribes nothing new evicts nothing, but neither can it grow the set. A browse +page requests far fewer tags than the cap, so in practice the set settles at 256. + The Alarms page does **not** use the dashboard session: alarm data comes from the gateway's always-on central monitor. `QueryAlarmsAsync` reads `IGatewayAlarmService.CurrentAlarms` — the monitor's in-process cache — so the diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardLiveDataService.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardLiveDataService.cs index d28a44c..ec2aa2e 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardLiveDataService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardLiveDataService.cs @@ -19,6 +19,16 @@ public sealed class DashboardLiveDataService : IDashboardLiveDataService, IAsync // One browse page of tags plus headroom. Bounds the standing advise load the // single dashboard worker carries — and the event churn that advise set feeds — // however much of a galaxy an operator browses through in one sitting. + // + // The bound is per-read, not absolute: a read may never evict a tag it is itself + // about to return, so a single read of more distinct tags than the cap leaves the + // set that large. The invariant EvictForAsync actually maintains is + // + // |advise set| after a read <= max(MaxSubscribedTags, distinct tags in that read) + // + // and any overshoot is squeezed back out by the next read that subscribes a tag + // (see EvictForAsync). A browse page requests far fewer tags than the cap, so in + // practice the set settles at MaxSubscribedTags. private const int MaxSubscribedTags = 256; private static readonly TimeSpan ReadTimeout = TimeSpan.FromSeconds(5); @@ -125,6 +135,11 @@ public sealed class DashboardLiveDataService : IDashboardLiveDataService, IAsync // order). `justReadCount` is how many distinct tags of this read were already // advised — they now occupy the front of the list and must never be evicted to // make room for the same read's new tags. Callers must hold _gate. + // + // Every tag of one read is equally recently read; the recency list needs a total + // order anyway, so the whole service uses one tie-break: later in the request wins. + // Promoting in request order gives that here, and TrackSubscribed inserts new tags + // the same way. private string[] TouchAndCollectNewTags(IReadOnlyCollection tagAddresses, out int justReadCount) { int touched = 0; @@ -161,6 +176,18 @@ public sealed class DashboardLiveDataService : IDashboardLiveDataService, IAsync // one batch. A failed unadvise must not fail the read: the tags are dropped // from tracking regardless, and the session-invalidation path already handles // gateway/worker drift. Callers must hold _gate. + // + // Eviction stops at the tags this read just touched (`justReadCount`), so a read + // whose own distinct tags outnumber the cap ends over it — see MaxSubscribedTags + // for the exact invariant. That overshoot is not sticky: the next read that + // subscribes anything computes `overflow` against the oversized set and evicts the + // whole excess in one pass (a 300-tag set plus one new tag evicts 45 and lands + // back at the cap). A read that subscribes nothing new evicts nothing, but it also + // cannot grow the set. + // + // Cancellation mid-eviction follows this file's policy: OperationCanceledException + // is deliberately not caught here or in ReadAsync, so it propagates with the tags + // already dropped from tracking — the same end state as a failed unadvise. private async Task EvictForAsync( GatewaySession session, int serverHandle, @@ -227,10 +254,10 @@ public sealed class DashboardLiveDataService : IDashboardLiveDataService, IAsync } } - // Inserted back-to-front so the read's first tag ends up most recent. - for (int i = tagAddresses.Count - 1; i >= 0; i--) + // Request order, so the read's last tag ends up most recent — the same + // tie-break TouchAndCollectNewTags applies to the tags it promotes. + foreach (string tag in tagAddresses) { - string tag = tagAddresses[i]; handles.TryGetValue(tag, out int itemHandle); _subscribed[tag] = _recency.AddFirst(new SubscribedTag(tag, itemHandle)); } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardLiveDataServiceTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardLiveDataServiceTests.cs index dcb12ca..3059458 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardLiveDataServiceTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardLiveDataServiceTests.cs @@ -46,12 +46,14 @@ public sealed class DashboardLiveDataServiceTests await using FakeSessionManager sessionManager = new(worker); await using DashboardLiveDataService service = CreateService(sessionManager); + // Within one read the last tag counts as most recently read, so filler[0] is + // the tail of the recency list. string[] filler = CreateTagAddresses(MaxSubscribedTags); await service.ReadAsync(filler, CancellationToken.None); - // Re-read the tag that is currently least recent, making it most recent: the - // tag read before it is now the eviction candidate. - await service.ReadAsync([filler[^1]], CancellationToken.None); + // Re-read the tail, making it most recent: the tag read before it is now the + // eviction candidate. + await service.ReadAsync([filler[0]], CancellationToken.None); Assert.Equal(MaxSubscribedTags, worker.SubscribedTags.Count); DashboardLiveReadResult overflow = await service.ReadAsync( @@ -59,16 +61,104 @@ public sealed class DashboardLiveDataServiceTests CancellationToken.None); Assert.Null(overflow.Error); - Assert.Equal([worker.HandleFor(filler[^2])], worker.UnsubscribedHandles); + Assert.Equal([worker.HandleFor(filler[1])], worker.UnsubscribedHandles); Assert.Equal("Overflow.PV", worker.SubscribedTags[^1]); Assert.Equal(MaxSubscribedTags + 1, worker.SubscribedTags.Count); // The evicted tag is no longer tracked and re-subscribes; the re-read one does not. - await service.ReadAsync([filler[^1], filler[^2]], CancellationToken.None); - Assert.Equal(filler[^2], worker.SubscribedTags[^1]); + await service.ReadAsync([filler[0], filler[1]], CancellationToken.None); + Assert.Equal(filler[1], worker.SubscribedTags[^1]); Assert.Equal(MaxSubscribedTags + 2, worker.SubscribedTags.Count); } + /// + /// Verifies the cap is per-read, not absolute: a single read of more distinct tags + /// than the cap keeps them all (a read never evicts a tag it is about to return), + /// and the next read that subscribes anything squeezes the overshoot back out. + /// + [Fact] + public async Task ReadAsync_WithMoreDistinctTagsThanCap_KeepsThemAllThenSelfCorrects() + { + RecordingWorkerClient worker = new(); + await using FakeSessionManager sessionManager = new(worker); + await using DashboardLiveDataService service = CreateService(sessionManager); + + string[] oversize = CreateTagAddresses(300); + DashboardLiveReadResult oversizeResult = await service.ReadAsync(oversize, CancellationToken.None); + + Assert.Null(oversizeResult.Error); + Assert.Equal(300, oversizeResult.Values.Count); + Assert.Equal(300, worker.SubscribedTags.Count); + Assert.Empty(worker.UnsubscribedHandles); + + // 300 + 1 - 256 = 45 evicted in one pass, landing the set back on the cap. + await service.ReadAsync(["Overflow.PV"], CancellationToken.None); + Assert.Equal(oversize[..45].Select(worker.HandleFor), worker.UnsubscribedHandles); + + // Exactly at the cap now: one more new tag evicts exactly one. + worker.UnsubscribedHandles.Clear(); + await service.ReadAsync(["Overflow2.PV"], CancellationToken.None); + Assert.Equal([worker.HandleFor(oversize[45])], worker.UnsubscribedHandles); + } + + /// + /// Verifies tags read in the same call are never evicted for each other: a read that + /// touches nearly the whole advise set evicts only the untouched remainder, ends over + /// the cap, and the following read trims it back. + /// + [Fact] + public async Task ReadAsync_WhenTouchedTagsFillTheCap_EvictsOnlyUntouchedTags() + { + RecordingWorkerClient worker = new(); + await using FakeSessionManager sessionManager = new(worker); + await using DashboardLiveDataService service = CreateService(sessionManager); + + string[] filler = CreateTagAddresses(MaxSubscribedTags); + await service.ReadAsync(filler, CancellationToken.None); + + // 250 already-advised tags + 10 new ones: only the 6 untouched tags are + // evictable, so the set ends at 260. + string[] fresh = CreateTagAddresses(10, "Fresh"); + await service.ReadAsync([.. filler[..250], .. fresh], CancellationToken.None); + + Assert.Equal(filler[250..].Select(worker.HandleFor), worker.UnsubscribedHandles); + + // 260 + 1 - 256 = 5 evicted on the next read that subscribes anything. + worker.UnsubscribedHandles.Clear(); + await service.ReadAsync(["Overflow.PV"], CancellationToken.None); + Assert.Equal(5, worker.UnsubscribedHandles.Count); + } + + /// + /// Verifies a tag the worker failed to advise still occupies a slot but is evicted + /// without any unsubscribe command — there is no item handle to unadvise. + /// + [Fact] + public async Task ReadAsync_WhenAdviseFailed_EvictsTagWithoutUnsubscribing() + { + RecordingWorkerClient worker = new(); + worker.FailSubscribeFor.Add("Bad.PV"); + await using FakeSessionManager sessionManager = new(worker); + await using DashboardLiveDataService service = CreateService(sessionManager); + + // Bad.PV is read first, so it is the least recently read of the batch and the + // first tag evicted. + string[] filler = CreateTagAddresses(MaxSubscribedTags - 1); + await service.ReadAsync(["Bad.PV", .. filler], CancellationToken.None); + + DashboardLiveReadResult overflow = await service.ReadAsync( + ["Overflow.PV"], + CancellationToken.None); + + Assert.Null(overflow.Error); + Assert.Empty(worker.UnsubscribedHandles); + Assert.Equal(0, worker.UnsubscribeCommandCount); + + // It was dropped from tracking all the same, so reading it again re-advises it. + await service.ReadAsync(["Bad.PV"], CancellationToken.None); + Assert.Equal("Bad.PV", worker.SubscribedTags[^1]); + } + /// /// Verifies a failed unadvise of an evicted tag does not fail the read, and the /// evicted tag is dropped from tracking anyway. @@ -92,8 +182,8 @@ public sealed class DashboardLiveDataServiceTests Assert.Equal(1, sessionManager.OpenCount); // The evicted tag was dropped from tracking despite the failed unadvise. - await service.ReadAsync([filler[^1]], CancellationToken.None); - Assert.Equal(filler[^1], worker.SubscribedTags[^1]); + await service.ReadAsync([filler[0]], CancellationToken.None); + Assert.Equal(filler[0], worker.SubscribedTags[^1]); } private static DashboardLiveDataService CreateService(ISessionManager sessionManager) @@ -104,12 +194,12 @@ public sealed class DashboardLiveDataServiceTests NullLogger.Instance); } - private static string[] CreateTagAddresses(int count) + private static string[] CreateTagAddresses(int count, string prefix = "Tank") { string[] addresses = new string[count]; for (int i = 0; i < count; i++) { - addresses[i] = $"Tank_{i:D4}.PV"; + addresses[i] = $"{prefix}_{i:D4}.PV"; } return addresses; @@ -227,6 +317,12 @@ public sealed class DashboardLiveDataServiceTests /// Gets the item handles the dashboard unsubscribed, in order. public List UnsubscribedHandles { get; } = []; + /// Gets the number of unsubscribe commands the dashboard sent. + public int UnsubscribeCommandCount { get; private set; } + + /// Gets the tag addresses the worker refuses to advise. + public HashSet FailSubscribeFor { get; } = new(StringComparer.OrdinalIgnoreCase); + /// Gets or sets a value indicating whether unsubscribe commands throw. public bool FailUnsubscribe { get; set; } @@ -262,6 +358,7 @@ public sealed class DashboardLiveDataServiceTests reply.SubscribeBulk = Subscribe(mxCommand.SubscribeBulk.TagAddresses); break; case MxCommandKind.UnsubscribeBulk: + UnsubscribeCommandCount++; if (FailUnsubscribe) { throw new InvalidOperationException("Simulated worker unsubscribe failure."); @@ -305,6 +402,19 @@ public sealed class DashboardLiveDataServiceTests foreach (string tagAddress in tagAddresses) { SubscribedTags.Add(tagAddress); + if (FailSubscribeFor.Contains(tagAddress)) + { + subscribeReply.Results.Add(new SubscribeResult + { + ServerHandle = RegisteredServerHandle, + TagAddress = tagAddress, + ItemHandle = 0, + WasSuccessful = false, + ErrorMessage = "Simulated advise failure.", + }); + continue; + } + if (!_itemHandles.TryGetValue(tagAddress, out int itemHandle)) { itemHandle = _nextItemHandle++; From f920b4cbf5767afe82b416105cb8941108a3b3e9 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:32:58 -0400 Subject: [PATCH 17/41] fix(dashboard): clamped CAS retry loop in snapshot hub connection counter + direct counter tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decrement was decrement-first with a single non-retried repair CAS. From zero, two unmatched decrements (SignalR calls OnDisconnectedAsync for a connection whose OnConnectedAsync faulted) capture -1 and -2; a real Increment then makes the count -1, and the first decrementer's stale CompareExchange(0, -1) matches and resets to zero — erasing a live connection, so the idle gate freezes an open dashboard. The same lost race also made Decrement report 0 when it had not written 0. Clamping now happens inside the compare-and-swap: read, clamp, publish, retry on loss. A lost race re-reads the fresh value instead of repairing a stale one. The counter moves to its own file per the one-public-type-per-file convention and gains direct tests: the zero floor under concurrent unmatched decrements, matched pairs settling at zero, and an interleaved connect/disconnect stress round. The stress test asserts the observable invariants only — the specific interleaving cannot be forced through the public API (verified: the previous implementation passes it), which its remarks now state rather than implying a reproducer. A hub wiring test is skipped for the EventsHub reason: driving Hub.OnConnectedAsync needs caller-clients and connection-context fakes, and the overrides are two lines of delegation to the tested type. Also documents that the API-key refresh's pre-gate time check races benignly. --- .../Dashboard/DashboardSnapshotService.cs | 3 + .../Dashboard/Hubs/DashboardSnapshotHub.cs | 40 ----- .../DashboardSnapshotHubConnectionCounter.cs | 52 ++++++ ...hboardSnapshotHubConnectionCounterTests.cs | 154 ++++++++++++++++++ 4 files changed, 209 insertions(+), 40 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHubConnectionCounter.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotHubConnectionCounterTests.cs diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs index 49dc7be..25b2e42 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs @@ -294,6 +294,9 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService // Inside the refresh window: reuse the cached summaries rather than // re-reading the API key table on this tick. Only a *successful* refresh // moves the timestamp, so a failed read is retried on the next tick. + // This check is deliberately outside the refresh gate, so it races + // benignly: if two callers both read a stale timestamp, the zero-timeout + // gate below admits one and the other returns without touching the store. return; } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHub.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHub.cs index 3cc8d9f..4f340be 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHub.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHub.cs @@ -39,43 +39,3 @@ public sealed class DashboardSnapshotHub( await base.OnDisconnectedAsync(exception).ConfigureAwait(false); } } - -/// -/// Process-wide count of live connections. -/// Registered as a singleton and read by -/// to idle-gate the snapshot tick: with no dashboard connected there is nothing -/// to broadcast to, so no snapshot is built. -/// -public sealed class DashboardSnapshotHubConnectionCounter -{ - private int _count; - - /// Gets the number of live snapshot hub connections. - public int Count => Volatile.Read(ref _count); - - /// Records a new snapshot hub connection. - /// The connection count after the increment. - public int Increment() - { - return Interlocked.Increment(ref _count); - } - - /// - /// Records a snapshot hub disconnection. The count is clamped at zero: SignalR - /// can invoke OnDisconnectedAsync for a connection whose - /// OnConnectedAsync faulted, and a negative count would idle-gate the - /// publisher while viewers are still attached. - /// - /// The connection count after the decrement. - public int Decrement() - { - int updated = Interlocked.Decrement(ref _count); - if (updated >= 0) - { - return updated; - } - - Interlocked.CompareExchange(ref _count, 0, updated); - return 0; - } -} diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHubConnectionCounter.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHubConnectionCounter.cs new file mode 100644 index 0000000..4695565 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/DashboardSnapshotHubConnectionCounter.cs @@ -0,0 +1,52 @@ +namespace ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; + +/// +/// Process-wide count of live connections. +/// Registered as a singleton and read by +/// to idle-gate the snapshot tick: with no dashboard connected there is nothing +/// to broadcast to, so no snapshot is built. +/// +public sealed class DashboardSnapshotHubConnectionCounter +{ + private int _count; + + /// Gets the number of live snapshot hub connections. + public int Count => Volatile.Read(ref _count); + + /// Records a new snapshot hub connection. + /// The connection count after the increment. + public int Increment() + { + return Interlocked.Increment(ref _count); + } + + /// + /// Records a snapshot hub disconnection, clamped at zero: SignalR can invoke + /// OnDisconnectedAsync for a connection whose OnConnectedAsync + /// faulted, and a negative count would idle-gate the publisher while viewers + /// are still attached. + /// + /// + /// The clamp is applied inside the compare-and-swap rather than as a repair + /// afterwards. Decrementing first and then correcting a negative result races: + /// two unmatched decrements from zero would both plan a repair, a real + /// connection could increment in between, and the stale repair would then + /// overwrite that live connection's increment — freezing a real viewer's + /// dashboard behind the idle gate. Reading, clamping, and publishing as one + /// atomic step means a lost race simply retries against the fresh value. + /// + /// The connection count after the decrement. + public int Decrement() + { + int current; + int next; + do + { + current = Volatile.Read(ref _count); + next = current > 0 ? current - 1 : 0; + } + while (Interlocked.CompareExchange(ref _count, next, current) != current); + + return next; + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotHubConnectionCounterTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotHubConnectionCounterTests.cs new file mode 100644 index 0000000..6c50716 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotHubConnectionCounterTests.cs @@ -0,0 +1,154 @@ +using ZB.MOM.WW.MxGateway.Server.Dashboard.Hubs; + +namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard; + +/// +/// Verifies , the seam +/// reads before building a snapshot. +/// An over-count leaves the publisher ticking for nobody; an under-count is +/// worse — it idle-gates a dashboard that is actually open, so the page silently +/// stops updating. The counter is exercised directly rather than through the hub, +/// mirroring the precedent: a SignalR +/// Hub instance needs a caller-clients and connection context fake to +/// invoke OnConnectedAsync, and the hub methods themselves are two lines +/// of delegation to this type. +/// +public sealed class DashboardSnapshotHubConnectionCounterTests +{ + /// A fresh counter reports no viewers, so the publisher starts idle. + [Fact] + public void Count_WhenNothingConnected_IsZero() + { + DashboardSnapshotHubConnectionCounter counter = new(); + + Assert.Equal(0, counter.Count); + } + + /// Connect/disconnect pairs move the count and return the post-operation value. + [Fact] + public void IncrementThenDecrement_TracksLiveConnections() + { + DashboardSnapshotHubConnectionCounter counter = new(); + + Assert.Equal(1, counter.Increment()); + Assert.Equal(2, counter.Increment()); + Assert.Equal(2, counter.Count); + + Assert.Equal(1, counter.Decrement()); + Assert.Equal(0, counter.Decrement()); + Assert.Equal(0, counter.Count); + } + + /// + /// SignalR calls OnDisconnectedAsync for a connection whose + /// OnConnectedAsync faulted, so unmatched decrements happen. They must + /// hold the floor at zero rather than driving the count negative. + /// + [Fact] + public void Decrement_WithoutMatchingIncrement_HoldsAtZero() + { + DashboardSnapshotHubConnectionCounter counter = new(); + + Assert.Equal(0, counter.Decrement()); + Assert.Equal(0, counter.Decrement()); + Assert.Equal(0, counter.Count); + + // A genuine connection after unmatched disconnects still registers as one. + Assert.Equal(1, counter.Increment()); + } + + /// + /// Many concurrent unmatched decrements must not leave the count below zero: + /// a negative floor would swallow the next real connection's increment and + /// keep the publisher idle-gated while a viewer waits. The clamp lives inside + /// the compare-and-swap, so the floor holds however the calls interleave. + /// + [Fact] + public void Decrement_UnderConcurrencyFromZero_NeverGoesNegative() + { + DashboardSnapshotHubConnectionCounter counter = new(); + + Parallel.For(0, 256, _ => counter.Decrement()); + + Assert.Equal(0, counter.Count); + + Assert.Equal(1, counter.Increment()); + Assert.Equal(1, counter.Count); + } + + /// + /// Stress check on the invariant the idle gate depends on: real connects + /// interleaved with unmatched disconnects leave the count in [0, connects], and a + /// connect after the storm is always visible to the publisher. The failure this + /// guards is the decrement-then-repair race the CAS retry loop replaced — an early + /// decrementer's stale repair either erases a live connection's increment or leaves + /// a negative value behind, and either way an open dashboard freezes behind the + /// gate. + /// + /// + /// This does not deterministically reproduce that race, and it is not claimed to: + /// the bad interleaving needs a specific few-instruction overlap that cannot be + /// forced through the public API, and a merely low count is a legitimate outcome + /// here (a decrement that runs while the count is positive consumes a real + /// connection). Verified by experiment: the previous implementation passes this + /// test. What is asserted are the observable consequences — never negative, the + /// floor holds, a later connect still registers — with the correctness argument + /// resting on the clamped CAS retry loop itself. + /// + [Fact] + public void IncrementAndDecrement_InterleavedUnderConcurrency_StayWithinTheRealConnectionCount() + { + const int LiveConnections = 8; + const int UnmatchedDisconnects = 128; + + for (int round = 0; round < 50; round++) + { + DashboardSnapshotHubConnectionCounter counter = new(); + + // A few workers are real connects that must survive; the rest are + // unmatched disconnects hammering the zero floor around them. + Parallel.For(0, LiveConnections + UnmatchedDisconnects, index => + { + if (index % 16 == 0 && index / 16 < LiveConnections) + { + counter.Increment(); + return; + } + + counter.Decrement(); + }); + + Assert.InRange(counter.Count, 0, LiveConnections); + + // Every unmatched decrement has completed, so the surviving connections + // disconnect cleanly and the counter must land exactly on zero — never + // below it, and a subsequent connect must be visible to the publisher. + int remaining = counter.Count; + for (int i = 0; i < remaining; i++) + { + counter.Decrement(); + } + + Assert.Equal(0, counter.Count); + Assert.Equal(1, counter.Increment()); + } + } + + /// Matched connect/disconnect pairs under concurrency settle back at zero. + [Fact] + public void IncrementAndDecrement_MatchedPairsUnderConcurrency_SettleAtZero() + { + DashboardSnapshotHubConnectionCounter counter = new(); + + Parallel.For(0, 64, _ => + { + for (int pass = 0; pass < 50; pass++) + { + counter.Increment(); + counter.Decrement(); + } + }); + + Assert.Equal(0, counter.Count); + } +} From 07b83561d1e341e286fae9b3c61a42eb73a1f81f Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:34:36 -0400 Subject: [PATCH 18/41] docs(events): correct the capture-under-replayLock rationale --- .../Sessions/SessionEventDistributor.cs | 107 ++++++++++++------ 1 file changed, 70 insertions(+), 37 deletions(-) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs index 441ce1c..9ea3ec4 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs @@ -111,9 +111,13 @@ public sealed class SessionEventDistributor : IAsyncDisposable // Copy-on-write fan-out snapshot of _subscribers.Values. Rebuilt (a whole new array) // inside the _lifecycleLock section of every register/unregister; never mutated in // place, so the pump can walk the array it captured with no lock and no allocation. - // Written with Volatile.Write / read with Volatile.Read so a reader on another core - // cannot observe a stale reference after the publishing store. See the type remarks - // for why fan-out must not touch ConcurrentDictionary.Values. + // Volatile.Write / Volatile.Read ORDER the access — they keep the publishing store from + // sinking past the lock release and keep the pump's read from being hoisted out of the + // fan-out loop. They do NOT promise freshness, and nothing here needs them to: a reader + // may legitimately observe the previous array, which IS the documented "late subscribers + // see events after they register" window. Where visibility must be guaranteed — the + // RegisterWithReplay handoff — it comes from the _replayLock edge, not from Volatile. + // See the type remarks for why fan-out must not touch ConcurrentDictionary.Values. private Subscriber[] _subscriberSnapshot = []; // Replay ring buffer. Appended on the pump thread and queried from arbitrary @@ -451,9 +455,10 @@ public sealed class SessionEventDistributor : IAsyncDisposable /// If the pump appended E before this critical section, E is in /// (when newer than /// ). The pump captured its subscriber array in - /// that same earlier section, so it cannot fan E into this not-yet-registered - /// channel. Even if it could, E's sequence is <= liveResumeSequence, so - /// the caller's live filter DROPS it — no duplicate either way. + /// that same earlier section, so it cannot also fan E into this + /// not-yet-registered channel — no duplicate. Belt and braces: even if it did, + /// E's sequence is <= liveResumeSequence and the caller's live filter + /// DROPS it. /// /// /// If the pump appends E after this critical section, E is NOT in the snapshot, @@ -464,6 +469,16 @@ public sealed class SessionEventDistributor : IAsyncDisposable /// /// /// + /// Capturing the fan-out array inside the append's _replayLock section is what + /// makes the first bullet's "cannot" hold. It is defense in depth rather than a + /// correctness fix: a capture taken after that lock released could not drop an event + /// either (the lock edge orders it), it could only produce the duplicate the live + /// filter already discards. Doing it under the lock costs nothing and stops + /// no-duplicate from depending on every caller remembering to apply the filter — + /// which callers MUST still do, since remains + /// part of this method's contract. + /// + /// /// Lock ordering: this is the only path that holds both _replayLock and /// _lifecycleLock; it always takes _replayLock first then /// _lifecycleLock. No other path acquires both, so there is no inversion. @@ -627,10 +642,13 @@ public sealed class SessionEventDistributor : IAsyncDisposable // Walk the captured copy-on-write array: no dictionary enumeration, no // per-event allocation. A subscriber registered after this capture misses // this event, which matches "late subscribers see events after they - // register". A subscriber unregistered after the capture is still written - // to — the identical window the previous ConcurrentDictionary.Values - // enumeration had (that property also materialized its list up front), so - // the race and its outcome are unchanged by the copy-on-write array. + // register". A subscriber UNREGISTERED after the capture is still written to, + // and TryWrite on its completed channel returns false — from here that is + // indistinguishable from a real overflow. The window predates the + // copy-on-write array (ConcurrentDictionary.Values materialized its list up + // front too) and its outcome is NOT benign, so telling a graceful unregister + // apart from a genuine overflow is OnSubscriberOverflow's job, not this + // loop's. foreach (Subscriber subscriber in subscribers) { // Non-blocking write: TryWrite never blocks the pump on a slow reader. @@ -854,14 +872,28 @@ public sealed class SessionEventDistributor : IAsyncDisposable } // Appends an event to the replay ring AND captures the fan-out array the pump will write it - // to, in ONE _replayLock section. The capture must share the append's critical section, not - // follow it: RegisterWithReplay snapshots the ring and registers under the same _replayLock, - // so mutual exclusion is what puts each event strictly before or strictly after a resume — - // "replayed, not fanned" or "fanned, not replayed", never neither. Capturing after the lock - // released would let a resume interleave between the append and the capture, replaying - // nothing for the event and fanning it to a stale array that omits the new subscriber: a - // silently dropped event. Returns the array; the pump fans OUTSIDE the lock so a slow - // reader can never stall replay. + // to, in ONE _replayLock section, making append+capture atomic with respect to + // RegisterWithReplay (which snapshots the ring and registers under that same lock). Each + // event therefore lands strictly on one side of a resume: replayed to that subscriber, or + // fanned to it live — never both. + // + // This is defense in depth, NOT a correctness fix; capturing after the lock released would + // also be correct. Monitor.Enter is an acquire (ECMA-335 I.12.6.5), so a later read cannot + // move above the append's lock acquisition, and a resume whose entire locked section + // (ring snapshot, registration, array republish) preceded the append is visible across that + // lock edge — no event can be silently dropped. What a late capture would allow is the + // benign case: an event both replayed AND written to the new subscriber's live channel, a + // duplicate the caller's liveResumeSequence filter discards. Capturing under the lock + // removes that duplicate at the source, so "no duplicate" no longer rests on the caller + // actually applying the filter — bought at zero cost, since the pump holds this lock anyway. + // + // Lock ordering: this helper only READS the already-published array, deliberately. The one + // permitted nesting in this type is RegisterWithReplay's _replayLock -> _lifecycleLock; + // every other path takes exactly one lock. Rebuilding here instead — an obvious-looking + // lock(_lifecycleLock) inside this _replayLock section — would drag the pump's hot path into + // that nesting and turn any future _lifecycleLock -> _replayLock path into a deadlock. + // + // Returns the array; the pump fans OUTSIDE the lock so a slow reader can never stall replay. private Subscriber[] AppendToReplayBufferAndCaptureSubscribers(MxEvent mxEvent) { lock (_replayLock) @@ -873,28 +905,29 @@ public sealed class SessionEventDistributor : IAsyncDisposable } // Capacity 0 disables retention: track the highest-seen sequence (so replay - // can still report a gap) but keep no events. - if (_replayBufferCapacity == 0) + // can still report a gap) but keep no events. The capture below still runs — + // retention being off says nothing about the fan-out set. + if (_replayBufferCapacity > 0) { - return Volatile.Read(ref _subscriberSnapshot); + // Append at the logical tail. When the ring is full the oldest entry is + // overwritten in place (its slot becomes the new tail) and the head advances, + // so the newest _replayBufferCapacity events are retained with no allocation. + ReplayEntry entry = new(mxEvent, _timeProvider.GetUtcNow()); + if (_replayCount < _replayBufferCapacity) + { + _replayBuffer[(_replayHead + _replayCount) % _replayBufferCapacity] = entry; + _replayCount++; + } + else + { + _replayBuffer[_replayHead] = entry; + _replayHead = (_replayHead + 1) % _replayBufferCapacity; + } + + EvictAged(); } - // Append at the logical tail. When the ring is full the oldest entry is - // overwritten in place (its slot becomes the new tail) and the head advances, - // so the newest _replayBufferCapacity events are retained with no allocation. - ReplayEntry entry = new(mxEvent, _timeProvider.GetUtcNow()); - if (_replayCount < _replayBufferCapacity) - { - _replayBuffer[(_replayHead + _replayCount) % _replayBufferCapacity] = entry; - _replayCount++; - } - else - { - _replayBuffer[_replayHead] = entry; - _replayHead = (_replayHead + 1) % _replayBufferCapacity; - } - - EvictAged(); + // Single capture point for both the retained and no-retention paths. return Volatile.Read(ref _subscriberSnapshot); } } From 7b2d04605e7b5892748242c613ae7e8c1a55d736 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:37:09 -0400 Subject: [PATCH 19/41] fix(audit): write-through on completed channel, poison-batch isolation, drain-fault fallback --- docs/DesignDecisions.md | 35 ++- .../Security/Audit/AuditDrainService.cs | 274 ++++++++++++++++++ .../Security/Audit/ChannelAuditWriter.cs | 217 +------------- .../Security/Audit/IAuditEventSink.cs | 38 +++ .../Audit/SqliteCanonicalAuditStore.cs | 47 +-- .../Security/Audit/ChannelAuditWriterTests.cs | 155 ++++++++-- 6 files changed, 492 insertions(+), 274 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Server/Security/Audit/AuditDrainService.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Server/Security/Audit/IAuditEventSink.cs diff --git a/docs/DesignDecisions.md b/docs/DesignDecisions.md index 2d1e479..2068527 100644 --- a/docs/DesignDecisions.md +++ b/docs/DesignDecisions.md @@ -244,14 +244,39 @@ authenticated call reads. The schema bootstrap now runs once, from the drain's ` When the channel is full the newest event is dropped and counted rather than blocking the producer: a stalled audit database must cost audit completeness, not gateway availability. Drops are logged once and reported in aggregate on each sweep. Shutdown drains what is buffered under a -2-second cap. Where no hosted service runs — the `apikey` admin CLI — the writer falls back to the -synchronous path, so audit is never buffered into a channel nobody drains. +2-second cap. + +Every other failure mode degrades to synchronous writes rather than to silent loss. The writer +falls back to the direct path whenever nothing is draining: before the drain attaches, after it +detaches, where no hosted service runs at all (the `apikey` admin CLI), and when the channel has +been completed — so no attach/detach sequence can leave producers filling a buffer with no reader. +If the drain loop itself dies it detaches the writer on the way out, which reverts every producer +to the direct path. A batch that will not commit is retried one event at a time, so an unwritable +row costs only itself instead of the up-to-63 good events sharing its transaction. + +**All** audit is channelled, including admin and CRUD records — dashboard key create/revoke/rotate, +session Close/Kill, and the library-forwarded API-key lifecycle entries. The alternative considered +was keeping those on the synchronous writer and channelling only high-volume denial audit. It was +rejected because a single dashboard key-create emits two records through two different seams (the +library's `create-key` via `IApiKeyAuditStore`, and the enriching `dashboard-create-key` via +`IAuditWriter`); splitting them across two durability regimes gives an auditor a per-producer +matrix to reason about instead of one rule. The residual exposure is explicit: **if the gateway +process dies between the enqueue and the batch commit, buffered audit events are lost.** The window +is bounded by drain latency — the drain wakes on every write and commits immediately, so it is +sub-millisecond under normal load — and it does not apply to the `apikey` CLI, which writes +synchronously. Audit is a best-effort record of what the gateway did, not a write-ahead log of what +it is about to do; a deployment that needs crash-durable admin audit should ship the events off-box +rather than rely on this table. `MxGateway:Security:AuditRetentionDays` (default 90, minimum 1) bounds the table: the drain sweeps at startup and hourly, deleting older rows. Retention cannot be configured off. The sweep compares -through SQLite's `datetime()` rather than on the stored ISO-8601 text, because timestamps written -from a non-UTC offset do not sort lexicographically against a UTC cutoff; a row whose timestamp -cannot be parsed yields NULL and is kept rather than deleted. +through SQLite's `datetime()` rather than on the stored ISO-8601 text. Text comparison is correct +only while every row is UTC-normalized — which the canonical model guarantees for rows written +through the store, but not for rows that entered the table any other way — and on a mixed-format +column it silently deletes live audit, because `2026-05-17T09:00:00-05:00` is two hours after a +`2026-05-17T12:00:00+00:00` cutoff yet sorts before it. Comparing instants is correct however the +text got there, and a timestamp `datetime()` cannot parse yields NULL, so undateable audit is kept +rather than swept. ## Authorization diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/AuditDrainService.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/AuditDrainService.cs new file mode 100644 index 0000000..d4519fb --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/AuditDrainService.cs @@ -0,0 +1,274 @@ +using ZB.MOM.WW.Audit; +using ZB.MOM.WW.MxGateway.Server.Configuration; + +namespace ZB.MOM.WW.MxGateway.Server.Security.Audit; + +/// +/// Drains onto the durable , +/// owns the one-time schema bootstrap, and sweeps audit rows past their retention window. +/// +/// +/// Batching is the point: up to buffered events are committed in a +/// single transaction, so a burst of constraint denials costs a handful of commits instead of +/// one per denied tag. The bootstrap runs here — before the writer starts enqueueing — so no +/// audit write ever pays a CREATE TABLE IF NOT EXISTS round-trip. +/// +/// Every failure mode ends in synchronous audit rather than silent loss: a batch that will not +/// commit is retried one event at a time so only the offending row is dropped, and a drain loop +/// that dies detaches the writer, which reverts every producer to the direct write path. +/// +/// +/// The channel writer whose buffered events are drained. +/// The durable sink events are committed to. +/// Security options carrying the audit retention window. +/// Clock used for the retention cutoff and sweep interval. +/// Logger for bootstrap, drain and sweep diagnostics. +public sealed class AuditDrainService( + ChannelAuditWriter writer, + IAuditEventSink sink, + SecurityOptions security, + TimeProvider timeProvider, + ILogger logger) : BackgroundService +{ + /// Maximum number of audit events committed in one transaction per drain pass. + public const int MaxBatchSize = 64; + + /// How often the retention sweep runs while the gateway is up. + public static readonly TimeSpan RetentionSweepInterval = TimeSpan.FromHours(1); + + /// Upper bound on how long shutdown waits for the remaining buffered events. + private static readonly TimeSpan ShutdownDrainCap = TimeSpan.FromSeconds(2); + + /// + /// Bootstraps the audit table, runs one retention sweep, then attaches the drain so the + /// writer switches from synchronous write-through to enqueueing. + /// + /// Token to observe for cancellation. + /// A task that represents the asynchronous operation. + public override async Task StartAsync(CancellationToken cancellationToken) + { + try + { + await sink.EnsureInitializedAsync(cancellationToken).ConfigureAwait(false); + await SweepRetentionAsync(cancellationToken).ConfigureAwait(false); + } + catch (Exception exception) + { + // Audit is best-effort: a bootstrap failure must not take the gateway down. The + // sink's own latch will retry the schema check on the first write. + logger.LogWarning(exception, "Audit store bootstrap failed; audit writes will retry the schema check."); + } + + writer.AttachDrain(); + + await base.StartAsync(cancellationToken).ConfigureAwait(false); + } + + /// + /// Detaches the drain (so late writes go straight to the sink) and gives the buffered + /// events a bounded window to reach the store. + /// + /// Token to observe for cancellation. + /// A task that represents the asynchronous operation. + public override async Task StopAsync(CancellationToken cancellationToken) + { + writer.DetachDrain(); + writer.CompleteWriting(); + + await base.StopAsync(cancellationToken).ConfigureAwait(false); + + using CancellationTokenSource drainCap = new(ShutdownDrainCap); + try + { + await DrainPendingAsync(drainCap.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + logger.LogWarning( + "Shutdown drain exceeded {CapSeconds}s; remaining buffered audit events were not persisted.", + ShutdownDrainCap.TotalSeconds); + } + } + + /// + /// Commits every event currently buffered, in batches of at most . + /// + /// Token to observe for cancellation. + /// The number of events persisted. + /// + /// The drain was cancelled — at shutdown this is the 2-second cap expiring, which the caller + /// reports as unpersisted audit rather than as a store fault. + /// + public async Task DrainPendingAsync(CancellationToken cancellationToken) + { + int persisted = 0; + List batch = new(MaxBatchSize); + + while (!cancellationToken.IsCancellationRequested) + { + batch.Clear(); + while (batch.Count < MaxBatchSize && writer.Reader.TryRead(out AuditEvent? auditEvent)) + { + batch.Add(auditEvent); + } + + if (batch.Count == 0) + { + break; + } + + try + { + await sink.InsertBatchAsync(batch, cancellationToken).ConfigureAwait(false); + persisted += batch.Count; + } + catch (OperationCanceledException) + { + // Cancellation is the shutdown cap, not a store fault: surface it so StopAsync + // reports unpersisted audit instead of misreporting it as a failed write. + throw; + } + catch (Exception exception) + { + logger.LogWarning( + exception, + "Failed to commit a batch of {Count} audit events; retrying them individually.", + batch.Count); + + persisted += await InsertIndividuallyAsync(batch, cancellationToken).ConfigureAwait(false); + } + } + + return persisted; + } + + /// + /// Deletes audit rows older than MxGateway:Security:AuditRetentionDays, and reports the + /// running total of audit events dropped by channel pressure since startup. + /// + /// Token to observe for cancellation. + /// A task that represents the asynchronous operation. + public async Task SweepRetentionAsync(CancellationToken cancellationToken) + { + DateTimeOffset cutoff = timeProvider.GetUtcNow() - TimeSpan.FromDays(security.AuditRetentionDays); + + try + { + int deleted = await sink.DeleteOlderThanAsync(cutoff, cancellationToken).ConfigureAwait(false); + if (deleted > 0) + { + logger.LogInformation( + "Audit retention sweep removed {Deleted} events older than {Cutoff:o} ({RetentionDays} days).", + deleted, + cutoff, + security.AuditRetentionDays); + } + } + catch (Exception exception) + { + logger.LogWarning(exception, "Audit retention sweep failed; it will be retried on the next interval."); + } + + long dropped = writer.DroppedCount; + if (dropped > 0) + { + logger.LogWarning( + "{Dropped} audit events have been dropped since startup because the audit channel was full.", + dropped); + } + } + + /// + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await Task.WhenAll( + DrainLoopAsync(stoppingToken), + RetentionLoopAsync(stoppingToken)).ConfigureAwait(false); + } + + // Re-inserts a failed batch one event at a time so a single unwritable row costs only itself + // rather than the up-to-MaxBatchSize good events that happened to share its transaction. + private async Task InsertIndividuallyAsync( + IReadOnlyList batch, + CancellationToken cancellationToken) + { + int persisted = 0; + + foreach (AuditEvent auditEvent in batch) + { + try + { + await sink.InsertAsync(auditEvent, cancellationToken).ConfigureAwait(false); + persisted++; + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception exception) + { + logger.LogWarning( + exception, + "Dropped audit event {EventId} (action {Action}); it could not be persisted individually.", + auditEvent.EventId, + auditEvent.Action); + } + } + + logger.LogWarning( + "Recovered {Persisted} of {Count} audit events from a failed batch.", + persisted, + batch.Count); + + return persisted; + } + + private async Task DrainLoopAsync(CancellationToken stoppingToken) + { + try + { + while (await writer.Reader.WaitToReadAsync(stoppingToken).ConfigureAwait(false)) + { + await DrainPendingAsync(stoppingToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + // Shutdown; StopAsync performs the final bounded drain. + } + catch (Exception exception) + { + // A dead drain loop would silently discard every later audit write, because producers + // keep enqueueing into a channel nobody reads. Detaching (below) reverts them to the + // synchronous path, so audit degrades in latency rather than disappearing. + logger.LogError(exception, "Audit drain loop failed; reverting to synchronous audit writes."); + } + finally + { + writer.DetachDrain(); + } + } + + private async Task RetentionLoopAsync(CancellationToken stoppingToken) + { + try + { + using PeriodicTimer timer = new(RetentionSweepInterval, timeProvider); + + while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)) + { + await SweepRetentionAsync(stoppingToken).ConfigureAwait(false); + } + } + catch (OperationCanceledException) + { + // Shutdown. + } + catch (Exception exception) + { + // Retention is unbounded growth if it stops: say so loudly rather than letting the + // audit table grow forever behind a silently dead timer loop. + logger.LogError(exception, "Audit retention loop failed; expired audit rows will no longer be swept."); + } + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/ChannelAuditWriter.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/ChannelAuditWriter.cs index 3b204cf..d85c5a2 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/ChannelAuditWriter.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/ChannelAuditWriter.cs @@ -1,6 +1,5 @@ using System.Threading.Channels; using ZB.MOM.WW.Audit; -using ZB.MOM.WW.MxGateway.Server.Configuration; namespace ZB.MOM.WW.MxGateway.Server.Security.Audit; @@ -27,6 +26,8 @@ namespace ZB.MOM.WW.MxGateway.Server.Security.Audit; /// straight through to . Enqueueing into a channel nobody will /// ever read would silently discard audit in the processes that have no hosted services — the /// apikey admin CLI and the DI-only tests — so those keep the original synchronous path. +/// The same fallback covers a completed channel, so no combination of attach/detach can leave +/// producers writing into a buffer that will never be read. /// /// public sealed class ChannelAuditWriter : IAuditWriter @@ -79,8 +80,9 @@ public sealed class ChannelAuditWriter : IAuditWriter public void AttachDrain() => Volatile.Write(ref _drainAttached, 1); /// - /// Marks the drain as no longer running, so writes revert to the synchronous path. Called - /// at shutdown so late audit is still persisted rather than buffered into a dead channel. + /// Marks the drain as no longer running, so writes revert to the synchronous path. Called at + /// shutdown, and whenever the drain loop dies, so late audit is still persisted rather than + /// buffered into a channel with no reader. /// public void DetachDrain() => Volatile.Write(ref _drainAttached, 0); @@ -100,11 +102,13 @@ public sealed class ChannelAuditWriter : IAuditWriter return _directWriter.WriteAsync(auditEvent, cancellationToken); } - // Under DropWrite a full channel still reports success (the discard surfaces through the - // itemDropped callback); a false here means the channel is completed, which is also a drop. + // Under DropWrite a full channel still reports SUCCESS — the discard surfaces through the + // itemDropped callback. So a false here does not mean "full", it means the channel has + // been completed and no drain will ever read it again (shutdown, or a re-attach onto a + // dead channel). Writing through is the only outcome that keeps the event. if (!_channel.Writer.TryWrite(auditEvent)) { - RecordDrop(auditEvent); + return _directWriter.WriteAsync(auditEvent, cancellationToken); } return Task.CompletedTask; @@ -130,204 +134,3 @@ public sealed class ChannelAuditWriter : IAuditWriter } } } - -/// -/// Drains onto the durable , -/// owns the one-time schema bootstrap, and sweeps audit rows past their retention window. -/// -/// -/// Batching is the point: up to buffered events are committed in a -/// single transaction, so a burst of constraint denials costs a handful of commits instead of -/// one per denied tag. The bootstrap runs here — before the writer starts enqueueing — so no -/// audit write ever pays a CREATE TABLE IF NOT EXISTS round-trip. -/// -/// The channel writer whose buffered events are drained. -/// The durable sink events are committed to. -/// Security options carrying the audit retention window. -/// Clock used for the retention cutoff and sweep interval. -/// Logger for bootstrap, drain and sweep diagnostics. -public sealed class AuditDrainService( - ChannelAuditWriter writer, - IAuditEventSink sink, - SecurityOptions security, - TimeProvider timeProvider, - ILogger logger) : BackgroundService -{ - /// Maximum number of audit events committed in one transaction per drain pass. - public const int MaxBatchSize = 64; - - /// How often the retention sweep runs while the gateway is up. - public static readonly TimeSpan RetentionSweepInterval = TimeSpan.FromHours(1); - - /// Upper bound on how long shutdown waits for the remaining buffered events. - private static readonly TimeSpan ShutdownDrainCap = TimeSpan.FromSeconds(2); - - /// - /// Bootstraps the audit table, runs one retention sweep, then attaches the drain so the - /// writer switches from synchronous write-through to enqueueing. - /// - /// Token to observe for cancellation. - /// A task that represents the asynchronous operation. - public override async Task StartAsync(CancellationToken cancellationToken) - { - try - { - await sink.EnsureInitializedAsync(cancellationToken).ConfigureAwait(false); - await SweepRetentionAsync(cancellationToken).ConfigureAwait(false); - } - catch (Exception exception) - { - // Audit is best-effort: a bootstrap failure must not take the gateway down. The - // sink's own latch will retry the schema check on the first write. - logger.LogWarning(exception, "Audit store bootstrap failed; audit writes will retry the schema check."); - } - - writer.AttachDrain(); - - await base.StartAsync(cancellationToken).ConfigureAwait(false); - } - - /// - /// Detaches the drain (so late writes go straight to the sink) and gives the buffered - /// events a bounded window to reach the store. - /// - /// Token to observe for cancellation. - /// A task that represents the asynchronous operation. - public override async Task StopAsync(CancellationToken cancellationToken) - { - writer.DetachDrain(); - writer.CompleteWriting(); - - await base.StopAsync(cancellationToken).ConfigureAwait(false); - - using CancellationTokenSource drainCap = new(ShutdownDrainCap); - try - { - await DrainPendingAsync(drainCap.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - logger.LogWarning( - "Shutdown drain exceeded {CapSeconds}s; remaining buffered audit events were not persisted.", - ShutdownDrainCap.TotalSeconds); - } - } - - /// - /// Commits every event currently buffered, in batches of at most . - /// - /// Token to observe for cancellation. - /// The number of events persisted. - public async Task DrainPendingAsync(CancellationToken cancellationToken) - { - int persisted = 0; - List batch = new(MaxBatchSize); - - while (!cancellationToken.IsCancellationRequested) - { - batch.Clear(); - while (batch.Count < MaxBatchSize && writer.Reader.TryRead(out AuditEvent? auditEvent)) - { - batch.Add(auditEvent); - } - - if (batch.Count == 0) - { - break; - } - - try - { - await sink.InsertBatchAsync(batch, cancellationToken).ConfigureAwait(false); - persisted += batch.Count; - } - catch (Exception exception) - { - // Best-effort: the batch is lost, but the drain keeps going so one poisoned - // batch cannot stall every later audit event behind it. - logger.LogWarning( - exception, - "Failed to persist a batch of {Count} audit events; audit writes are best-effort and were suppressed.", - batch.Count); - } - } - - return persisted; - } - - /// - /// Deletes audit rows older than MxGateway:Security:AuditRetentionDays, and reports - /// any audit events dropped by channel pressure since the last sweep. - /// - /// Token to observe for cancellation. - /// A task that represents the asynchronous operation. - public async Task SweepRetentionAsync(CancellationToken cancellationToken) - { - DateTimeOffset cutoff = timeProvider.GetUtcNow() - TimeSpan.FromDays(security.AuditRetentionDays); - - try - { - int deleted = await sink.DeleteOlderThanAsync(cutoff, cancellationToken).ConfigureAwait(false); - if (deleted > 0) - { - logger.LogInformation( - "Audit retention sweep removed {Deleted} events older than {Cutoff:o} ({RetentionDays} days).", - deleted, - cutoff, - security.AuditRetentionDays); - } - } - catch (Exception exception) - { - logger.LogWarning(exception, "Audit retention sweep failed; it will be retried on the next interval."); - } - - long dropped = writer.DroppedCount; - if (dropped > 0) - { - logger.LogWarning( - "{Dropped} audit events have been dropped since startup because the audit channel was full.", - dropped); - } - } - - /// - protected override async Task ExecuteAsync(CancellationToken stoppingToken) - { - await Task.WhenAll( - DrainLoopAsync(stoppingToken), - RetentionLoopAsync(stoppingToken)).ConfigureAwait(false); - } - - private async Task DrainLoopAsync(CancellationToken stoppingToken) - { - try - { - while (await writer.Reader.WaitToReadAsync(stoppingToken).ConfigureAwait(false)) - { - await DrainPendingAsync(stoppingToken).ConfigureAwait(false); - } - } - catch (OperationCanceledException) - { - // Shutdown; StopAsync performs the final bounded drain. - } - } - - private async Task RetentionLoopAsync(CancellationToken stoppingToken) - { - using PeriodicTimer timer = new(RetentionSweepInterval, timeProvider); - - try - { - while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false)) - { - await SweepRetentionAsync(stoppingToken).ConfigureAwait(false); - } - } - catch (OperationCanceledException) - { - // Shutdown. - } - } -} diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/IAuditEventSink.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/IAuditEventSink.cs new file mode 100644 index 0000000..7f12fb5 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/IAuditEventSink.cs @@ -0,0 +1,38 @@ +using ZB.MOM.WW.Audit; + +namespace ZB.MOM.WW.MxGateway.Server.Security.Audit; + +/// +/// Durable sink the audit pipeline persists canonical s through. +/// It exists so the write path () and the batching drain +/// () depend on the storage contract rather than on the +/// concrete . +/// +public interface IAuditEventSink +{ + /// + /// Bootstraps the backing storage. Called once at startup so no write path pays a schema + /// round-trip; implementations must be idempotent and safe to call concurrently. + /// + /// Token to observe for cancellation. + /// A task that represents the asynchronous operation. + Task EnsureInitializedAsync(CancellationToken cancellationToken); + + /// Persists a single canonical audit event. + /// The canonical event to persist. + /// Token to observe for cancellation. + /// A task that represents the asynchronous operation. + Task InsertAsync(AuditEvent auditEvent, CancellationToken cancellationToken); + + /// Persists a batch of canonical audit events as one unit of work. + /// The canonical events to persist. + /// Token to observe for cancellation. + /// A task that represents the asynchronous operation. + Task InsertBatchAsync(IReadOnlyList auditEvents, CancellationToken cancellationToken); + + /// Deletes every audit row that occurred strictly before . + /// The retention cutoff; rows older than this are removed. + /// Token to observe for cancellation. + /// The number of rows deleted. + Task DeleteOlderThanAsync(DateTimeOffset cutoffUtc, CancellationToken cancellationToken); +} diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/SqliteCanonicalAuditStore.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/SqliteCanonicalAuditStore.cs index db83c62..3ad7f91 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/SqliteCanonicalAuditStore.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/SqliteCanonicalAuditStore.cs @@ -5,41 +5,6 @@ using ZB.MOM.WW.Auth.ApiKeys.Sqlite; namespace ZB.MOM.WW.MxGateway.Server.Security.Audit; -/// -/// Durable sink the audit pipeline persists canonical s through. -/// It exists so the write path () and the batching drain -/// () depend on the storage contract rather than on the -/// concrete . -/// -public interface IAuditEventSink -{ - /// - /// Bootstraps the backing storage. Called once at startup so no write path pays a schema - /// round-trip; implementations must be idempotent and safe to call concurrently. - /// - /// Token to observe for cancellation. - /// A task that represents the asynchronous operation. - Task EnsureInitializedAsync(CancellationToken cancellationToken); - - /// Persists a single canonical audit event. - /// The canonical event to persist. - /// Token to observe for cancellation. - /// A task that represents the asynchronous operation. - Task InsertAsync(AuditEvent auditEvent, CancellationToken cancellationToken); - - /// Persists a batch of canonical audit events as one unit of work. - /// The canonical events to persist. - /// Token to observe for cancellation. - /// A task that represents the asynchronous operation. - Task InsertBatchAsync(IReadOnlyList auditEvents, CancellationToken cancellationToken); - - /// Deletes every audit row that occurred strictly before . - /// The retention cutoff; rows older than this are removed. - /// Token to observe for cancellation. - /// The number of rows deleted. - Task DeleteOlderThanAsync(DateTimeOffset cutoffUtc, CancellationToken cancellationToken); -} - /// /// MxGateway-owned, append-only SQLite store for canonical /// s. It writes to a NEW audit_event table in the @@ -178,10 +143,14 @@ public sealed class SqliteCanonicalAuditStore(AuthSqliteConnectionFactory connec /// /// /// The comparison goes through SQLite's datetime() rather than comparing the stored - /// ISO-8601 text directly: rows written from a non-UTC (the - /// library's CreatedUtc is caller-supplied) do not sort lexicographically against a - /// UTC cutoff. A row whose timestamp datetime() cannot parse yields NULL and is - /// therefore never deleted — audit that cannot be dated is kept, not swept. + /// ISO-8601 text directly. Text comparison is only correct while every row is UTC-normalized + /// ISO-8601 — which guarantees for rows written through + /// this store, but not for rows that entered the table any other way (a repair script, an + /// older schema, a future producer). On a mixed-format column a text comparison silently + /// deletes live audit: 2026-05-17T09:00:00-05:00 is two hours AFTER a + /// 2026-05-17T12:00:00+00:00 cutoff yet sorts before it. Comparing instants is correct + /// regardless of how the text got there, and anything datetime() cannot parse yields + /// NULL and is therefore never deleted — audit that cannot be dated is kept, not swept. /// public async Task DeleteOlderThanAsync(DateTimeOffset cutoffUtc, CancellationToken cancellationToken) { diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Security/Audit/ChannelAuditWriterTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Security/Audit/ChannelAuditWriterTests.cs index c96dce5..3b5a273 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Security/Audit/ChannelAuditWriterTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Security/Audit/ChannelAuditWriterTests.cs @@ -1,3 +1,4 @@ +using Microsoft.Data.Sqlite; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; using ZB.MOM.WW.Audit; @@ -10,10 +11,11 @@ namespace ZB.MOM.WW.MxGateway.Tests.Security.Audit; /// /// Tests the bounded, asynchronous audit path: (enqueue-only -/// once a drain is attached, drops rather than blocks when the channel is full) and -/// (batched drain, one-time table bootstrap, retention sweep). -/// The channel makes the already-documented best-effort audit contract explicit and bounded: -/// a partially denied bulk RPC no longer pays a SQLite round-trip per denied tag. +/// once a drain is attached, drops rather than blocks when the channel is full, writes through +/// whenever nothing is draining) and (batched drain, poison-batch +/// isolation, one-time table bootstrap, retention sweep). The channel makes the already-documented +/// best-effort audit contract explicit and bounded: a partially denied bulk RPC no longer pays a +/// SQLite round-trip per denied tag. /// public sealed class ChannelAuditWriterTests : IDisposable { @@ -71,6 +73,29 @@ public sealed class ChannelAuditWriterTests : IDisposable Assert.All(sink.BatchSizes, size => Assert.True(size <= AuditDrainService.MaxBatchSize)); } + /// + /// A batch that will not commit is retried one event at a time, so a single unwritable row + /// costs only itself instead of taking the up-to-63 good events sharing its transaction. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task DrainPendingAsync_WhenBatchFails_RetriesIndividuallyAndKeepsSurvivors() + { + CountingAuditSink sink = new() { PoisonAction = "poison" }; + (ChannelAuditWriter writer, AuditDrainService drain) = CreateWriterAndDrain(sink); + writer.AttachDrain(); + + await writer.WriteAsync(MakeEvent("good-1"), CancellationToken.None); + await writer.WriteAsync(MakeEvent("poison"), CancellationToken.None); + await writer.WriteAsync(MakeEvent("good-2"), CancellationToken.None); + + int persisted = await drain.DrainPendingAsync(CancellationToken.None); + + // The batch commit fails on the poison event; the per-event retry still lands both others. + Assert.Equal(2, persisted); + Assert.Equal(["good-1", "good-2"], sink.Events.Select(auditEvent => auditEvent.Action)); + } + /// /// When the bounded channel is full the write is dropped rather than blocking the caller, /// and the drop is counted. A stalled or slow audit database must never stall an RPC. @@ -89,13 +114,12 @@ public sealed class ChannelAuditWriterTests : IDisposable // Every call must complete synchronously: the channel never blocks a producer. Task write = writer.WriteAsync(MakeEvent($"denied-{index}"), CancellationToken.None); Assert.True(write.IsCompletedSuccessfully); + await write; } Assert.Equal(overflow, writer.DroppedCount); // The channel still holds exactly its capacity; nothing reached the sink (no drain ran). Assert.Equal(0, sink.InsertBatchCalls); - - await Task.CompletedTask; } /// @@ -114,6 +138,26 @@ public sealed class ChannelAuditWriterTests : IDisposable Assert.Equal("dashboard-create-key", Assert.Single(sink.Events).Action); } + /// + /// A completed channel has no future reader, so an attached writer must write through rather + /// than discard. This is the re-attach-after-shutdown footgun: enqueueing would lose 100% of + /// audit silently. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task WriteAsync_AfterChannelCompleted_WritesThroughInsteadOfDiscarding() + { + CountingAuditSink sink = new(); + (ChannelAuditWriter writer, _) = CreateWriterAndDrain(sink); + writer.AttachDrain(); + writer.CompleteWriting(); + + await writer.WriteAsync(MakeEvent("constraint-denied"), CancellationToken.None); + + Assert.Equal("constraint-denied", Assert.Single(sink.Events).Action); + Assert.Equal(0, writer.DroppedCount); + } + /// /// The retention sweep deletes rows older than AuditRetentionDays, measured from the /// injected clock, and runs once at startup after the one-time table bootstrap. @@ -137,29 +181,42 @@ public sealed class ChannelAuditWriterTests : IDisposable } /// - /// The real SQLite sink deletes only rows older than the cutoff, and its bootstrap latch - /// means the CREATE TABLE round-trip is not repeated on every subsequent write. + /// The sweep compares instants, not stored text, and never deletes a row it cannot date. /// /// A task that represents the asynchronous operation. [Fact] - public async Task SqliteStore_DeleteOlderThan_RemovesOnlyRowsBeyondTheRetentionWindow() + public async Task SqliteStore_DeleteOlderThan_ComparesInstantsAndKeepsUnparseableRows() { - SqliteCanonicalAuditStore store = CreateStore(); - DateTimeOffset now = new(2026, 8, 15, 12, 0, 0, TimeSpan.Zero); + (SqliteCanonicalAuditStore store, AuthSqliteConnectionFactory factory) = CreateStore(); + + DateTimeOffset cutoff = new(2026, 5, 17, 12, 0, 0, TimeSpan.Zero); await store.InsertBatchAsync( [ - MakeEvent("stale", now.AddDays(-120)), - // A non-UTC offset must still compare correctly against the cutoff. - MakeEvent("stale-offset", now.AddDays(-100).ToOffset(TimeSpan.FromHours(-5))), - MakeEvent("fresh", now.AddDays(-1)), + MakeEvent("swept", new DateTimeOffset(2026, 5, 17, 11, 0, 0, TimeSpan.Zero)), + MakeEvent("kept-fresh", new DateTimeOffset(2026, 5, 18, 0, 0, 0, TimeSpan.Zero)), ], CancellationToken.None); - int deleted = await store.DeleteOlderThanAsync(now.AddDays(-90), CancellationToken.None); + // Both discriminating rows are written as raw text, because AuditEvent.OccurredAtUtc + // normalizes to UTC and so cannot express them. They stand for audit that reached the + // table any other way — a repair script, an older schema, a future producer. + // + // 09:00 at -05:00 is 14:00 UTC, two hours AFTER the cutoff, yet its text sorts BEFORE + // "2026-05-17T12:00:00.0000000+00:00". A lexicographic sweep deletes it; comparing + // instants keeps it. + await InsertRawRowAsync(factory, "kept-offset", "2026-05-17T09:00:00.1234567-05:00"); - Assert.Equal(2, deleted); - Assert.Equal("fresh", Assert.Single(await store.ListRecentAsync(10, CancellationToken.None)).Action); + // Undateable audit is kept, never guessed at. This text sorts BELOW the cutoff, so a + // lexicographic sweep deletes it, while datetime() yields NULL and leaves it alone. + await InsertRawRowAsync(factory, "kept-unparseable", "0000-not-a-timestamp"); + + int deleted = await store.DeleteOlderThanAsync(cutoff, CancellationToken.None); + + Assert.Equal(1, deleted); + Assert.Equal( + ["kept-fresh", "kept-offset", "kept-unparseable"], + (await ListActionsAsync(factory)).OrderBy(action => action, StringComparer.Ordinal)); } private static (ChannelAuditWriter Writer, AuditDrainService Drain) CreateWriterAndDrain( @@ -179,11 +236,51 @@ public sealed class ChannelAuditWriterTests : IDisposable return (writer, drain); } - private SqliteCanonicalAuditStore CreateStore() + private (SqliteCanonicalAuditStore Store, AuthSqliteConnectionFactory Factory) CreateStore() { TempDatabaseDirectory directory = TempDatabaseDirectory.Create("mxgateway-channel-audit"); _tempDirectories.Add(directory); - return new SqliteCanonicalAuditStore(new AuthSqliteConnectionFactory(directory.DatabasePath())); + AuthSqliteConnectionFactory factory = new(directory.DatabasePath()); + return (new SqliteCanonicalAuditStore(factory), factory); + } + + // Writes an audit row whose occurred_at_utc bypasses the store's DateTimeOffset formatting, + // so the sweep can be shown to leave undateable audit alone. + private static async Task InsertRawRowAsync( + AuthSqliteConnectionFactory factory, + string action, + string occurredAtUtc) + { + await using SqliteConnection connection = await factory.OpenConnectionAsync(CancellationToken.None); + await using SqliteCommand command = connection.CreateCommand(); + command.CommandText = + """ + INSERT INTO audit_event (event_id, occurred_at_utc, actor, action, outcome) + VALUES ($event_id, $occurred_at_utc, 'operator01', $action, 'Denied'); + """; + command.Parameters.AddWithValue("$event_id", Guid.NewGuid().ToString()); + command.Parameters.AddWithValue("$occurred_at_utc", occurredAtUtc); + command.Parameters.AddWithValue("$action", action); + + await command.ExecuteNonQueryAsync(CancellationToken.None); + } + + // Reads actions straight from SQL: ListRecentAsync would throw on the unparseable timestamp. + private static async Task> ListActionsAsync(AuthSqliteConnectionFactory factory) + { + await using SqliteConnection connection = await factory.OpenConnectionAsync(CancellationToken.None); + await using SqliteCommand command = connection.CreateCommand(); + command.CommandText = "SELECT action FROM audit_event;"; + + List actions = []; + + await using SqliteDataReader reader = await command.ExecuteReaderAsync(CancellationToken.None); + while (await reader.ReadAsync(CancellationToken.None)) + { + actions.Add(reader.GetString(0)); + } + + return actions; } private static AuditEvent MakeEvent(string action, DateTimeOffset? occurredAtUtc = null) => new() @@ -209,22 +306,27 @@ public sealed class ChannelAuditWriterTests : IDisposable /// /// In-memory that records every call, so a test can prove the - /// write path did NOT touch the sink and that the drain batched what it did write. + /// write path did NOT touch the sink and that the drain batched what it did write. Setting + /// makes any write containing that action fail, modelling a row + /// the store refuses. /// private sealed class CountingAuditSink : IAuditEventSink { private readonly Lock _gate = new(); + /// Gets the action whose presence makes a write fail, or null to accept everything. + public string? PoisonAction { get; init; } + /// Gets the events handed to the sink, in the order they were written. public List Events { get; } = []; - /// Gets the size of each batch the sink was asked to insert. + /// Gets the size of each batch the sink accepted. public List BatchSizes { get; } = []; /// Gets the cutoffs the sink was asked to delete below. public List DeleteCutoffs { get; } = []; - /// Gets the number of calls. + /// Gets the number of accepted calls. public int InsertBatchCalls => BatchSizes.Count; /// Gets the number of calls. @@ -248,6 +350,13 @@ public sealed class ChannelAuditWriterTests : IDisposable /// public Task InsertBatchAsync(IReadOnlyList auditEvents, CancellationToken cancellationToken) { + if (PoisonAction is not null + && auditEvents.Any(auditEvent => auditEvent.Action == PoisonAction)) + { + return Task.FromException( + new InvalidOperationException($"Refused a write of {auditEvents.Count} audit events.")); + } + lock (_gate) { BatchSizes.Add(auditEvents.Count); From 1742e38c103812eaf5343082b1248e5e15eac199 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:38:20 -0400 Subject: [PATCH 20/41] fix(events): graceful unregister no longer masquerades as overflow under FailFast --- .../Sessions/SessionEventDistributor.cs | 55 +++++++---- .../Sessions/SessionEventDistributorTests.cs | 91 +++++++++++++++++++ 2 files changed, 129 insertions(+), 17 deletions(-) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs index 9ea3ec4..ba97a03 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs @@ -683,14 +683,35 @@ public sealed class SessionEventDistributor : IAsyncDisposable } // Applies the per-subscriber backpressure policy when a subscriber's bounded channel is - // full. Runs on the pump thread. The offending subscriber is ALWAYS disconnected with an - // overflow fault and unregistered, so it can never wedge the pump again; the overflow - // handler decides the observable side effects (overflow metric, and — for legacy + // full — or, indistinguishably from the pump's side, already completed. A subscriber that + // really overflowed is ALWAYS disconnected with an overflow fault and unregistered, so it + // can never wedge the pump again; one that merely unregistered itself is dropped silently + // (see the discriminator below). Runs on the pump thread. The overflow handler decides the + // observable side effects (overflow metric, and — for legacy // single-subscriber FailFast — faulting the owning session). Multi-subscriber FailFast // intentionally degrades to a plain disconnect (see SubscriberOverflowHandler docs): one // slow consumer must not fault a session shared by other healthy subscribers. private void OnSubscriberOverflow(Subscriber subscriber, ulong workerSequence) { + // Claim the disconnect FIRST, because a false TryWrite is ambiguous. It means either + // "channel full" (a genuine overflow) or "channel already completed" — which happens + // when the subscriber unregistered after the pump captured the fan-out array and is + // therefore a GRACEFUL close, not backpressure. RemoveSubscriber separates the two: + // every path that completes a channel during fan-out (lease disposal via Unregister, + // and this method) removes the subscriber from the set BEFORE completing it, so a + // completed channel implies the subscriber is already gone and RemoveSubscriber + // returns false. (CompleteAllSubscribers completes without removing, but only after + // the pump has left its loop, so it cannot be observed here.) + // + // Bailing out on false is what keeps a normal stream ending mid-traffic from emitting + // a bogus EventQueueOverflow metric and — under the default single-subscriber FailFast + // policy — faulting the whole session. Winning the removal also guarantees the side + // effects below run exactly once per subscriber. + if (!RemoveSubscriber(subscriber)) + { + return; + } + // Decide whether FailFast may fault the whole session for this overflow. This is the // "isOnlySubscriber" signal the legacy single-subscriber FailFast path keys on. bool isOnlySubscriber = !subscriber.IsInternal && _singleSubscriberMode; @@ -717,20 +738,15 @@ public sealed class SessionEventDistributor : IAsyncDisposable subscriber.Id); } - // Disconnect ONLY this subscriber: complete its channel with the overflow fault and - // remove it from the fan-out set. Its gRPC reader's MoveNextAsync then throws the - // SessionManagerException, which EventStreamService surfaces to the client exactly as - // the pre-epic per-RPC overflow did. The pump and every other subscriber are untouched. - // - // The removal takes _lifecycleLock because it must republish the fan-out snapshot; the - // pump holds no other lock here (fan-out runs outside _replayLock), so this cannot - // invert the _replayLock-then-_lifecycleLock order RegisterWithReplay uses. - if (RemoveSubscriber(subscriber)) - { - subscriber.Channel.Writer.TryComplete(new SessionManagerException( - SessionManagerErrorCode.EventQueueOverflow, - $"Session {_sessionId} event stream queue overflowed.")); - } + // Disconnect ONLY this subscriber: it is already out of the fan-out set (removed above), + // so complete its channel with the overflow fault. Its gRPC reader's MoveNextAsync then + // throws the SessionManagerException, which EventStreamService surfaces to the client + // exactly as the pre-epic per-RPC overflow did. The pump and every other subscriber are + // untouched. This runs even when the handler above threw — the subscriber must never be + // left attached with an un-completed channel. + subscriber.Channel.Writer.TryComplete(new SessionManagerException( + SessionManagerErrorCode.EventQueueOverflow, + $"Session {_sessionId} event stream queue overflowed.")); } private void CompleteAllSubscribers(Exception? error) @@ -765,6 +781,11 @@ public sealed class SessionEventDistributor : IAsyncDisposable // Returns true only for the caller that actually removed it, so the channel is completed // exactly once however many disposal/overflow paths race. Completing the channel is left to // that caller and happens OUTSIDE the lock: this lock guards set membership only. + // + // Remove-then-complete (never the reverse) is load-bearing, not incidental: it is what lets + // OnSubscriberOverflow read a false return as "this subscriber unregistered gracefully" + // rather than "this subscriber overflowed". Completing before removing would resurrect the + // spurious-session-fault bug. private bool RemoveSubscriber(Subscriber subscriber) { lock (_lifecycleLock) diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs index 75a74ad..1338a71 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using System.Threading.Channels; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; @@ -1114,6 +1115,96 @@ public sealed class SessionEventDistributorTests Assert.Equal(1, distributor.SubscriberCount); } + /// + /// Regression: a subscriber that unregisters (lease disposed) after the pump captured the + /// fan-out array is still written to, and TryWrite on its now-completed channel + /// returns false — the same signal a full channel gives. Treating that as backpressure + /// emitted a bogus EventQueueOverflow metric and, under the default + /// single-subscriber FailFast policy, faulted the whole session: a stream ending normally + /// during traffic could kill the session. The overflow path must claim the removal first + /// and bail out when the subscriber is already gone. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task GracefulUnregisterDuringFanOut_DoesNotReportOverflow_OrFaultTheSession() + { + Channel source = Channel.CreateUnbounded(); + + // Records every overflow-handler invocation. isInternal distinguishes the deliberate + // overflow (the internal subscriber below) from the graceful unregister under test. + ConcurrentQueue<(bool IsOnlySubscriber, bool IsInternal)> invocations = new(); + IEventSubscriberLease? gracefulLease = null; + int disposedGracefulLease = 0; + + await using SessionEventDistributor distributor = new( + "session-graceful-unregister", + ct => source.Reader.ReadAllAsync(ct), + subscriberQueueCapacity: 1, + replayBufferCapacity: 0, + replayRetentionSeconds: 0, + NullLogger.Instance, + TimeProvider.System, + (isOnlySubscriber, isInternal) => + { + invocations.Enqueue((isOnlySubscriber, isInternal)); + + // The seam that makes the race deterministic: this handler runs ON the pump + // thread, part-way through fanning one event to the array it already captured. + // Disposing the graceful lease here unregisters and completes that subscriber + // in exactly the window the fix targets — after the capture, before the pump + // reaches its TryWrite. Only on the first invocation, so a genuine repeat + // overflow cannot re-trigger it. + if (Interlocked.Exchange(ref disposedGracefulLease, 1) == 0) + { + gracefulLease!.Dispose(); + } + }, + singleSubscriberMode: true); + + await distributor.StartAsync(CancellationToken.None); + + // Registered FIRST so it precedes the graceful subscriber in the captured fan-out array, + // putting the graceful subscriber's TryWrite after this one's overflow handler. Internal + // so its own (expected) overflow reports isOnlySubscriber == false and can never fault + // the session by itself. Never read from, so its capacity-1 channel fills immediately. + using IEventSubscriberLease overflowing = distributor.Register(isInternal: true); + + // External subscriber that will unregister gracefully mid-fan-out. Under the old + // behavior its completed-channel TryWrite reported isOnlySubscriber == true, which is + // precisely the legacy FailFast "fault the session" signal. + gracefulLease = distributor.Register(); + + // Event 1 fills the internal subscriber's channel and is drained from the graceful one, + // so on event 2 the internal subscriber overflows while the graceful one has room — + // whichever order the array happens to hold, only the internal subscriber overflows. + source.Writer.TryWrite(Event(1)); + MxEvent first = await ReadOneAsync(gracefulLease.Reader); + Assert.Equal(1ul, first.WorkerSequence); + + // Event 2: the internal subscriber overflows, the handler disposes the graceful lease, + // and the pump then writes event 2 to that already-completed channel. + source.Writer.TryWrite(Event(2)); + + // The graceful subscriber's channel must complete cleanly — no EventQueueOverflow fault. + await AssertCompletedAsync(gracefulLease.Reader); + + // The pump survives and keeps serving a freshly-attached subscriber. + using IEventSubscriberLease later = distributor.Register(); + source.Writer.TryWrite(Event(3)); + Assert.Equal(3ul, (await ReadOneAsync(later.Reader)).WorkerSequence); + + // Guards against a vacuous pass: the deliberate internal overflow must actually have + // fired, since that handler call is the seam that disposes the lease mid-fan-out. + Assert.NotEmpty(invocations); + Assert.Equal(1, Volatile.Read(ref disposedGracefulLease)); + + // The deliberate internal overflow is expected; the graceful unregister must NOT have + // produced an overflow report of its own. An isOnlySubscriber == true invocation is the + // exact signal that would have faulted the session. + Assert.All(invocations, invocation => Assert.True(invocation.IsInternal)); + Assert.DoesNotContain(invocations, invocation => invocation.IsOnlySubscriber); + } + private static SessionEventDistributor CreateDistributor(ChannelReader source) => CreateDistributor(source, replayBufferCapacity: 1024, replayRetentionSeconds: 300); From a1a38b5538ff57007124233b8ce5ab9e1ab1901d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:41:38 -0400 Subject: [PATCH 21/41] fix(ipc): cancellation-priority timeout classification; structurally-enforced no-throw cancel send --- docs/GatewayProcessDesign.md | 12 +- .../Workers/WorkerClient.cs | 157 ++++++++++++------ .../Gateway/Workers/WorkerClientTests.cs | 20 ++- 3 files changed, 130 insertions(+), 59 deletions(-) diff --git a/docs/GatewayProcessDesign.md b/docs/GatewayProcessDesign.md index 98ce742..194800b 100644 --- a/docs/GatewayProcessDesign.md +++ b/docs/GatewayProcessDesign.md @@ -604,8 +604,16 @@ running on it, and the rule above still holds. A command whose envelope is still in the gateway's outbound queue needs no special handling: the queue is FIFO, so the worker reads the command and then its cancel and drops it before execution. -Failing to send the cancel is logged at debug and never replaces the -`CommandTimeout` the caller is owed. +Cancels ride the same outbound channel as commands, whose capacity is +`MaxPendingCommands + 4`: the reserve above the pending-command limit is what +absorbs them, so a burst of timeouts stays bounded and cannot deadlock the +enqueue path. Failing to send the cancel is logged at debug and never replaces +the `CommandTimeout` the caller is owed. + +Cancellation outranks the deadline. When a caller's token is canceled around the +same time the timeout fires, the command is reported as canceled +(`GatewayShutdown`, `OperationCanceledException`), not as `CommandTimeout`, and +no cancel is forwarded. ## Fault Model diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs index b0c6d84..bff47b1 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs @@ -256,26 +256,20 @@ public sealed class WorkerClient : IWorkerClient // continuation — the old shape allocated a linked CancellationTokenSource, its // registration, a delay Task, and the WhenAny Task on every invoke, and left the delay // Task rooted until the cancel completed. WaitAsync raises TimeoutException for the - // deadline and OperationCanceledException for the caller's token, which is exactly the - // two-way split the old completedTask/IsCancellationRequested test made by hand; the - // error codes and messages below are unchanged. + // deadline and OperationCanceledException for the caller's token — but unlike the old + // wait it races the two and reports whichever fired first, whereas the old code inspected + // cancellationToken.IsCancellationRequested BEFORE classifying a won delay as a timeout. + // The filter on the CommandTimeout clause restores that priority: a token canceled around + // the deadline is still classified as cancellation, never as CommandTimeout. Error codes + // and messages are unchanged. try { return await pendingCommand.Task.WaitAsync(timeout, cancellationToken).ConfigureAwait(false); } - catch (OperationCanceledException) - { - RemovePendingCommandAsFailed( - correlationId, - pendingCommand, - WorkerClientErrorCode.GatewayShutdown, - "Command wait was canceled."); - throw; - } - catch (TimeoutException) + catch (TimeoutException) when (!cancellationToken.IsCancellationRequested) { string timeoutMessage = $"Worker command {method} timed out after {timeout}."; - RemovePendingCommandAsFailed( + bool removed = RemovePendingCommandAsFailed( correlationId, pendingCommand, WorkerClientErrorCode.CommandTimeout, @@ -284,14 +278,45 @@ public sealed class WorkerClient : IWorkerClient // The gateway has stopped waiting, but the worker has not stopped working: the // correlation is still on its single STA queue and would execute (or keep executing) // regardless. Tell it, so a queued-but-not-started command is dropped instead of - // occupying the STA behind a caller that is already gone. Best-effort by design — a - // failure here must never replace the timeout the caller is owed. - TrySendCancelForTimedOutCommand(correlationId, method, timeout); + // occupying the STA behind a caller that is already gone. Gated on the removal so a + // reply that won the race — the pending entry is already gone and the caller is about + // to see it — never has a cancel chase it. Best-effort by design; the send cannot + // throw, so it can never replace the timeout the caller is owed. + if (removed) + { + TrySendCancelForTimedOutCommand(correlationId, method, timeout); + } throw new WorkerClientException( WorkerClientErrorCode.CommandTimeout, timeoutMessage); } + catch (OperationCanceledException) + { + RemovePendingCommandAsFailed( + correlationId, + pendingCommand, + WorkerClientErrorCode.GatewayShutdown, + "Command wait was canceled."); + + // WaitAsync surfaces TaskCanceledException; throwing through the token keeps the + // exception the caller observes exactly what the hand-rolled wait produced. + cancellationToken.ThrowIfCancellationRequested(); + throw; + } + catch (TimeoutException) + { + // The deadline and the caller's cancellation raced and WaitAsync picked the timer. + // The old wait classified this as cancellation, so this clause — reached only when + // the filter above saw a canceled token — reproduces that treatment exactly. + RemovePendingCommandAsFailed( + correlationId, + pendingCommand, + WorkerClientErrorCode.GatewayShutdown, + "Command wait was canceled."); + cancellationToken.ThrowIfCancellationRequested(); + throw; + } } catch { @@ -755,7 +780,12 @@ public sealed class WorkerClient : IWorkerClient /// The pending command. /// Error code. /// Error message. - private void RemovePendingCommandAsFailed( + /// + /// true when this call removed the pending entry and owns the failure; false when + /// the entry was already gone — a reply, fault, or shutdown got there first, so the caller must + /// not take any further action on behalf of that correlation. + /// + private bool RemovePendingCommandAsFailed( string correlationId, PendingCommand pendingCommand, WorkerClientErrorCode errorCode, @@ -763,13 +793,14 @@ public sealed class WorkerClient : IWorkerClient { if (!_pendingCommands.TryRemove(correlationId, out _)) { - return; + return false; } ReleasePendingCommandSlot(); TimeSpan duration = _timeProvider.GetElapsedTime(pendingCommand.StartTimestamp); _metrics?.CommandFailed(pendingCommand.Method, errorCode.ToString(), duration); pendingCommand.SetException(new WorkerClientException(errorCode, message)); + return true; } /// @@ -786,14 +817,17 @@ public sealed class WorkerClient : IWorkerClient /// /// /// - /// Every failure is swallowed and logged at debug: the caller is on the throw path for the - /// timeout, and losing the cancel costs the worker one wasted command, whereas letting an - /// exception escape would replace the the - /// caller is owed. TryWrite is the fast path and cannot throw for a closed or completed - /// channel; only when the outbound channel is momentarily at its bound is a detached task - /// started, and that task catches everything it can observe — including the - /// from _stopCts if the client is disposed - /// underneath it — so nothing is left unobserved. + /// The whole body sits under one catch-all that debug-logs, because the guarantee this method + /// owes its caller is structural, not incidental: the caller is on the throw path for the + /// timeout, so anything escaping here — an envelope that failed to build, a TryWrite + /// against a disposed channel, a scheduler refusing the detached task — would replace the + /// the caller is owed with an unrelated + /// exception. Losing the cancel costs the worker one wasted command; losing the timeout + /// misreports why the call failed. The detached task carries its own handler for the same + /// reason: its failures (including the from + /// _stopCts if the client is disposed underneath it) happen after this method returns + /// and would otherwise be unobserved. It is deliberately not tracked or awaited — it holds no + /// resource the shutdown path needs back, and the outbound channel is completed on close. /// /// Correlation id of the command that timed out. /// Command method name, for the cancel reason and diagnostics. @@ -803,35 +837,54 @@ public sealed class WorkerClient : IWorkerClient string method, TimeSpan timeout) { - WorkerEnvelope cancelEnvelope = CreateEnvelope( - correlationId, - envelope => envelope.WorkerCancel = new WorkerCancel + try + { + WorkerEnvelope cancelEnvelope = CreateEnvelope( + correlationId, + envelope => envelope.WorkerCancel = new WorkerCancel + { + Reason = $"gateway command timeout after {timeout}", + }); + + if (_outboundEnvelopes.Writer.TryWrite(cancelEnvelope)) { - Reason = $"gateway command timeout after {timeout}", + return; + } + + _ = Task.Run(async () => + { + try + { + await EnqueueAsync(cancelEnvelope, _stopCts.Token).ConfigureAwait(false); + } + catch (Exception exception) + { + LogCancelNotForwarded(exception, method, correlationId); + } }); - - if (_outboundEnvelopes.Writer.TryWrite(cancelEnvelope)) - { - return; } - - _ = Task.Run(async () => + catch (Exception exception) { - try - { - await EnqueueAsync(cancelEnvelope, _stopCts.Token).ConfigureAwait(false); - } - catch (Exception exception) - { - _logger.LogDebug( - exception, - "Could not forward a cancel for timed-out worker command {Method} on session {SessionId} " - + "and correlation {CorrelationId}.", - method, - SessionId, - correlationId); - } - }); + LogCancelNotForwarded(exception, method, correlationId); + } + } + + /// Records a cancel that could not be forwarded for a timed-out command. + /// The failure that stopped the cancel from being sent. + /// Command method name of the timed-out command. + /// Correlation id of the timed-out command. + private void LogCancelNotForwarded( + Exception exception, + string method, + string correlationId) + { + _logger.LogDebug( + exception, + "Could not forward a cancel for timed-out worker command {Method} on session {SessionId} " + + "and correlation {CorrelationId}.", + method, + SessionId, + correlationId); } /// Reads and validates a handshake envelope. diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs index b4f34dd..807249d 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs @@ -163,7 +163,7 @@ public sealed class WorkerClientTests // The timeout also emits a WorkerCancel for the abandoned correlation (GWC-31), which sits // ahead of the second command on the FIFO pipe; skip it rather than mistaking it for the // command this assertion is about. - WorkerEnvelope secondCommand = await ReadNextCommandAsync(pipePair); + WorkerEnvelope secondCommand = await ReadNextCommandAsync(pipePair, timedOutCommand.CorrelationId); await pipePair.WriteAsync( CreateCommandReplyEnvelope(secondCommand.CorrelationId, MxCommandKind.GetWorkerInfo)); @@ -188,8 +188,10 @@ public sealed class WorkerClientTests await using WorkerClient client = CreateClient(pipePair); await CompleteHandshakeAsync(client, pipePair); + // Advise rather than a control command: control commands bypass the worker's STA queue, so a + // data command is the case the cancel actually exists for. Task invokeTask = client.InvokeAsync( - CreateCommand(MxCommandKind.Ping), + CreateCommand(MxCommandKind.Advise), TimeSpan.FromMilliseconds(50), CancellationToken.None); @@ -1076,12 +1078,17 @@ public sealed class WorkerClientTests } /// - /// Reads gateway envelopes until a WorkerCommand arrives, skipping the control envelopes - /// (such as the WorkerCancel a command timeout emits) that may precede it on the FIFO pipe. + /// Reads gateway envelopes until a WorkerCommand arrives, skipping the WorkerCancel + /// a command timeout emits for . Anything else on the pipe + /// fails the test rather than being skipped: the point of the skip is to tolerate exactly the one + /// known interleaving, not to make the assertion blind to unexpected gateway traffic. /// /// The connected pipe pair whose worker side is read. + /// Correlation id of the timed-out command whose cancel is expected. /// The next command envelope written by the gateway. - private static async Task ReadNextCommandAsync(PipePair pipePair) + private static async Task ReadNextCommandAsync( + PipePair pipePair, + string canceledCorrelationId) { while (true) { @@ -1090,6 +1097,9 @@ public sealed class WorkerClientTests { return envelope; } + + Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerCancel, envelope.BodyCase); + Assert.Equal(canceledCorrelationId, envelope.CorrelationId); } } From 0bc13b5292bbd12844338ca0947fc823f40e6627 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 12:43:30 -0400 Subject: [PATCH 22/41] perf(sessions): bounded-parallel teardown in lease sweep and shutdown --- docs/Sessions.md | 29 ++-- .../Sessions/SessionManager.cs | 117 ++++++++++--- .../Gateway/Sessions/SessionManagerTests.cs | 157 ++++++++++++++++++ 3 files changed, 267 insertions(+), 36 deletions(-) diff --git a/docs/Sessions.md b/docs/Sessions.md index 4ebb1ca..8502c96 100644 --- a/docs/Sessions.md +++ b/docs/Sessions.md @@ -207,6 +207,12 @@ The repair transitions the monitor's reconcile broadcasts on the alarm feed (Rai Sessions open with `MxGateway:Sessions:DefaultLeaseSeconds` (default 1800) added to the open timestamp. Unary client activity refreshes the lease by the same duration. `ExtendLease` and `IsLeaseExpired` cooperate with `SessionManager.CloseExpiredLeasesAsync`, which iterates a registry snapshot and closes any session whose lease has expired with `LeaseExpiredReason`. `SessionLeaseMonitorHostedService` runs that sweep every `MxGateway:Sessions:LeaseSweepIntervalSeconds` seconds (default 30). +#### Teardown parallelism + +A sweep pass is two phases. *Selection* stays a single sequential pass over the snapshot, because that is what gives the precedence rule (lease-expiry, then faulted, then detach-grace) and the `TryBeginCloseIfExpired` TOCTOU re-check their meaning. *Closing* then runs over the already-selected set with `Parallel.ForEachAsync` at `MaxParallelSessionCloses` (4). Each close is bounded by `MxGateway:Worker:ShutdownTimeoutSeconds` (default 10), so a one-at-a-time sweep lets a few hung workers serialize reaping and starve session slots for the rest. Parallel closing is safe because `TryBeginCloseIfExpired` already flipped each selected session to `Closing` under its own lock — that idempotent begin-close is the per-session exclusivity invariant, so no two teardowns can ever run against one session. The degree is bounded rather than unlimited because every concurrent close is one x86 worker process being shut down or killed; the fan-out exists to hide a few hung workers, not to tear the whole registry down at once. + +A close that throws no longer abandons the rest of the selected set: the sweep captures the first failure, finishes the remaining closes, then rethrows it so `SessionLeaseMonitorHostedService` still logs the sweep failure as before. `ShutdownAsync` drains sessions with the same bounded fan-out, keeping its existing per-session catch → `KillWorkerAsync` fallback. + #### Detach-grace retention `MxGateway:Sessions:DetachGraceSeconds` (default 30) is a bounded retention window kept after a session's *last external (gRPC) event-stream subscriber* drops, so a client can reconnect to the same session instead of having it torn down on the first stream disconnect. While the window is open the session stays `Ready` and fully usable — worker commands continue to work and a reconnecting subscriber re-attaches normally. Because retention is keyed on the *external* subscriber count (`_activeEventSubscriberCount`), and the gateway-owned internal dashboard mirror registers directly on the distributor with `isInternal: true` and is therefore *not* counted, a session whose only remaining subscriber is the dashboard mirror still enters detach-grace. @@ -276,16 +282,21 @@ If both graceful shutdown and the kill fall-back fail, the original and kill exc ## Shutdown Coordination -`SessionShutdownHostedService.StopAsync` calls `SessionManager.ShutdownAsync`, which closes every registered session with `GatewayShutdownReason`. The shutdown loop catches per-session exceptions, calls `KillWorker`, and removes the session so that one stuck worker cannot block the rest of the host: +`SessionShutdownHostedService.StopAsync` calls `SessionManager.ShutdownAsync`, which closes every registered session with `GatewayShutdownReason`. Sessions are drained with the same bounded fan-out the lease sweep uses (`MaxParallelSessionCloses`), because a one-at-a-time drain of a full registry at a worst-case worker shutdown timeout each outruns any host stop-timeout and leaves the tail to the orphan killer. Each iteration catches its own exceptions, calls `KillWorkerAsync`, and removes the session so that one stuck worker cannot block the rest of the host: ```csharp -public async Task ShutdownAsync(CancellationToken cancellationToken) -{ - foreach (GatewaySession session in _registry.Snapshot()) +await Parallel.ForEachAsync( + _registry.Snapshot(), + new ParallelOptions + { + MaxDegreeOfParallelism = MaxParallelSessionCloses, + CancellationToken = cancellationToken, + }, + async (session, closeToken) => { try { - await CloseSessionCoreAsync(session, GatewayShutdownReason, cancellationToken).ConfigureAwait(false); + await CloseSessionCoreAsync(session, GatewayShutdownReason, closeToken).ConfigureAwait(false); } catch (Exception exception) { @@ -295,15 +306,13 @@ public async Task ShutdownAsync(CancellationToken cancellationToken) session.SessionId); if (_registry.TryGet(session.SessionId, out _)) { - session.KillWorker(GatewayShutdownReason); - await RemoveSessionAsync(session).ConfigureAwait(false); + await KillWorkerAsync(session.SessionId, GatewayShutdownReason, closeToken).ConfigureAwait(false); } } - } -} + }).ConfigureAwait(false); ``` -Iterating over `Snapshot` rather than the live dictionary lets `RemoveSessionAsync` mutate the registry inside the loop without throwing. +Iterating over `Snapshot` rather than the live dictionary lets `RemoveSessionAsync` mutate the registry from inside the loop without throwing, and gives the parallel drain a stable, already-materialized source. ## Dependency Injection diff --git a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs index f5c9a6d..ecff690 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs @@ -1,4 +1,5 @@ using System.Diagnostics.CodeAnalysis; +using System.Runtime.ExceptionServices; using System.Security.Cryptography; using Google.Protobuf.WellKnownTypes; using Microsoft.Extensions.Logging; @@ -20,6 +21,12 @@ public sealed class SessionManager : ISessionManager public const string DetachGraceExpiredReason = "detach-grace-expired"; public const string FaultedReason = "faulted-reaped"; + // Bounded so a mass expiry (or a host stop with a full registry) cannot stampede + // worker-process teardown: every concurrent close is one x86 worker being shut down or + // killed, and the point of the fan-out is to hide a few hung workers, not to tear the whole + // registry down at once. + private const int MaxParallelSessionCloses = 4; + private readonly ISessionRegistry _registry; private readonly ISessionWorkerClientFactory _workerClientFactory; private readonly GatewayMetrics _metrics; @@ -252,7 +259,10 @@ public sealed class SessionManager : ISessionManager DateTimeOffset now, CancellationToken cancellationToken) { - int closedCount = 0; + // Selection phase — deliberately sequential. Only the close calls below run in parallel: + // deciding WHICH sessions to close must stay a single ordered pass so the sweep-precedence + // rule and the TOCTOU re-check keep their meaning. + List<(GatewaySession Session, string Reason)> selected = []; foreach (GatewaySession session in _registry.Snapshot()) { // A session is swept when its normal lease has expired, it has FAULTED (a faulted @@ -288,45 +298,100 @@ public sealed class SessionManager : ISessionManager continue; } - await CloseSessionCoreAsync(session, reason, cancellationToken).ConfigureAwait(false); - closedCount++; + selected.Add((session, reason)); } + if (selected.Count == 0) + { + return 0; + } + + int closedCount = 0; + object failureSyncRoot = new(); + ExceptionDispatchInfo? firstFailure = null; + + // Close phase. Each close is bounded by the worker shutdown timeout (default 10 s), so a + // mass expiry with a few hung workers would serialize reaping and starve session slots. + // Parallel close is safe because TryBeginCloseIfExpired above already flipped every + // selected session to Closing under its own lock — that idempotent begin-close is the + // per-session exclusivity invariant, so no two teardowns can run against one session and + // a session selected here cannot be re-selected by a concurrent sweep. + await Parallel.ForEachAsync( + selected, + new ParallelOptions + { + MaxDegreeOfParallelism = MaxParallelSessionCloses, + CancellationToken = cancellationToken, + }, + async (candidate, closeToken) => + { + try + { + await CloseSessionCoreAsync(candidate.Session, candidate.Reason, closeToken).ConfigureAwait(false); + Interlocked.Increment(ref closedCount); + } + catch (Exception exception) + { + // The sequential sweep let a close failure propagate to the lease monitor, + // which logs it; that signal is preserved by rethrowing the first failure + // below. It is captured rather than thrown here so one failed (or hung) + // teardown does not abandon the rest of the already-selected set. + lock (failureSyncRoot) + { + firstFailure ??= ExceptionDispatchInfo.Capture(exception); + } + } + }).ConfigureAwait(false); + + firstFailure?.Throw(); + return closedCount; } /// public async Task ShutdownAsync(CancellationToken cancellationToken) { - foreach (GatewaySession session in _registry.Snapshot()) - { - try + // Sessions are drained in parallel: at a worst-case worker shutdown timeout each, a + // one-at-a-time drain of a full registry outruns any host stop-timeout and leaves the + // tail to the orphan killer. Per-session exclusivity comes from GatewaySession.CloseAsync's + // own close gate, and each iteration touches only its own session plus thread-safe + // registry/metrics state. + await Parallel.ForEachAsync( + _registry.Snapshot(), + new ParallelOptions { - await CloseSessionCoreAsync(session, GatewayShutdownReason, cancellationToken).ConfigureAwait(false); - } - catch (Exception exception) + MaxDegreeOfParallelism = MaxParallelSessionCloses, + CancellationToken = cancellationToken, + }, + async (session, closeToken) => { - _logger.LogWarning( - exception, - "Graceful shutdown failed for session {SessionId}; killing worker.", - session.SessionId); - - if (_registry.TryGet(session.SessionId, out _)) + try { - try + await CloseSessionCoreAsync(session, GatewayShutdownReason, closeToken).ConfigureAwait(false); + } + catch (Exception exception) + { + _logger.LogWarning( + exception, + "Graceful shutdown failed for session {SessionId}; killing worker.", + session.SessionId); + + if (_registry.TryGet(session.SessionId, out _)) { - await KillWorkerAsync(session.SessionId, GatewayShutdownReason, cancellationToken).ConfigureAwait(false); - } - catch (SessionManagerException killException) - { - _logger.LogWarning( - killException, - "Worker kill fallback failed for session {SessionId}.", - session.SessionId); + try + { + await KillWorkerAsync(session.SessionId, GatewayShutdownReason, closeToken).ConfigureAwait(false); + } + catch (SessionManagerException killException) + { + _logger.LogWarning( + killException, + "Worker kill fallback failed for session {SessionId}.", + session.SessionId); + } } } - } - } + }).ConfigureAwait(false); } private async Task CloseSessionCoreAsync( diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs index c320ff1..f978ae1 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs @@ -1087,6 +1087,90 @@ public sealed class SessionManagerTests Assert.Equal(1, workerClient.ShutdownCount); } + /// + /// A sweep pass tears the selected sessions down concurrently rather than one worker + /// shutdown after another: with a mass expiry, a few hung workers would otherwise + /// serialize reaping (each close is bounded by Worker:ShutdownTimeoutSeconds) and + /// starve session slots. Overlap is asserted by counting concurrent entries into the fake + /// worker's shutdown rather than by wall clock, which is sturdier on a loaded box. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task CloseExpiredLeasesAsync_ClosesExpiredSessionsConcurrently() + { + ShutdownConcurrencyProbe probe = new(expectedConcurrency: 2); + FakeWorkerClient firstClient = new() { ShutdownConcurrencyProbe = probe }; + FakeWorkerClient secondClient = new() { ShutdownConcurrencyProbe = probe }; + SessionManager manager = CreateManager(new QueueingSessionWorkerClientFactory(firstClient, secondClient)); + GatewaySession firstSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None); + GatewaySession secondSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None); + DateTimeOffset now = DateTimeOffset.UtcNow; + firstSession.ExtendLease(now.AddSeconds(-1)); + secondSession.ExtendLease(now.AddSeconds(-1)); + + int closedCount = await manager.CloseExpiredLeasesAsync(now, CancellationToken.None); + + Assert.Equal(2, closedCount); + Assert.Equal(SessionState.Closed, firstSession.State); + Assert.Equal(SessionState.Closed, secondSession.State); + Assert.Equal(2, probe.MaxObservedConcurrency); + } + + /// + /// Host stop drains sessions concurrently: 50 sessions at a worst-case 10 s shutdown each + /// would exceed any host stop-timeout if drained one at a time, leaving the tail to the + /// orphan killer. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task ShutdownAsync_ClosesSessionsConcurrently() + { + ShutdownConcurrencyProbe probe = new(expectedConcurrency: 2); + FakeWorkerClient firstClient = new() { ShutdownConcurrencyProbe = probe }; + FakeWorkerClient secondClient = new() { ShutdownConcurrencyProbe = probe }; + SessionManager manager = CreateManager(new QueueingSessionWorkerClientFactory(firstClient, secondClient)); + GatewaySession firstSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None); + GatewaySession secondSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None); + + await manager.ShutdownAsync(CancellationToken.None); + + Assert.Equal(SessionState.Closed, firstSession.State); + Assert.Equal(SessionState.Closed, secondSession.State); + Assert.Equal(2, probe.MaxObservedConcurrency); + } + + /// + /// A close that throws must not abandon the rest of the selected set: the sweep still + /// tears the healthy expired session down, and the failure still surfaces to the lease + /// monitor (which logs it) exactly as the sequential loop did. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task CloseExpiredLeasesAsync_WhenOneCloseFails_StillClosesRemainingSessionsAndRethrows() + { + FakeWorkerClient failingClient = new() + { + ShutdownException = new InvalidOperationException("worker shutdown failed"), + KillException = new InvalidOperationException("worker kill failed"), + }; + FakeWorkerClient healthyClient = new(); + SessionManager manager = CreateManager(new QueueingSessionWorkerClientFactory(failingClient, healthyClient)); + GatewaySession failingSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None); + GatewaySession healthySession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None); + DateTimeOffset now = DateTimeOffset.UtcNow; + failingSession.ExtendLease(now.AddSeconds(-1)); + healthySession.ExtendLease(now.AddSeconds(-1)); + + SessionManagerException exception = await Assert.ThrowsAsync( + async () => await manager.CloseExpiredLeasesAsync(now, CancellationToken.None)); + + Assert.Equal(SessionManagerErrorCode.CloseFailed, exception.ErrorCode); + Assert.Equal(1, healthyClient.ShutdownCount); + Assert.Equal(SessionState.Closed, healthySession.State); + Assert.False(manager.TryGetSession(healthySession.SessionId, out _)); + Assert.False(manager.TryGetSession(failingSession.SessionId, out _)); + } + /// Verifies that shutdown closes all registered sessions. /// A task that represents the asynchronous operation. [Fact] @@ -1274,6 +1358,13 @@ public sealed class SessionManagerTests /// Gets a value indicating whether to block shutdown on the fake worker client. public bool BlockShutdown { get; init; } + /// + /// Gets the rendezvous that records how many shutdowns overlap, shared by the fakes of + /// the sessions a single teardown pass closes. Null when the test does not measure + /// teardown concurrency. + /// + public ShutdownConcurrencyProbe? ShutdownConcurrencyProbe { get; init; } + /// Gets the last command invoked on the fake worker client. public WorkerCommand? LastCommand { get; private set; } @@ -1335,6 +1426,11 @@ public sealed class SessionManagerTests throw ShutdownException; } + if (ShutdownConcurrencyProbe is not null) + { + await ShutdownConcurrencyProbe.EnterAsync(cancellationToken); + } + if (BlockShutdown) { ShutdownStarted.TrySetResult(); @@ -1379,4 +1475,65 @@ public sealed class SessionManagerTests } } + /// + /// Rendezvous that measures how many worker shutdowns a teardown pass runs at once. Each + /// entering shutdown records the in-flight count and waits until + /// shutdowns are in flight, so a genuinely parallel teardown releases immediately while a + /// sequential one can only release on the bounded timeout — with a max observed concurrency + /// of one, which is the assertion that fails. + /// + /// Number of overlapping shutdowns that releases the rendezvous. + private sealed class ShutdownConcurrencyProbe(int expectedConcurrency) + { + private static readonly TimeSpan RendezvousTimeout = TimeSpan.FromSeconds(5); + + private readonly TaskCompletionSource _reached = new(TaskCreationOptions.RunContinuationsAsynchronously); + private int _inFlight; + private int _maxInFlight; + + /// Gets the highest number of shutdowns observed in flight at the same time. + public int MaxObservedConcurrency => Volatile.Read(ref _maxInFlight); + + /// Enters the rendezvous for one worker shutdown and waits for the expected overlap. + /// Token that abandons the wait. + /// A task that represents the asynchronous operation. + public async Task EnterAsync(CancellationToken cancellationToken) + { + int inFlight = Interlocked.Increment(ref _inFlight); + RecordMax(inFlight); + if (inFlight >= expectedConcurrency) + { + _reached.TrySetResult(); + } + + try + { + await _reached.Task.WaitAsync(RendezvousTimeout, cancellationToken); + } + catch (TimeoutException) + { + // Sequential teardown: the expected overlap never happens, so let the shutdown + // finish and let MaxObservedConcurrency report the (failing) truth. + } + finally + { + Interlocked.Decrement(ref _inFlight); + } + } + + private void RecordMax(int inFlight) + { + int observed = Volatile.Read(ref _maxInFlight); + while (inFlight > observed) + { + int previous = Interlocked.CompareExchange(ref _maxInFlight, inFlight, observed); + if (previous == observed) + { + return; + } + + observed = previous; + } + } + } } From 95f8ba918db991c8b4ee33d2dbd1897825cc86f3 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 13:19:08 -0400 Subject: [PATCH 23/41] fix(sessions): exception-total shutdown body with non-cancellable kill fallback --- docs/Sessions.md | 35 +++++++++++------- .../Sessions/ISessionManager.cs | 6 ++++ .../Sessions/SessionManager.cs | 36 +++++++++++++------ .../Gateway/Sessions/SessionManagerTests.cs | 33 +++++++++++++++-- 4 files changed, 86 insertions(+), 24 deletions(-) diff --git a/docs/Sessions.md b/docs/Sessions.md index 8502c96..418a368 100644 --- a/docs/Sessions.md +++ b/docs/Sessions.md @@ -209,9 +209,15 @@ Sessions open with `MxGateway:Sessions:DefaultLeaseSeconds` (default 1800) added #### Teardown parallelism -A sweep pass is two phases. *Selection* stays a single sequential pass over the snapshot, because that is what gives the precedence rule (lease-expiry, then faulted, then detach-grace) and the `TryBeginCloseIfExpired` TOCTOU re-check their meaning. *Closing* then runs over the already-selected set with `Parallel.ForEachAsync` at `MaxParallelSessionCloses` (4). Each close is bounded by `MxGateway:Worker:ShutdownTimeoutSeconds` (default 10), so a one-at-a-time sweep lets a few hung workers serialize reaping and starve session slots for the rest. Parallel closing is safe because `TryBeginCloseIfExpired` already flipped each selected session to `Closing` under its own lock — that idempotent begin-close is the per-session exclusivity invariant, so no two teardowns can ever run against one session. The degree is bounded rather than unlimited because every concurrent close is one x86 worker process being shut down or killed; the fan-out exists to hide a few hung workers, not to tear the whole registry down at once. +A sweep pass is two phases. *Selection* stays a single sequential pass over the snapshot, because that is what gives the precedence rule (lease-expiry, then faulted, then detach-grace) and the `TryBeginCloseIfExpired` TOCTOU re-check their meaning. *Closing* then runs over the already-selected set with `Parallel.ForEachAsync` at `MaxParallelSessionCloses`, a compile-time constant of `4` in `SessionManager`. Each close is bounded by `MxGateway:Worker:ShutdownTimeoutSeconds` (default 10), so a one-at-a-time sweep lets a few hung workers serialize reaping and starve session slots for the rest. Parallel closing is safe because `TryBeginCloseIfExpired` already flipped each selected session to `Closing` under its own lock — that idempotent begin-close is the per-session exclusivity invariant, so no two teardowns can ever run against one session. The degree is a fixed constant rather than an option, and bounded rather than unlimited, because every concurrent close is one x86 worker process being shut down or killed; the fan-out exists to hide a few hung workers, not to tear the whole registry down at once. -A close that throws no longer abandons the rest of the selected set: the sweep captures the first failure, finishes the remaining closes, then rethrows it so `SessionLeaseMonitorHostedService` still logs the sweep failure as before. `ShutdownAsync` drains sessions with the same bounded fan-out, keeping its existing per-session catch → `KillWorkerAsync` fallback. +Splitting the phases moves the TOCTOU re-check earlier, and that is an accepted trade rather than an unchanged behavior: selection now flips **every** chosen session to `Closing` up front, before any teardown runs, whereas the sequential sweep re-checked session *N* only after sessions *1..N-1* had finished closing. A client that re-attaches a subscriber while the close phase is running therefore loses a race it could previously win — the eligibility snapshot is taken at one instant for the whole pass. Expiry evaluation itself is unaffected, because `now` is a parameter and is not re-read per session. + +A close that throws no longer abandons the rest of the selected set: the sweep attempts every selected session, captures the first failure, and rethrows it once the pass is done so `SessionLeaseMonitorHostedService` still logs the sweep failure as before. Because the closes run concurrently, *which* failure surfaces when several fail in one pass is nondeterministic; the log line is the diagnostic, not the identity of the exception. + +`ShutdownAsync` drains sessions with the same bounded fan-out and the same per-session catch → `KillWorkerAsync` fallback, with two rules that keep a stop deadline from turning into leaked workers. First, its body is **exception-total** — nothing escapes it — because `Parallel.ForEachAsync` cancels the token handed to the sibling bodies as soon as one body throws, which would abort in-flight graceful shutdowns *and* make their kill fallback fail instantly on the freshly cancelled token. Second, the drain loop is deliberately **not** bound to the caller's `CancellationToken` and the kill fallback runs on `CancellationToken.None`: a cancelled `ParallelOptions` token stops dispatching the remaining sessions entirely, whereas the sequential drain this replaced let every remaining session fail its graceful close fast and still kill its worker. The token is passed to the graceful close instead, so a host stop deadline turns the drain into a kill sweep rather than into a leak. This matters because nothing reattaches to a leaked worker — a restarted gateway terminates orphans (see [Design Decisions](DesignDecisions.md)). + +**Stranded-`Closing` bound.** A sweep pass that is cancelled after selection leaves its unclosed selections in `Closing` with close already started. `IsFaultedReapableCore` requires `state == Faulted`, so a session selected under `FaultedReason` and stranded this way is not re-selected as faulted; it is swept only when its normal lease expires (up to `MxGateway:Sessions:DefaultLeaseSeconds`, default 1800 s), since `IsLeaseExpiredCore` and `IsDetachGraceExpiredCore` are state-agnostic. This bound is documented rather than closed with a re-selection clause: the sweep's only caller cancels on the host's `stoppingToken`, so the very next thing that runs is `ShutdownAsync`, which drains (or kills) the whole registry — and any worker that still survives that is terminated as an orphan on the next gateway start. Adding a "`Closing` and close-started" re-selection clause would also have to distinguish an abandoned close from one that is merely still in flight, which would weaken the single invariant that makes the parallel close phase safe. #### Detach-grace retention @@ -282,21 +288,17 @@ If both graceful shutdown and the kill fall-back fail, the original and kill exc ## Shutdown Coordination -`SessionShutdownHostedService.StopAsync` calls `SessionManager.ShutdownAsync`, which closes every registered session with `GatewayShutdownReason`. Sessions are drained with the same bounded fan-out the lease sweep uses (`MaxParallelSessionCloses`), because a one-at-a-time drain of a full registry at a worst-case worker shutdown timeout each outruns any host stop-timeout and leaves the tail to the orphan killer. Each iteration catches its own exceptions, calls `KillWorkerAsync`, and removes the session so that one stuck worker cannot block the rest of the host: +`SessionShutdownHostedService.StopAsync` calls `SessionManager.ShutdownAsync`, which closes every registered session with `GatewayShutdownReason`. Sessions are drained with the same bounded fan-out the lease sweep uses (`MaxParallelSessionCloses`), because a one-at-a-time drain of a full registry at a worst-case worker shutdown timeout each outruns any host stop-timeout and leaves the tail to the orphan killer. Each iteration catches its own exceptions — *every* exception, including from the fallback — calls `KillWorkerAsync` on an uncancellable token, and removes the session, so that neither one stuck worker nor one failing teardown can block or abort the rest of the host's drain: ```csharp await Parallel.ForEachAsync( _registry.Snapshot(), - new ParallelOptions - { - MaxDegreeOfParallelism = MaxParallelSessionCloses, - CancellationToken = cancellationToken, - }, - async (session, closeToken) => + new ParallelOptions { MaxDegreeOfParallelism = MaxParallelSessionCloses }, + async (session, _) => { try { - await CloseSessionCoreAsync(session, GatewayShutdownReason, closeToken).ConfigureAwait(false); + await CloseSessionCoreAsync(session, GatewayShutdownReason, cancellationToken).ConfigureAwait(false); } catch (Exception exception) { @@ -304,9 +306,18 @@ await Parallel.ForEachAsync( exception, "Graceful shutdown failed for session {SessionId}; killing worker.", session.SessionId); - if (_registry.TryGet(session.SessionId, out _)) + if (_registry.TryGet(session.SessionId, out GatewaySession? registeredSession) + && registeredSession is not null) { - await KillWorkerAsync(session.SessionId, GatewayShutdownReason, closeToken).ConfigureAwait(false); + try + { + // Not the caller's token: the kill is the last-resort orphan preventer. + await KillWorkerAsync(session.SessionId, GatewayShutdownReason, CancellationToken.None).ConfigureAwait(false); + } + catch (Exception killException) + { + _logger.LogWarning(killException, "Worker kill fallback failed for session {SessionId}.", session.SessionId); + } } } }).ConfigureAwait(false); diff --git a/src/ZB.MOM.WW.MxGateway.Server/Sessions/ISessionManager.cs b/src/ZB.MOM.WW.MxGateway.Server/Sessions/ISessionManager.cs index f9c200e..232a17c 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Sessions/ISessionManager.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Sessions/ISessionManager.cs @@ -68,6 +68,12 @@ public interface ISessionManager /// The current time to evaluate expiration against. /// Token to cancel the asynchronous operation. /// The number of sessions closed. + /// + /// A close that fails does not abandon the rest of the pass: every session selected by this + /// sweep is attempted, and the first failure is then rethrown so the caller still observes + /// (and logs) that the sweep failed. Which failure surfaces is nondeterministic when several + /// closes fail in the same pass, because the closes run concurrently. + /// Task CloseExpiredLeasesAsync( DateTimeOffset now, CancellationToken cancellationToken); diff --git a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs index ecff690..9042aad 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs @@ -356,18 +356,28 @@ public sealed class SessionManager : ISessionManager // tail to the orphan killer. Per-session exclusivity comes from GatewaySession.CloseAsync's // own close gate, and each iteration touches only its own session plus thread-safe // registry/metrics state. + // + // The body must be exception-TOTAL. Parallel.ForEachAsync cancels the token it hands the + // sibling bodies as soon as one body throws, so a single escaping exception would abort up + // to MaxParallelSessionCloses - 1 in-flight graceful shutdowns AND make their kill fallback + // throw immediately on the freshly cancelled token — sessions neither closed nor killed, + // i.e. leaked x86 workers that nothing reattaches to (a gateway restart terminates orphans + // rather than adopting them). + // + // For the same reason the loop itself is NOT bound to cancellationToken: a cancelled + // ParallelOptions token stops dispatching the remaining sessions entirely, whereas the + // sequential drain this replaced let every remaining session fail its graceful close fast + // and still kill its worker. The token is passed to the graceful close instead, which + // preserves that behavior — a host stop deadline turns the drain into a kill sweep rather + // than into a leak. await Parallel.ForEachAsync( _registry.Snapshot(), - new ParallelOptions - { - MaxDegreeOfParallelism = MaxParallelSessionCloses, - CancellationToken = cancellationToken, - }, - async (session, closeToken) => + new ParallelOptions { MaxDegreeOfParallelism = MaxParallelSessionCloses }, + async (session, _) => { try { - await CloseSessionCoreAsync(session, GatewayShutdownReason, closeToken).ConfigureAwait(false); + await CloseSessionCoreAsync(session, GatewayShutdownReason, cancellationToken).ConfigureAwait(false); } catch (Exception exception) { @@ -376,13 +386,19 @@ public sealed class SessionManager : ISessionManager "Graceful shutdown failed for session {SessionId}; killing worker.", session.SessionId); - if (_registry.TryGet(session.SessionId, out _)) + if (_registry.TryGet(session.SessionId, out GatewaySession? registeredSession) + && registeredSession is not null) { try { - await KillWorkerAsync(session.SessionId, GatewayShutdownReason, closeToken).ConfigureAwait(false); + // Deliberately NOT the caller's token: the kill is the last-resort orphan + // preventer, so it must still run when the host stop deadline (or a + // sibling body's failure) has already cancelled the drain. It is a + // synchronous Kill plus registry/dispose bookkeeping, not a wait on + // the worker, so it cannot extend the drain meaningfully. + await KillWorkerAsync(session.SessionId, GatewayShutdownReason, CancellationToken.None).ConfigureAwait(false); } - catch (SessionManagerException killException) + catch (Exception killException) { _logger.LogWarning( killException, diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs index f978ae1..f4e16f1 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionManagerTests.cs @@ -1171,6 +1171,33 @@ public sealed class SessionManagerTests Assert.False(manager.TryGetSession(failingSession.SessionId, out _)); } + /// + /// A drain whose token is already cancelled (the host stop deadline elapsed) must still + /// kill every worker rather than skip the teardown: an unkilled worker is a leaked x86 + /// process, and a restarted gateway terminates orphans instead of reattaching to them. This + /// pins both halves of the fix — the parallel loop is not bound to the caller's token, and + /// the kill fallback does not run on it. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task ShutdownAsync_WhenCancelledBeforeDraining_StillKillsEveryWorker() + { + FakeWorkerClient firstClient = new(); + FakeWorkerClient secondClient = new(); + SessionManager manager = CreateManager(new QueueingSessionWorkerClientFactory(firstClient, secondClient)); + GatewaySession firstSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-1", ownerKeyId: null, CancellationToken.None); + GatewaySession secondSession = await manager.OpenSessionAsync(CreateOpenRequest(), "client-2", ownerKeyId: null, CancellationToken.None); + using CancellationTokenSource cancellation = new(); + await cancellation.CancelAsync(); + + await manager.ShutdownAsync(cancellation.Token); + + Assert.Equal(1, firstClient.KillCount); + Assert.Equal(1, secondClient.KillCount); + Assert.False(manager.TryGetSession(firstSession.SessionId, out _)); + Assert.False(manager.TryGetSession(secondSession.SessionId, out _)); + } + /// Verifies that shutdown closes all registered sessions. /// A task that represents the asynchronous operation. [Fact] @@ -1510,10 +1537,12 @@ public sealed class SessionManagerTests { await _reached.Task.WaitAsync(RendezvousTimeout, cancellationToken); } - catch (TimeoutException) + catch (Exception exception) when (exception is TimeoutException or OperationCanceledException) { // Sequential teardown: the expected overlap never happens, so let the shutdown - // finish and let MaxObservedConcurrency report the (failing) truth. + // finish and let MaxObservedConcurrency report the (failing) truth. Cancellation is + // swallowed for the same reason — a cancelled rendezvous must not turn into a + // second, misleading failure on top of the concurrency assertion. } finally { From 13df92fbd88a93f5f0881eb0d0d34dc817e4621d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 13:25:05 -0400 Subject: [PATCH 24/41] =?UTF-8?q?chore(plans):=20Phase=20A=20complete=20?= =?UTF-8?q?=E2=80=94=20gate=20green=20at=201015/1015?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...8-15-perf-review-remediation.md.tasks.json | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json b/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json index 44e8522..1a2be11 100644 --- a/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json +++ b/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json @@ -1,21 +1,21 @@ { "planPath": "docs/plans/2026-08-15-perf-review-remediation.md", "tasks": [ - {"id": 1, "subject": "Task 1: Named-pipe buffer sizes", "status": "pending"}, - {"id": 2, "subject": "Task 2: Metrics pull-gauge + Interlocked counters", "status": "pending"}, - {"id": 3, "subject": "Task 3: Distributor copy-on-write subscriber snapshot", "status": "pending"}, - {"id": 4, "subject": "Task 4: Dashboard event mirror viewer gating", "status": "pending"}, - {"id": 5, "subject": "Task 5: Snapshot pipeline idle gating + cached config", "status": "pending"}, - {"id": 6, "subject": "Task 6: Reply ownership transfer in MapCommandReply", "status": "pending"}, - {"id": 7, "subject": "Task 7: WorkerClient WaitAsync timeout + WorkerCancel", "status": "pending", "blockedBy": [2]}, - {"id": 8, "subject": "Task 8: Audit pipeline background writer + retention", "status": "pending"}, - {"id": 9, "subject": "Task 9: Parallel session teardown", "status": "pending"}, - {"id": 10, "subject": "Task 10: Dashboard live-data subscription cap", "status": "pending"}, - {"id": 11, "subject": "Task 11: Alarm monitor cached CurrentAlarms", "status": "pending"}, - {"id": 12, "subject": "Task 12: Logging middleware hoist + bearer redaction fix", "status": "pending"}, - {"id": 13, "subject": "Task 13: Auth-path span parsing + limiter keys", "status": "pending"}, - {"id": 14, "subject": "Task 14: Bulk constraint loops, caches, hygiene", "status": "pending"}, - {"id": 15, "subject": "Task 15: Phase A gate — full gateway suite", "status": "pending", "blockedBy": [1,2,3,4,5,6,7,8,9,10,11,12,13,14]}, + {"id": 1, "subject": "Task 1: Named-pipe buffer sizes", "status": "completed"}, + {"id": 2, "subject": "Task 2: Metrics pull-gauge + Interlocked counters", "status": "completed"}, + {"id": 3, "subject": "Task 3: Distributor copy-on-write subscriber snapshot", "status": "completed"}, + {"id": 4, "subject": "Task 4: Dashboard event mirror viewer gating", "status": "completed"}, + {"id": 5, "subject": "Task 5: Snapshot pipeline idle gating + cached config", "status": "completed"}, + {"id": 6, "subject": "Task 6: Reply ownership transfer in MapCommandReply", "status": "completed"}, + {"id": 7, "subject": "Task 7: WorkerClient WaitAsync timeout + WorkerCancel", "status": "completed", "blockedBy": [2]}, + {"id": 8, "subject": "Task 8: Audit pipeline background writer + retention", "status": "completed"}, + {"id": 9, "subject": "Task 9: Parallel session teardown", "status": "completed"}, + {"id": 10, "subject": "Task 10: Dashboard live-data subscription cap", "status": "completed"}, + {"id": 11, "subject": "Task 11: Alarm monitor cached CurrentAlarms", "status": "completed"}, + {"id": 12, "subject": "Task 12: Logging middleware hoist + bearer redaction fix", "status": "completed"}, + {"id": 13, "subject": "Task 13: Auth-path span parsing + limiter keys", "status": "completed"}, + {"id": 14, "subject": "Task 14: Bulk constraint loops, caches, hygiene", "status": "completed"}, + {"id": 15, "subject": "Task 15: Phase A gate — full gateway suite", "status": "in_progress", "blockedBy": [1,2,3,4,5,6,7,8,9,10,11,12,13,14]}, {"id": 16, "subject": "Task 16: Event drain wake signal", "status": "pending", "blockedBy": [15]}, {"id": 17, "subject": "Task 17: Event queue capacity env plumbing", "status": "pending", "blockedBy": [16]}, {"id": 18, "subject": "Task 18: STA message-driven completion waits", "status": "pending", "blockedBy": [15]}, @@ -25,7 +25,7 @@ {"id": 22, "subject": "Task 22: Worker frame writer/reader pooled buffers", "status": "pending", "blockedBy": [15]}, {"id": 23, "subject": "Task 23: Alarm consumer parse + truncation + cadence", "status": "pending", "blockedBy": [15]}, {"id": 24, "subject": "Task 24: Phase B verification on windev (psbridge)", "status": "pending", "blockedBy": [16,17,18,19,20,21,22,23]}, - {"id": 25, "subject": "Task 25: Wrap-up docs sweep + umbrella check", "status": "pending", "blockedBy": [24]} + {"id": 25, "subject": "Task 25: Wrap-up docs sweep + follow-ups", "status": "pending", "blockedBy": [24]} ], - "lastUpdated": "2026-08-15T00:00:00Z" + "lastUpdated": "2026-08-15T13:30:00Z" } From f56798aeb9c5cb105d384202c43540c1faa2d19e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 13:28:29 -0400 Subject: [PATCH 25/41] =?UTF-8?q?perf(worker):=20pooled=20frame=20buffers?= =?UTF-8?q?=20=E2=80=94=20brings=20the=20net48=20codec=20up=20to=20the=20g?= =?UTF-8?q?ateway=20side's=20GWC-30=20pattern?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Ipc/WorkerFrameReader.cs | 15 ++++++++--- .../Ipc/WorkerFrameWriter.cs | 27 +++++++++++++------ 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameReader.cs b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameReader.cs index 8a03900..041e879 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameReader.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameReader.cs @@ -9,11 +9,21 @@ using ZB.MOM.WW.MxGateway.Contracts.Proto; namespace ZB.MOM.WW.MxGateway.Worker.Ipc; /// Reads length-prefixed WorkerEnvelope protobuf frames from a stream. +/// +/// is not reentrant: the reader keeps a per-instance length-prefix scratch +/// buffer, so exactly one consumer may be inside a read at a time. That matches how the reader is +/// used — a single read loop per WorkerPipeSession, with the startup handshake read +/// completing before the loop starts. +/// public sealed class WorkerFrameReader { private readonly WorkerFrameProtocolOptions _options; private readonly Stream _stream; + // Reused across frames rather than allocated per read (GWC-30). Safe because ReadAsync is + // single-consumer by construction; the prefix is fully overwritten by every read. + private readonly byte[] _lengthPrefix = new byte[sizeof(uint)]; + /// Initializes the reader with a stream and protocol options. /// Stream to read frames from. /// Protocol options for frame validation. @@ -30,10 +40,9 @@ public sealed class WorkerFrameReader /// The validated read from the stream. public async Task ReadAsync(CancellationToken cancellationToken = default) { - byte[] lengthPrefix = new byte[sizeof(uint)]; - await ReadExactlyOrThrowAsync(lengthPrefix, lengthPrefix.Length, cancellationToken).ConfigureAwait(false); + await ReadExactlyOrThrowAsync(_lengthPrefix, sizeof(uint), cancellationToken).ConfigureAwait(false); - uint payloadLength = ReadUInt32LittleEndian(lengthPrefix); + uint payloadLength = ReadUInt32LittleEndian(_lengthPrefix); if (payloadLength == 0) { throw new WorkerFrameProtocolException( diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWriter.cs b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWriter.cs index 7fab801..cfb20e7 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWriter.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWriter.cs @@ -1,4 +1,5 @@ using System; +using System.Buffers; using System.Collections.Generic; using System.IO; using System.Runtime.ExceptionServices; @@ -459,16 +460,26 @@ public sealed class WorkerFrameWriter _nextSequence = candidateSequence; - // Serialize once into a single buffer that carries the 4-byte length prefix followed by the - // payload, then issue one stream write. This avoids a second serialization pass, a separate - // prefix array, and a separate prefix write. The flush is deferred to the end of the drained - // batch (see DrainQueuedFramesAsync) so a burst of frames shares one flush. + // Serialize once into a single pooled buffer that carries the 4-byte length prefix followed + // by the payload, then issue one stream write. This avoids a second serialization pass, a + // separate prefix array, a separate prefix write, and any per-frame heap allocation — the + // gateway-side writer's GWC-30 shape, now matched on the net48 side. The rented buffer may + // be larger than requested, so only the first frameLength bytes are ever written. The buffer + // is returned only after this frame's write has completed; the batch flush deferred to the + // end of the drain (see DrainQueuedFramesAsync) does not read from it. int frameLength = sizeof(uint) + payloadLength; - byte[] frame = new byte[frameLength]; - WriteUInt32LittleEndian(frame, (uint)payloadLength); - envelope.WriteTo(new Span(frame, sizeof(uint), payloadLength)); + byte[] frame = ArrayPool.Shared.Rent(frameLength); + try + { + WriteUInt32LittleEndian(frame, (uint)payloadLength); + envelope.WriteTo(new Span(frame, sizeof(uint), payloadLength)); - await _stream.WriteAsync(frame, 0, frameLength, CancellationToken.None).ConfigureAwait(false); + await _stream.WriteAsync(frame, 0, frameLength, CancellationToken.None).ConfigureAwait(false); + } + finally + { + ArrayPool.Shared.Return(frame); + } } private static void WriteUInt32LittleEndian( From afec56d03b16f933a5973400821f7b2bda980e71 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 16:58:02 -0400 Subject: [PATCH 26/41] perf(worker): reverse tag index + memoized views + indexed removals in the handle registry --- .../MxAccess/MxAccessHandleRegistryTests.cs | 272 ++++++++++++++++ .../MxAccess/MxAccessHandleRegistry.cs | 291 ++++++++++++++++-- .../MxAccess/MxAccessSession.cs | 31 +- 3 files changed, 555 insertions(+), 39 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessHandleRegistryTests.cs diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessHandleRegistryTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessHandleRegistryTests.cs new file mode 100644 index 0000000..14814d7 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessHandleRegistryTests.cs @@ -0,0 +1,272 @@ +using System; +using System.Collections.Generic; +using ZB.MOM.WW.MxGateway.Worker.MxAccess; + +namespace ZB.MOM.WW.MxGateway.Worker.Tests.MxAccess; + +/// +/// Unit tests for . The registry gained a +/// reverse (server handle, item definition) index, memoized sorted views, and +/// secondary removal indexes so bulk reads and bulk teardown stop rescanning +/// the whole handle table. These tests pin the observable behaviour those +/// structures must preserve: the same lookup semantics the old linear scan +/// had, snapshot views that only rebuild after a mutation, and removals that +/// leave every index consistent. +/// +public sealed class MxAccessHandleRegistryTests +{ + /// + /// Verifies the reverse index answers by server handle and ordinal item + /// definition, returns every duplicate registration in ascending item + /// handle order (the order a scan of + /// would have visited them), and misses on the wrong server, a different + /// tag, and a case-differing tag. + /// + [Fact] + public void GetItemHandlesForDefinition_MatchesServerAndOrdinalTag_InAscendingHandleOrder() + { + MxAccessHandleRegistry registry = new(); + registry.RegisterServerHandle(serverHandle: 1, clientName: "client"); + registry.RegisterServerHandle(serverHandle: 2, clientName: "client"); + + // Same tag added twice under server 1 — MXAccess hands back a distinct + // item handle per AddItem, so the index must keep both, lowest first. + registry.RegisterItemHandle(1, itemHandle: 40, "Tank1.PV", string.Empty, hasItemContext: false); + registry.RegisterItemHandle(1, itemHandle: 10, "Tank1.PV", string.Empty, hasItemContext: false); + registry.RegisterItemHandle(1, itemHandle: 11, "Tank1.SP", string.Empty, hasItemContext: false); + registry.RegisterItemHandle(2, itemHandle: 12, "Tank1.PV", string.Empty, hasItemContext: false); + + Assert.Equal(new[] { 10, 40 }, registry.GetItemHandlesForDefinition(1, "Tank1.PV")); + Assert.Equal(new[] { 11 }, registry.GetItemHandlesForDefinition(1, "Tank1.SP")); + Assert.Equal(new[] { 12 }, registry.GetItemHandlesForDefinition(2, "Tank1.PV")); + + // Misses: unknown server, unknown tag, and a case-differing tag — the + // read path compares tag addresses ordinally, so casing must not match. + Assert.Empty(registry.GetItemHandlesForDefinition(3, "Tank1.PV")); + Assert.Empty(registry.GetItemHandlesForDefinition(1, "Tank9.PV")); + Assert.Empty(registry.GetItemHandlesForDefinition(1, "tank1.pv")); + } + + /// + /// Verifies re-registering an item handle under a new item definition + /// retires the old reverse-index entry instead of leaving a stale hit. + /// + [Fact] + public void RegisterItemHandle_ReusedHandleWithNewDefinition_RetiresStaleIndexEntry() + { + MxAccessHandleRegistry registry = new(); + registry.RegisterItemHandle(1, itemHandle: 10, "Tank1.PV", string.Empty, hasItemContext: false); + registry.RegisterItemHandle(1, itemHandle: 10, "Tank2.PV", string.Empty, hasItemContext: false); + + Assert.Empty(registry.GetItemHandlesForDefinition(1, "Tank1.PV")); + Assert.Equal(new[] { 10 }, registry.GetItemHandlesForDefinition(1, "Tank2.PV")); + Assert.Single(registry.ItemHandles); + } + + /// + /// Verifies the reverse index drops handles as they are removed, both for + /// a single item removal and for a whole-server teardown. + /// + [Fact] + public void GetItemHandlesForDefinition_AfterRemoval_MissesRemovedHandles() + { + MxAccessHandleRegistry registry = new(); + registry.RegisterItemHandle(1, itemHandle: 10, "Tank1.PV", string.Empty, hasItemContext: false); + registry.RegisterItemHandle(1, itemHandle: 11, "Tank1.PV", string.Empty, hasItemContext: false); + + registry.RemoveItemHandle(1, itemHandle: 10); + Assert.Equal(new[] { 11 }, registry.GetItemHandlesForDefinition(1, "Tank1.PV")); + + registry.UnregisterServerHandle(1); + Assert.Empty(registry.GetItemHandlesForDefinition(1, "Tank1.PV")); + } + + /// + /// Verifies each sorted view is memoized: repeated reads hand back the + /// same instance until a mutation of that table invalidates it, and the + /// rebuilt view reflects the mutation while keeping its sort order. + /// + [Fact] + public void Views_AreMemoizedUntilTheirTableMutates() + { + MxAccessHandleRegistry registry = new(); + registry.RegisterServerHandle(serverHandle: 2, clientName: "client"); + registry.RegisterServerHandle(serverHandle: 1, clientName: "client"); + registry.RegisterItemHandle(1, itemHandle: 11, "Tank1.PV", string.Empty, hasItemContext: false); + registry.RegisterItemHandle(1, itemHandle: 10, "Tank1.SP", string.Empty, hasItemContext: false); + registry.RegisterAdviceHandle(1, itemHandle: 10, MxAccessAdviceKind.Supervisory); + registry.RegisterAdviceHandle(1, itemHandle: 10, MxAccessAdviceKind.Plain); + + IReadOnlyList servers = registry.ServerHandles; + IReadOnlyList items = registry.ItemHandles; + IReadOnlyList advices = registry.AdviceHandles; + + // No mutation in between: the memoized arrays are handed back as-is. + Assert.Same(servers, registry.ServerHandles); + Assert.Same(items, registry.ItemHandles); + Assert.Same(advices, registry.AdviceHandles); + + // Sort orders are unchanged by memoization. + Assert.Equal(new[] { 1, 2 }, Map(servers, handle => handle.ServerHandle)); + Assert.Equal(new[] { 10, 11 }, Map(items, handle => handle.ItemHandle)); + Assert.Equal( + new[] { MxAccessAdviceKind.Plain, MxAccessAdviceKind.Supervisory }, + Map(advices, handle => handle.AdviceKind)); + + // A mutation of one table invalidates that view only. + registry.RegisterServerHandle(serverHandle: 3, clientName: "client"); + Assert.NotSame(servers, registry.ServerHandles); + Assert.Equal(3, registry.ServerHandles.Count); + Assert.Same(items, registry.ItemHandles); + Assert.Same(advices, registry.AdviceHandles); + + registry.RemoveItemHandle(1, itemHandle: 11); + Assert.NotSame(items, registry.ItemHandles); + Assert.Single(registry.ItemHandles); + + registry.RemoveAdviceHandles(1, itemHandle: 10); + Assert.NotSame(advices, registry.AdviceHandles); + Assert.Empty(registry.AdviceHandles); + } + + /// + /// Verifies removing an item also removes every advice recorded for it and + /// nothing recorded for a sibling item — the bulk unadvise/remove path + /// leans on this to leave a consistent table. + /// + [Fact] + public void RemoveItemHandle_RemovesOnlyThatItemsAdvices() + { + MxAccessHandleRegistry registry = new(); + registry.RegisterItemHandle(1, itemHandle: 10, "Tank1.PV", string.Empty, hasItemContext: false); + registry.RegisterItemHandle(1, itemHandle: 11, "Tank1.SP", string.Empty, hasItemContext: false); + registry.RegisterAdviceHandle(1, itemHandle: 10, MxAccessAdviceKind.Plain); + registry.RegisterAdviceHandle(1, itemHandle: 10, MxAccessAdviceKind.Supervisory); + registry.RegisterAdviceHandle(1, itemHandle: 11, MxAccessAdviceKind.Plain); + + registry.RemoveItemHandle(1, itemHandle: 10); + + Assert.False(registry.ContainsItemHandle(1, 10)); + Assert.False(registry.ContainsAdviceHandle(1, 10, MxAccessAdviceKind.Plain)); + Assert.False(registry.ContainsAdviceHandle(1, 10, MxAccessAdviceKind.Supervisory)); + Assert.True(registry.ContainsItemHandle(1, 11)); + Assert.True(registry.ContainsAdviceHandle(1, 11, MxAccessAdviceKind.Plain)); + Assert.Single(registry.AdviceHandles); + } + + /// + /// Verifies a bulk unadvise followed by a bulk remove drains the registry + /// completely, so the indexed removals cannot leave orphaned entries that + /// a later lookup would resurrect. + /// + [Fact] + public void BulkUnadviseThenRemove_LeavesRegistryEmpty() + { + MxAccessHandleRegistry registry = new(); + registry.RegisterServerHandle(serverHandle: 1, clientName: "client"); + + for (int itemHandle = 1; itemHandle <= 50; itemHandle++) + { + registry.RegisterItemHandle(1, itemHandle, "Tank." + itemHandle, string.Empty, hasItemContext: false); + registry.RegisterAdviceHandle(1, itemHandle, MxAccessAdviceKind.Plain); + } + + for (int itemHandle = 1; itemHandle <= 50; itemHandle++) + { + registry.RemoveAdviceHandles(1, itemHandle); + } + + Assert.Empty(registry.AdviceHandles); + Assert.Equal(50, registry.ItemHandles.Count); + + for (int itemHandle = 1; itemHandle <= 50; itemHandle++) + { + registry.RemoveItemHandle(1, itemHandle); + } + + Assert.Empty(registry.ItemHandles); + Assert.Empty(registry.GetItemHandlesForDefinition(1, "Tank.25")); + Assert.True(registry.ContainsServerHandle(1)); + } + + /// + /// Verifies unregistering a server drops its items and advices and leaves + /// every other server's handles untouched, including an advice registered + /// for an item that was never added to the item table. + /// + [Fact] + public void UnregisterServerHandle_RemovesOnlyThatServersHandles() + { + MxAccessHandleRegistry registry = new(); + registry.RegisterServerHandle(serverHandle: 1, clientName: "client-one"); + registry.RegisterServerHandle(serverHandle: 2, clientName: "client-two"); + + registry.RegisterItemHandle(1, itemHandle: 10, "Tank1.PV", string.Empty, hasItemContext: false); + registry.RegisterItemHandle(1, itemHandle: 11, "Tank1.SP", string.Empty, hasItemContext: false); + registry.RegisterAdviceHandle(1, itemHandle: 10, MxAccessAdviceKind.Plain); + + // Advice without a matching item registration: the old scan removed it by + // server handle, so the per-server index must reach it too. + registry.RegisterAdviceHandle(1, itemHandle: 99, MxAccessAdviceKind.Supervisory); + + // Server 2 reuses the same item handle values — packing must keep them apart. + registry.RegisterItemHandle(2, itemHandle: 10, "Tank1.PV", string.Empty, hasItemContext: false); + registry.RegisterAdviceHandle(2, itemHandle: 10, MxAccessAdviceKind.Plain); + + registry.UnregisterServerHandle(1); + + Assert.False(registry.ContainsServerHandle(1)); + Assert.False(registry.ContainsItemHandle(1, 10)); + Assert.False(registry.ContainsItemHandle(1, 11)); + Assert.False(registry.ContainsAdviceHandle(1, 10, MxAccessAdviceKind.Plain)); + Assert.False(registry.ContainsAdviceHandle(1, 99, MxAccessAdviceKind.Supervisory)); + Assert.Empty(registry.GetItemHandlesForDefinition(1, "Tank1.PV")); + + Assert.True(registry.ContainsServerHandle(2)); + Assert.True(registry.ContainsItemHandle(2, 10)); + Assert.True(registry.ContainsAdviceHandle(2, 10, MxAccessAdviceKind.Plain)); + Assert.Equal(new[] { 10 }, registry.GetItemHandlesForDefinition(2, "Tank1.PV")); + Assert.Single(registry.ItemHandles); + Assert.Single(registry.AdviceHandles); + Assert.Single(registry.ServerHandles); + } + + /// + /// Verifies re-registering an advice that is already present does not + /// duplicate it in the per-item removal index — a duplicate would survive + /// the removal that drains that index. + /// + [Fact] + public void RegisterAdviceHandle_RegisteredTwice_StillRemovedByOneCall() + { + MxAccessHandleRegistry registry = new(); + registry.RegisterAdviceHandle(1, itemHandle: 10, MxAccessAdviceKind.Plain); + registry.RegisterAdviceHandle(1, itemHandle: 10, MxAccessAdviceKind.Plain); + + Assert.Single(registry.AdviceHandles); + + registry.RemoveAdviceHandles(1, itemHandle: 10); + + Assert.Empty(registry.AdviceHandles); + Assert.False(registry.ContainsAdviceHandle(1, 10, MxAccessAdviceKind.Plain)); + + // Re-advising after the removal must work off a clean index. + registry.RegisterAdviceHandle(1, itemHandle: 10, MxAccessAdviceKind.Plain); + Assert.Single(registry.AdviceHandles); + Assert.True(registry.ContainsAdviceHandle(1, 10, MxAccessAdviceKind.Plain)); + } + + private static List Map( + IReadOnlyList source, + Func selector) + { + List mapped = new(source.Count); + + for (int index = 0; index < source.Count; index++) + { + mapped.Add(selector(source[index])); + } + + return mapped; + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessHandleRegistry.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessHandleRegistry.cs index e761594..31531e6 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessHandleRegistry.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessHandleRegistry.cs @@ -4,27 +4,52 @@ using System.Linq; namespace ZB.MOM.WW.MxGateway.Worker.MxAccess; +/// +/// Tracks the server, item, and advice handles owned by one MXAccess session. +/// +/// +/// The registry is STA-confined by contract: every mutation and every query +/// runs on the session's dedicated STA thread (see +/// , which marshals all registry access +/// through that thread), so there is no locking here and the memoized views +/// and secondary indexes below need none either. +/// public sealed class MxAccessHandleRegistry { private readonly Dictionary serverHandles = new(); private readonly Dictionary itemHandles = new(); private readonly Dictionary adviceHandles = new(); + // Secondary indexes. Every register/remove method below maintains these so + // lookups by tag and removals stay proportional to what is actually removed + // instead of scanning a whole primary table. + private readonly Dictionary> itemHandlesByDefinition = new(); + private readonly Dictionary> itemKeysByServer = new(); + private readonly Dictionary> adviceKeysByItem = new(); + private readonly Dictionary> advisedItemKeysByServer = new(); + + // Memoized materializations of the three sorted views. Each is dropped by + // the mutation that invalidates it, so repeated reads between mutations + // hand back the same array instead of re-sorting and re-copying the table. + private RegisteredServerHandle[]? serverHandlesView; + private RegisteredItemHandle[]? itemHandlesView; + private RegisteredAdviceHandle[]? adviceHandlesView; + /// Gets a read-only list of registered server handles ordered by handle value. - public IReadOnlyList ServerHandles => serverHandles + public IReadOnlyList ServerHandles => serverHandlesView ??= serverHandles .Values .OrderBy(handle => handle.ServerHandle) .ToArray(); /// Gets a read-only list of registered item handles ordered by server handle then item handle. - public IReadOnlyList ItemHandles => itemHandles + public IReadOnlyList ItemHandles => itemHandlesView ??= itemHandles .Values .OrderBy(handle => handle.ServerHandle) .ThenBy(handle => handle.ItemHandle) .ToArray(); /// Gets a read-only list of registered advice handles ordered by server handle, item handle, and advice kind. - public IReadOnlyList AdviceHandles => adviceHandles + public IReadOnlyList AdviceHandles => adviceHandlesView ??= adviceHandles .Values .OrderBy(handle => handle.ServerHandle) .ThenBy(handle => handle.ItemHandle) @@ -39,28 +64,38 @@ public sealed class MxAccessHandleRegistry string clientName) { serverHandles[serverHandle] = new RegisteredServerHandle(serverHandle, clientName); + serverHandlesView = null; } /// Unregisters a server handle and all associated item and advice handles from the registry. /// Handle returned by the worker. public void UnregisterServerHandle(int serverHandle) { - serverHandles.Remove(serverHandle); - - foreach (long key in itemHandles - .Where(pair => pair.Value.ServerHandle == serverHandle) - .Select(pair => pair.Key) - .ToArray()) + if (serverHandles.Remove(serverHandle)) { - itemHandles.Remove(key); + serverHandlesView = null; } - foreach (AdviceHandleKey key in adviceHandles - .Where(pair => pair.Value.ServerHandle == serverHandle) - .Select(pair => pair.Key) - .ToArray()) + // Per-server indexes: the removal cost is proportional to the handles + // this server actually owns, not to the size of the whole table. + if (itemKeysByServer.TryGetValue(serverHandle, out HashSet? ownedItemKeys)) { - adviceHandles.Remove(key); + itemKeysByServer.Remove(serverHandle); + + foreach (long itemKey in ownedItemKeys) + { + RemoveItemEntry(itemKey); + } + } + + if (advisedItemKeysByServer.TryGetValue(serverHandle, out HashSet? advisedItemKeys)) + { + advisedItemKeysByServer.Remove(serverHandle); + + foreach (long itemKey in advisedItemKeys) + { + RemoveAdviceEntries(itemKey); + } } } @@ -85,12 +120,32 @@ public sealed class MxAccessHandleRegistry string itemContext, bool hasItemContext) { - itemHandles[CreateItemKey(serverHandle, itemHandle)] = new RegisteredItemHandle( + long itemKey = CreateItemKey(serverHandle, itemHandle); + + if (itemHandles.TryGetValue(itemKey, out RegisteredItemHandle? existing)) + { + // Re-registering the same handle can carry a different item + // definition; drop the stale reverse-index entry first. + RemoveDefinitionIndexEntry(existing); + } + + itemHandles[itemKey] = new RegisteredItemHandle( serverHandle, itemHandle, itemDefinition, itemContext, hasItemContext); + + AddDefinitionIndexEntry(serverHandle, itemHandle, itemDefinition); + + if (!itemKeysByServer.TryGetValue(serverHandle, out HashSet? ownedItemKeys)) + { + ownedItemKeys = new HashSet(); + itemKeysByServer[serverHandle] = ownedItemKeys; + } + + ownedItemKeys.Add(itemKey); + itemHandlesView = null; } /// Removes an item handle and all associated advice handles from the registry. @@ -100,7 +155,19 @@ public sealed class MxAccessHandleRegistry int serverHandle, int itemHandle) { - itemHandles.Remove(CreateItemKey(serverHandle, itemHandle)); + long itemKey = CreateItemKey(serverHandle, itemHandle); + + if (RemoveItemEntry(itemKey) + && itemKeysByServer.TryGetValue(serverHandle, out HashSet? ownedItemKeys)) + { + ownedItemKeys.Remove(itemKey); + + if (ownedItemKeys.Count == 0) + { + itemKeysByServer.Remove(serverHandle); + } + } + RemoveAdviceHandles(serverHandle, itemHandle); } @@ -115,6 +182,34 @@ public sealed class MxAccessHandleRegistry return itemHandles.ContainsKey(CreateItemKey(serverHandle, itemHandle)); } + /// + /// Gets the item handles registered under the specified server handle for + /// the specified item definition, ordered by item handle. + /// + /// Handle returned by the worker. + /// Item definition name, compared ordinally. + /// + /// The matching item handles in ascending order — the same order a scan of + /// would visit them — or an empty list when the + /// definition is not registered for that server. MXAccess allows the same + /// tag to be added more than once under one server handle, so this is a + /// list rather than a single handle. The returned list is a live view of + /// the index and is only valid until the next registry mutation. + /// + public IReadOnlyList GetItemHandlesForDefinition( + int serverHandle, + string itemDefinition) + { + if (itemHandlesByDefinition.TryGetValue( + new ItemDefinitionKey(serverHandle, itemDefinition), + out List? handles)) + { + return handles; + } + + return Array.Empty(); + } + /// Registers an advice handle with the registry. /// Handle returned by the worker. /// Handle returned by the worker. @@ -125,10 +220,36 @@ public sealed class MxAccessHandleRegistry MxAccessAdviceKind adviceKind) { AdviceHandleKey key = new(serverHandle, itemHandle, adviceKind); + + if (!adviceHandles.ContainsKey(key)) + { + // Only index a key the primary table does not already hold, so the + // per-item list never accumulates duplicates. + long itemKey = CreateItemKey(serverHandle, itemHandle); + + if (!adviceKeysByItem.TryGetValue(itemKey, out List? itemAdviceKeys)) + { + itemAdviceKeys = new List(2); + adviceKeysByItem[itemKey] = itemAdviceKeys; + } + + itemAdviceKeys.Add(key); + + if (!advisedItemKeysByServer.TryGetValue(serverHandle, out HashSet? advisedItemKeys)) + { + advisedItemKeys = new HashSet(); + advisedItemKeysByServer[serverHandle] = advisedItemKeys; + } + + advisedItemKeys.Add(itemKey); + } + adviceHandles[key] = new RegisteredAdviceHandle( serverHandle, itemHandle, adviceKind); + + adviceHandlesView = null; } /// Removes all advice handles for the specified server and item handles from the registry. @@ -138,12 +259,17 @@ public sealed class MxAccessHandleRegistry int serverHandle, int itemHandle) { - foreach (AdviceHandleKey key in adviceHandles - .Where(pair => pair.Value.ServerHandle == serverHandle && pair.Value.ItemHandle == itemHandle) - .Select(pair => pair.Key) - .ToArray()) + long itemKey = CreateItemKey(serverHandle, itemHandle); + RemoveAdviceEntries(itemKey); + + if (advisedItemKeysByServer.TryGetValue(serverHandle, out HashSet? advisedItemKeys)) { - adviceHandles.Remove(key); + advisedItemKeys.Remove(itemKey); + + if (advisedItemKeys.Count == 0) + { + advisedItemKeysByServer.Remove(serverHandle); + } } } @@ -167,6 +293,81 @@ public sealed class MxAccessHandleRegistry return ((long)serverHandle << 32) | (uint)itemHandle; } + /// Removes one item from the primary table and the definition index. + /// Packed server/item key produced by . + /// if an item was removed; otherwise, . + private bool RemoveItemEntry(long itemKey) + { + if (!itemHandles.TryGetValue(itemKey, out RegisteredItemHandle? registered)) + { + return false; + } + + itemHandles.Remove(itemKey); + RemoveDefinitionIndexEntry(registered); + itemHandlesView = null; + + return true; + } + + /// Removes every advice handle recorded for one item from the primary table and the per-item index. + /// Packed server/item key produced by . + private void RemoveAdviceEntries(long itemKey) + { + if (!adviceKeysByItem.TryGetValue(itemKey, out List? itemAdviceKeys)) + { + return; + } + + adviceKeysByItem.Remove(itemKey); + + foreach (AdviceHandleKey key in itemAdviceKeys) + { + adviceHandles.Remove(key); + } + + adviceHandlesView = null; + } + + private void AddDefinitionIndexEntry( + int serverHandle, + int itemHandle, + string itemDefinition) + { + ItemDefinitionKey definitionKey = new(serverHandle, itemDefinition); + + if (!itemHandlesByDefinition.TryGetValue(definitionKey, out List? handles)) + { + handles = new List(1); + itemHandlesByDefinition[definitionKey] = handles; + } + + // Kept sorted so callers see the same order a scan of ItemHandles would. + int index = handles.BinarySearch(itemHandle); + + if (index < 0) + { + handles.Insert(~index, itemHandle); + } + } + + private void RemoveDefinitionIndexEntry(RegisteredItemHandle registered) + { + ItemDefinitionKey definitionKey = new(registered.ServerHandle, registered.ItemDefinition); + + if (!itemHandlesByDefinition.TryGetValue(definitionKey, out List? handles)) + { + return; + } + + handles.Remove(registered.ItemHandle); + + if (handles.Count == 0) + { + itemHandlesByDefinition.Remove(definitionKey); + } + } + private readonly struct AdviceHandleKey : IEquatable { private readonly int serverHandle; @@ -214,4 +415,50 @@ public sealed class MxAccessHandleRegistry } } } + + /// + /// Reverse-index key: a server handle plus an item definition, compared + /// ordinally to match the tag comparison the read path performs. + /// + private readonly struct ItemDefinitionKey : IEquatable + { + private readonly int serverHandle; + private readonly string itemDefinition; + + /// Initializes a new instance of the struct. + /// Handle returned by the worker. + /// Item definition name from MXAccess. + public ItemDefinitionKey( + int serverHandle, + string itemDefinition) + { + this.serverHandle = serverHandle; + this.itemDefinition = itemDefinition; + } + + /// + public bool Equals(ItemDefinitionKey other) + { + return serverHandle == other.serverHandle + && string.Equals(itemDefinition, other.itemDefinition, StringComparison.Ordinal); + } + + /// + public override bool Equals(object? obj) + { + return obj is ItemDefinitionKey other && Equals(other); + } + + /// + public override int GetHashCode() + { + unchecked + { + int hashCode = serverHandle; + hashCode = (hashCode * 397) ^ (itemDefinition ?? string.Empty).GetHashCode(); + + return hashCode; + } + } + } } diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessSession.cs index 78f1c0d..ee394ab 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessSession.cs @@ -990,23 +990,20 @@ public sealed class MxAccessSession : IDisposable out int itemHandle, out MxAccessValueCache.CachedValue cachedValue) { - // Linear scan — bulk-read sizes are small in practice and the registry - // is keyed by handle, not by tag. If profiling ever shows this hot, a - // reverse tag→handle map can be added on the registry side. - foreach (RegisteredItemHandle registered in handleRegistry.ItemHandles) + // Dictionary probe, not a scan: the registry keeps a reverse + // (server handle, item definition) → item handle index, so a bulk read + // no longer walks the whole item table once per tag. The index matches + // the tag ordinally and hands the handles back in ascending order — the + // same order the previous linear scan over the sorted ItemHandles view + // visited them — so the first advised-and-cached candidate still wins. + IReadOnlyList candidates = handleRegistry.GetItemHandlesForDefinition(serverHandle, tagAddress); + + for (int index = 0; index < candidates.Count; index++) { - if (registered.ServerHandle != serverHandle) - { - continue; - } + int candidate = candidates[index]; - if (!string.Equals(registered.ItemDefinition, tagAddress, StringComparison.Ordinal)) - { - continue; - } - - if (!handleRegistry.ContainsAdviceHandle(serverHandle, registered.ItemHandle, MxAccessAdviceKind.Plain) - && !handleRegistry.ContainsAdviceHandle(serverHandle, registered.ItemHandle, MxAccessAdviceKind.Supervisory)) + if (!handleRegistry.ContainsAdviceHandle(serverHandle, candidate, MxAccessAdviceKind.Plain) + && !handleRegistry.ContainsAdviceHandle(serverHandle, candidate, MxAccessAdviceKind.Supervisory)) { // Tag is added but not advised — no fresh OnDataChange will // arrive without us advising. Fall through to the snapshot @@ -1014,9 +1011,9 @@ public sealed class MxAccessSession : IDisposable continue; } - if (valueCache.TryGet(serverHandle, registered.ItemHandle, out cachedValue)) + if (valueCache.TryGet(serverHandle, candidate, out cachedValue)) { - itemHandle = registered.ItemHandle; + itemHandle = candidate; return true; } } From dc9424d3bd05e752876a4b8ae3f30a5a2919b96e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 16:58:54 -0400 Subject: [PATCH 27/41] =?UTF-8?q?perf(worker):=20message-driven=20completi?= =?UTF-8?q?on=20waits=20=E2=80=94=20the=20STA=20pumps=20continuously=20whi?= =?UTF-8?q?le=20waiting?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/MxAccessWorkerInstanceDesign.md | 82 ++++++++++-- .../MxAccess/MxAccessValueCacheTests.cs | 72 +++++++++++ .../MxAccessWriteCompletionCacheTests.cs | 45 +++++++ .../MxAccess/MxAccessValueCache.cs | 50 +++++++- .../MxAccess/MxAccessWriteCompletionCache.cs | 48 ++++++- .../Sta/StaWaitHelper.cs | 118 ++++++++++++++++++ 6 files changed, 396 insertions(+), 19 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Worker/Sta/StaWaitHelper.cs diff --git a/docs/MxAccessWorkerInstanceDesign.md b/docs/MxAccessWorkerInstanceDesign.md index 29211b5..4372eba 100644 --- a/docs/MxAccessWorkerInstanceDesign.md +++ b/docs/MxAccessWorkerInstanceDesign.md @@ -261,6 +261,54 @@ is still responsive. Shutdown marks the runtime as closing, wakes the pump, rejects new commands, cancels queued work, uninitializes COM on the STA, and waits for the thread to exit. +### Inner Completion Waits + +Two commands hold the STA while waiting for a COM event they just provoked: the +unary write path waits for its `OnWriteComplete` +(`MxAccessWriteCompletionCache.TryWaitForCompletion`), and `ReadBulk` waits per +tag for the first `OnDataChange` (`MxAccessValueCache.TryWaitForUpdate`). Both +run the same loop shape as the outer pump, for the same reason — the event they +are waiting for *is* a Windows message, so the thread must keep dispatching to +receive it: + +```text +loop: + pumpStep() # PeekMessage / TranslateMessage / DispatchMessage + if cache entry newer than baseline: return it + if now >= deadline: return the timed-out shape + + MsgWaitForMultipleObjectsEx( + cache_update_event, + min(remaining, 50 ms), + QS_ALLINPUT, + MWMO_INPUTAVAILABLE) +``` + +The idle slice is a Win32 wait (`StaWaitHelper.WaitForSignalOrMessages`), never +`Thread.Sleep`. A sleeping STA pumps no messages, so a sleep-polled loop could +only dispatch the awaited COM event at poll-tick granularity while stalling +*every other* event for the same tick — up to 1.5 s for a write completion and +up to `timeout_ms` per tag for `ReadBulk`. The Win32 wait returns the instant a +message needs pumping, so the apartment dispatches continuously for the whole +wait. Each cache also sets an `AutoResetEvent` from its update path (outside the +cache lock) so a cross-thread producer wakes the waiter immediately; in the live +worker the update arrives on the STA from inside `pumpStep` itself, and the +message wake is what carries it. + +The wait slice is capped at 50 ms so `pumpStep` runs periodically even when +nothing wakes the wait — a process with no STA message queue (unit tests drive +these caches from ordinary threads, standing in for the STA by updating the +cache from a fake `pumpStep`) must not block for a full poll interval. Timeouts, +deadline math, and return values are unchanged by the wait mechanism: an expired +write wait still yields the empty-`statuses` unconfirmed reply, and an expired +per-tag `ReadBulk` wait still reports its own timeout. + +The write wait's budget is `MxGateway:Worker:WriteCompletionWaitMilliseconds` +(default 1500). It is a bounded hold on the STA per unary write, so deployments +whose write workload is effectively fire-and-forget — no consumer reads the +reply's `statuses` — can lower it, or set `0` to skip the wait entirely and +reply on acceptance alone. + ## COM Creation The MXAccess analysis source at `C:\Users\dohertj2\Desktop\mxaccess` identifies @@ -378,16 +426,30 @@ If event conversion throws, catch it inside the event handler, record a structured `WorkerFault`, and keep the worker alive only if the fault policy allows it. -The event drain loop streams queued events as `WorkerEvent` frames. A single -event whose envelope exceeds the negotiated frame maximum is **undeliverable end -to end** — the pipe maximum sits only the envelope-overhead reserve above the -public gRPC cap, so a frame the pipe rejects would also be rejected on the -client-facing stream. The session therefore faults on it rather than dropping it -(a silent drop makes the event stream unfaithful, and a synthesized placeholder -is barred by the no-synthesized-events rule), but the death is structured: the -worker logs the event's identity — family, handles, worker sequence, and sizes, -never the value — writes a `WorkerFault` with category `ProtocolViolation` and -command method `EventDrain` carrying the same identity, and only then exits. +The event drain loop streams queued events as `WorkerEvent` frames. It is +**signal-driven, not polled**: `MxAccessEventQueue` carries a wake signal that +`Enqueue` and `RecordFault` release (outside the queue lock, so the STA's enqueue +stays a lock acquire plus a non-blocking release), and a drain that comes back +empty waits on that signal rather than sleeping. The signal is capped at one +pending wake, so a burst coalesces into a single wake and the waiter re-drains +everything that arrived — the loop must therefore re-check `DrainFault()` and +re-drain after every wait, never treat a wake as "exactly one event". The 25 ms +`EventDrainInterval` survives as the **fallback ceiling** on an unsignalled wait, +not as a latency floor: an event arriving at an idle worker is framed at signal +latency instead of waiting out a tick, an idle worker parks instead of waking 40 +times a second, and the interval only bounds how long the loop may sleep if some +future path mutates the queue without signalling. + +A single event whose envelope exceeds the negotiated frame maximum is +**undeliverable end to end** — the pipe maximum sits only the envelope-overhead +reserve above the public gRPC cap, so a frame the pipe rejects would also be +rejected on the client-facing stream. The session therefore faults on it rather +than dropping it (a silent drop makes the event stream unfaithful, and a +synthesized placeholder is barred by the no-synthesized-events rule), but the +death is structured: the worker logs the event's identity — family, handles, +worker sequence, and sizes, never the value — writes a `WorkerFault` with +category `ProtocolViolation` and command method `EventDrain` carrying the same +identity, and only then exits. Operator remediation is configuration: raise `MxGateway:Worker:MaxMessageBytes` for that workload. Other per-frame rejection codes keep their previous behavior because they indicate worker bugs, not workload size. diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessValueCacheTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessValueCacheTests.cs index fded4a2..6b2ef48 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessValueCacheTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessValueCacheTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using Google.Protobuf.WellKnownTypes; @@ -177,6 +178,77 @@ public sealed class MxAccessValueCacheTests Assert.Equal(1UL, value.Version); } + /// + /// Verifies the wait is message/signal-driven rather than sleep-polled: + /// with a poll interval far longer than the deadline, a value set from + /// another thread still wakes the wait and is returned well inside the + /// deadline. The sleep-polled loop this replaced would have sat blind for + /// the whole 10 s interval — on the STA that means 10 s with no Windows + /// messages dispatched, so the OnDataChange being waited for could not + /// have arrived at all — and then reported a timeout. + /// + [Fact] + public void TryWaitForUpdate_WakesOnCrossThreadSet_DespiteLongPollInterval() + { + MxAccessValueCache cache = new(); + Timestamp sourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow); + using ManualResetEventSlim waitEntered = new(false); + + Task setter = Task.Run(() => + { + // Handshake so the value cannot land before the wait starts — + // otherwise the first check would satisfy it and prove nothing. + waitEntered.Wait(TimeSpan.FromSeconds(5)); + Thread.Sleep(50); + cache.Set(7, 21, BuildEvent(7, 21, intValue: 8080, quality: 192, sourceTimestamp)); + }); + + Stopwatch elapsed = Stopwatch.StartNew(); + bool found = cache.TryWaitForUpdate( + serverHandle: 7, + itemHandle: 21, + sinceVersion: 0, + deadlineUtc: DateTime.UtcNow.AddSeconds(3), + pumpStep: () => waitEntered.Set(), + out MxAccessValueCache.CachedValue value, + pollIntervalMs: 10_000); + elapsed.Stop(); + setter.Wait(TimeSpan.FromSeconds(5)); + + Assert.True(found); + Assert.Equal(8080, value.Value.Int32Value); + Assert.True( + elapsed.Elapsed < TimeSpan.FromSeconds(2), + $"The wait should have woken on the cached value, not slept out the poll interval; took {elapsed.ElapsedMilliseconds} ms."); + } + + /// + /// Verifies the pump step keeps running throughout a wait that nothing + /// ever signals: the caller's poll interval is capped at the 50 ms + /// fallback tick, so a timing-out wait still pumps repeatedly instead of + /// once. This is what keeps the STA dispatching COM events in a process + /// whose message queue never wakes the wait, and it is the safety net + /// behind the ReadBulk per-tag timeout. + /// + [Fact] + public void TryWaitForUpdate_KeepsPumping_WhenPollIntervalExceedsTheFallbackTick() + { + MxAccessValueCache cache = new(); + int pumpCalls = 0; + + bool found = cache.TryWaitForUpdate( + serverHandle: 7, + itemHandle: 21, + sinceVersion: 0, + deadlineUtc: DateTime.UtcNow.AddMilliseconds(400), + pumpStep: () => Interlocked.Increment(ref pumpCalls), + out _, + pollIntervalMs: 10_000); + + Assert.False(found); + Assert.True(pumpCalls >= 3, $"Expected repeated pumping across the 400 ms wait, saw {pumpCalls} calls."); + } + private static MxEvent BuildEvent( int serverHandle, int itemHandle, diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessWriteCompletionCacheTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessWriteCompletionCacheTests.cs index 670bc5f..fd817f1 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessWriteCompletionCacheTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessWriteCompletionCacheTests.cs @@ -1,4 +1,7 @@ using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; using Google.Protobuf.Collections; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Worker.MxAccess; @@ -109,6 +112,48 @@ public sealed class MxAccessWriteCompletionCacheTests Assert.Equal(55, Assert.Single(statuses).Detail); } + /// + /// Verifies the wait is message/signal-driven rather than sleep-polled: + /// with a poll interval far longer than the deadline, a completion + /// recorded from another thread still wakes the wait and is returned well + /// inside the deadline. The sleep-polled loop this replaced would have sat + /// blind for the whole 10 s interval — on the STA that means 10 s with no + /// Windows messages dispatched — and then reported a timeout. + /// + [Fact] + public void TryWaitForCompletion_WakesOnCrossThreadRecord_DespiteLongPollInterval() + { + MxAccessWriteCompletionCache cache = new(); + using ManualResetEventSlim waitEntered = new(false); + + Task recorder = Task.Run(() => + { + // Handshake so the completion cannot land before the wait starts — + // otherwise the first check would satisfy it and prove nothing. + waitEntered.Wait(TimeSpan.FromSeconds(5)); + Thread.Sleep(50); + cache.Record(7, 21, BuildStatuses(detail: 8080)); + }); + + Stopwatch elapsed = Stopwatch.StartNew(); + bool found = cache.TryWaitForCompletion( + 7, + 21, + sinceVersion: 0UL, + deadlineUtc: DateTime.UtcNow.AddSeconds(3), + pumpStep: () => waitEntered.Set(), + out RepeatedField statuses, + pollIntervalMs: 10_000); + elapsed.Stop(); + recorder.Wait(TimeSpan.FromSeconds(5)); + + Assert.True(found); + Assert.Equal(8080, Assert.Single(statuses).Detail); + Assert.True( + elapsed.Elapsed < TimeSpan.FromSeconds(2), + $"The wait should have woken on the recorded completion, not slept out the poll interval; took {elapsed.ElapsedMilliseconds} ms."); + } + /// Verifies that Record stores an independent clone of the caller's rows. [Fact] public void Record_ClonesStatuses() diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessValueCache.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessValueCache.cs index f82429b..abe133f 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessValueCache.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessValueCache.cs @@ -4,6 +4,7 @@ using System.Threading; using Google.Protobuf.Collections; using Google.Protobuf.WellKnownTypes; using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.MxGateway.Worker.Sta; namespace ZB.MOM.WW.MxGateway.Worker.MxAccess; @@ -26,6 +27,18 @@ public sealed class MxAccessValueCache private readonly Dictionary entries = new(); private readonly object syncRoot = new(); + // Set by Set() so a waiter blocked in TryWaitForUpdate wakes the moment a + // value lands instead of after the next blind poll tick. In the live worker + // the value is recorded from inside the waiter's own pumpStep (COM + // dispatches OnDataChange on the STA), so the message wake is what does the + // work there; this handle covers the cross-thread producers — the tests, and + // any future off-STA writer. Not IDisposable on purpose: the cache is + // co-owned by MxAccessBaseEventSink and MxAccessSession with no single + // teardown owner, and a late COM callback must never Set a disposed handle. + // One kernel event per session (one session per worker process), released by + // the handle's finalizer. + private readonly AutoResetEvent updateSignal = new(false); + /// Records a fresh OnDataChange payload for the given handle pair. /// MXAccess server handle. /// MXAccess item handle. @@ -68,6 +81,11 @@ public sealed class MxAccessValueCache cachedTimestamp, mxEvent.Statuses.Clone()); } + + // Signaled outside the lock: the waiter re-takes syncRoot (through + // TryGet) the instant it wakes, so signaling while holding it would hand + // it guaranteed contention. + updateSignal.Set(); } /// Tries to read the most recent cached value for the handle pair. @@ -107,16 +125,28 @@ public sealed class MxAccessValueCache /// /// Waits until the cache entry's version exceeds - /// or the deadline elapses, calling on every poll + /// or the deadline elapses, calling on every /// iteration so the worker's STA can dispatch the inbound MXAccess message. /// + /// + /// The idle part of each iteration blocks in + /// rather than + /// Thread.Sleep: a sleeping STA pumps no Windows messages, so a + /// ReadBulk waiting on the first OnDataChange would only dispatch it at + /// poll-tick granularity — and it holds the apartment for up to + /// timeout_ms per tag, stalling every other COM event with it. The + /// Win32 wait returns the instant a message needs pumping (or the value + /// is recorded from another thread), so the apartment keeps dispatching + /// continuously for the whole wait. The per-tag timeout, the deadline + /// math, and both return shapes are unchanged. + /// /// MXAccess server handle. /// MXAccess item handle. /// Version snapshot captured before the wait. /// Absolute UTC deadline. /// Action that pumps any pending Windows messages. /// The cached value if the update was received before the deadline. - /// How long to sleep between pump cycles. Default 5 ms. + /// Upper bound on one idle wait slice — the cadence at which runs when nothing wakes the wait. Default 5 ms, itself capped at . /// if an update newer than arrived before the deadline; otherwise . public bool TryWaitForUpdate( int serverHandle, @@ -134,6 +164,12 @@ public sealed class MxAccessValueCache while (true) { + // Pumped unconditionally at the top of every iteration — including + // the one that immediately follows a signaled wait, and the very + // first before any waiting. Dispatching the message is what fills + // the cache in the live worker, and unit tests stand in for the STA + // by driving the cache from their fake pumpStep, so this call must + // never be conditional on a real message queue. pumpStep(); if (TryGet(serverHandle, itemHandle, out value) && value.Version > sinceVersion) @@ -141,12 +177,18 @@ public sealed class MxAccessValueCache return true; } - if (DateTime.UtcNow >= deadlineUtc) + // Same expiry test as the sleep-polled version: remaining <= zero is + // exactly DateTime.UtcNow >= deadlineUtc. `value` keeps whatever the + // TryGet above left in it on this path, as before. + TimeSpan remaining = deadlineUtc - DateTime.UtcNow; + if (remaining <= TimeSpan.Zero) { return false; } - Thread.Sleep(pollIntervalMs); + StaWaitHelper.WaitForSignalOrMessages( + updateSignal, + StaWaitHelper.ClampWaitMilliseconds(remaining, pollIntervalMs)); } } diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessWriteCompletionCache.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessWriteCompletionCache.cs index 1bb5e35..bfbfeaa 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessWriteCompletionCache.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessWriteCompletionCache.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Threading; using Google.Protobuf.Collections; using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.MxGateway.Worker.Sta; namespace ZB.MOM.WW.MxGateway.Worker.MxAccess; @@ -25,6 +26,18 @@ public sealed class MxAccessWriteCompletionCache private readonly Dictionary entries = new(); private readonly object syncRoot = new(); + // Set by Record so a waiter blocked in TryWaitForCompletion wakes the moment + // a completion lands instead of after the next blind poll tick. In the live + // worker the completion is recorded from inside the waiter's own pumpStep + // (COM dispatches OnWriteComplete on the STA), so the message wake is what + // does the work there; this handle covers the cross-thread producers — the + // tests, and any future off-STA recorder. Not IDisposable on purpose: the + // cache is co-owned by MxAccessBaseEventSink and MxAccessSession with no + // single teardown owner, and a late COM callback must never Set a disposed + // handle. One kernel event per session (one session per worker process), + // released by the handle's finalizer. + private readonly AutoResetEvent completionSignal = new(false); + /// Records the status rows of a fresh OnWriteComplete callback for the given handle pair. /// MXAccess server handle. /// MXAccess item handle. @@ -47,6 +60,10 @@ public sealed class MxAccessWriteCompletionCache : 1UL; entries[key] = new CompletionEntry(version, statuses.Clone()); } + + // Signaled outside the lock: the waiter re-takes syncRoot the instant it + // wakes, so signaling while holding it would hand it guaranteed contention. + completionSignal.Set(); } /// Returns the current completion version for a handle pair, or 0 if none was recorded. @@ -66,19 +83,29 @@ public sealed class MxAccessWriteCompletionCache } /// - /// Polls for a completion newer than until it + /// Waits for a completion newer than until it /// arrives or the deadline elapses, calling on every - /// poll iteration so the worker's STA can dispatch the inbound MXAccess + /// iteration so the worker's STA can dispatch the inbound MXAccess /// OnWriteComplete message. Same loop shape as /// . /// + /// + /// The idle part of each iteration blocks in + /// rather than + /// Thread.Sleep: a sleeping STA pumps no Windows messages, so the + /// inbound OnWriteComplete could only ever be dispatched at poll-tick + /// granularity. The Win32 wait returns the instant a message needs + /// pumping (or the completion is recorded from another thread), so the + /// apartment keeps dispatching COM events continuously for the whole + /// wait. Timeouts, deadline math, and both return shapes are unchanged. + /// /// MXAccess server handle. /// MXAccess item handle. /// Version snapshot captured before the write COM call. /// Absolute UTC deadline. /// Action that pumps any pending Windows messages. /// The recorded status rows if a completion arrived before the deadline; empty otherwise. - /// How long to sleep between pump cycles. Default 5 ms. + /// Upper bound on one idle wait slice — the cadence at which runs when nothing wakes the wait. Default 5 ms, itself capped at . /// if a completion newer than arrived before the deadline; otherwise . public bool TryWaitForCompletion( int serverHandle, @@ -96,6 +123,12 @@ public sealed class MxAccessWriteCompletionCache while (true) { + // Pumped unconditionally at the top of every iteration — including + // the one that immediately follows a signaled wait, and the very + // first before any waiting. Dispatching the message is what fills + // the cache in the live worker, and unit tests stand in for the STA + // by recording the completion from inside their fake pumpStep, so + // this call must never be conditional on a real message queue. pumpStep(); lock (syncRoot) @@ -108,13 +141,18 @@ public sealed class MxAccessWriteCompletionCache } } - if (DateTime.UtcNow >= deadlineUtc) + // Same expiry test as the sleep-polled version: remaining <= zero is + // exactly DateTime.UtcNow >= deadlineUtc. + TimeSpan remaining = deadlineUtc - DateTime.UtcNow; + if (remaining <= TimeSpan.Zero) { statuses = new RepeatedField(); return false; } - Thread.Sleep(pollIntervalMs); + StaWaitHelper.WaitForSignalOrMessages( + completionSignal, + StaWaitHelper.ClampWaitMilliseconds(remaining, pollIntervalMs)); } } diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Sta/StaWaitHelper.cs b/src/ZB.MOM.WW.MxGateway.Worker/Sta/StaWaitHelper.cs new file mode 100644 index 0000000..7549d29 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Worker/Sta/StaWaitHelper.cs @@ -0,0 +1,118 @@ +using System; +using System.Runtime.InteropServices; +using System.Threading; +using Microsoft.Win32.SafeHandles; + +namespace ZB.MOM.WW.MxGateway.Worker.Sta; + +/// +/// Message-aware blocking wait shared by STA code that must sit idle without +/// starving the Windows message queue. owns the +/// runtime's outer idle wait; this helper serves the inner waits performed +/// by commands that already hold the STA — the write-completion wait and the +/// ReadBulk first-value wait — which need the same "wake on a signal or on +/// an inbound message" behavior without owning a pump instance. +/// +/// +/// Deliberately separate from rather than a new +/// method on it: the two caches that use this are plain per-session objects +/// with no pump dependency (they take the pump step as an injected delegate +/// so tests can drive them off the STA). Keeping the helper in this folder +/// keeps every user32 wait signature in the worker reviewable in one +/// place; the declaration below mirrors the pump's exactly. +/// +internal static class StaWaitHelper +{ + /// + /// Ceiling on the blind fallback tick. A caller's poll interval is + /// clamped to this so no wait can go longer than 50 ms without calling + /// its pump step, even when the process has no message queue to wake it + /// (unit-test threads) and the wait handle is never signaled. + /// + internal const int MaxFallbackTickMilliseconds = 50; + + private const uint MsgWaitFailed = 0xFFFFFFFF; + private const uint MwmoInputAvailable = 0x0004; + private const uint QsAllInput = 0x04FF; + + /// + /// Blocks the calling thread until is set, a + /// Windows message is available to pump, or the timeout elapses — + /// whichever comes first. Unlike this + /// never blinds the thread to inbound COM event messages: the wait + /// returns the instant one arrives, so the caller can dispatch it. + /// + /// Wait handle set by whoever produces the awaited result. + /// Maximum wait; values at or below zero return immediately. + internal static void WaitForSignalOrMessages( + WaitHandle signal, + int timeoutMilliseconds) + { + if (signal is null) + { + throw new ArgumentNullException(nameof(signal)); + } + + if (timeoutMilliseconds <= 0) + { + return; + } + + SafeWaitHandle safeHandle = signal.SafeWaitHandle; + IntPtr[] handles = [safeHandle.DangerousGetHandle()]; + uint result = MsgWaitForMultipleObjectsEx( + (uint)handles.Length, + handles, + (uint)timeoutMilliseconds, + QsAllInput, + MwmoInputAvailable); + + // The handle is a field of a live caller object, so it cannot be + // collected mid-call; KeepAlive states that dependency explicitly + // rather than relying on it. + GC.KeepAlive(signal); + + if (result == MsgWaitFailed) + { + // Degrade to the pre-existing blind sleep instead of throwing or + // hot-spinning: the caller is a bounded, deadline-driven poll loop, + // and a failed wait must not turn a timed-out write into a fault. + Thread.Sleep(timeoutMilliseconds); + } + } + + /// + /// Computes the wait slice for one iteration of a deadline-driven poll + /// loop: never past the deadline, never longer than the caller's poll + /// interval, and never longer than . + /// + /// Time left before the caller's absolute deadline. + /// The caller's requested pump cadence. + /// Milliseconds to wait; zero or less when the deadline has passed. + internal static int ClampWaitMilliseconds( + TimeSpan remaining, + int pollIntervalMilliseconds) + { + int tick = pollIntervalMilliseconds < 1 + ? 1 + : Math.Min(pollIntervalMilliseconds, MaxFallbackTickMilliseconds); + + double remainingMilliseconds = remaining.TotalMilliseconds; + if (remainingMilliseconds <= 0d) + { + return 0; + } + + return remainingMilliseconds < tick + ? (int)Math.Ceiling(remainingMilliseconds) + : tick; + } + + [DllImport("user32.dll", SetLastError = true)] + private static extern uint MsgWaitForMultipleObjectsEx( + uint count, + IntPtr[] handles, + uint milliseconds, + uint wakeMask, + uint flags); +} From f4a6cb1db2f662a301dd0abae67d666228c2c954 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 16:59:13 -0400 Subject: [PATCH 28/41] =?UTF-8?q?perf(worker):=20signal-driven=20event=20d?= =?UTF-8?q?rain=20=E2=80=94=20removes=20the=2025=20ms=20latency=20floor=20?= =?UTF-8?q?and=20idle=20wakeups?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Ipc/WorkerPipeSessionTests.cs | 55 ++++++++ .../MxAccess/MxAccessEventQueueTests.cs | 119 +++++++++++++++++ .../TestSupport/FakeRuntimeSession.cs | 75 +++++++++++ .../Ipc/WorkerPipeSession.cs | 15 ++- .../MxAccess/IWorkerRuntimeSession.cs | 17 +++ .../MxAccess/MxAccessEventQueue.cs | 120 ++++++++++++++---- .../MxAccess/MxAccessStaSession.cs | 6 + 7 files changed, 383 insertions(+), 24 deletions(-) diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs index 7cbff85..4f6703c 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs @@ -311,6 +311,61 @@ public sealed class WorkerPipeSessionTests await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } + /// + /// The event drain loop waits on the runtime's wake signal instead of sleeping a fixed tick, + /// so an event enqueued at an idle worker is framed as soon as it is enqueued rather than up + /// to EventDrainInterval later. The fake's wait honours only the signal here, so the + /// event reaching the pipe is proof the enqueue woke the loop — a poll-driven loop would + /// never run again, and the test would fail on its cancellation deadline instead of passing + /// on a fallback tick that happened to fire. The loop is left parked on that wait before the + /// enqueue, which also makes the recorded fallback ceiling assertable. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task RunAsync_EventAfterIdle_DrainLoopWakesOnSignalNotOnPollTick() + { + using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10)); + using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); + FakeRuntimeSession runtime = new() + { + WaitForEventsOnSignalOnly = true, + }; + + // A far-off heartbeat interval keeps the drain loop the only thing that can produce a frame + // after the first beat, so nothing else can mask a drain loop that never woke. + WorkerPipeSession session = CreatePipeSession( + pipePair.WorkerStream, + runtime, + new WorkerPipeSessionOptions + { + HeartbeatInterval = TimeSpan.FromMinutes(5), + HeartbeatGrace = TimeSpan.FromSeconds(30), + }); + Task runTask = session.RunAsync(cancellation.Token); + await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); + + // Park the drain loop: it drains empty once and then waits. Enqueuing before it parks would + // let the first drain pass find the event, which proves nothing about the wake. + while (runtime.LastWaitForEventsTimeout is null) + { + await Task.Delay(TimeSpan.FromMilliseconds(5), cancellation.Token); + } + + runtime.EnqueueEvent(CreateWorkerEvent(sequence: 7)); + + WorkerEnvelope workerEvent = await ReadUntilAsync( + pipePair.GatewayReader, + WorkerEnvelope.BodyOneofCase.WorkerEvent, + cancellation.Token); + + Assert.Equal(7UL, workerEvent.WorkerEvent.Event.WorkerSequence); + + // The 25 ms survives as the ceiling the loop passes to every wait, not as a poll period. + Assert.Equal(TimeSpan.FromMilliseconds(25), runtime.LastWaitForEventsTimeout); + + await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); + } + /// /// Verifies that a Ping control command is answered on the worker side /// (not dispatched to the STA) with an OK reply that echoes the ping diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventQueueTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventQueueTests.cs index 1492ec0..e783fde 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventQueueTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventQueueTests.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Worker.MxAccess; @@ -300,6 +302,123 @@ public sealed class MxAccessEventQueueTests Assert.Equal(WorkerFaultCategory.MxaccessEventConversionFailed, queue.Fault?.Category); } + // Wake-signal timings. The fallback is far longer than the patience deliberately: every wait + // below asserts "the signal completed this", which is only a faithful claim while the fallback + // timeout cannot have completed it within the patience window. The patience itself is generous + // so a loaded CI box cannot fail a test that is not about latency. + private static readonly TimeSpan WakeFallback = TimeSpan.FromSeconds(30); + private static readonly TimeSpan WakePatience = TimeSpan.FromSeconds(5); + + /// + /// Verifies the queue wakes a parked waiter as soon as an event is enqueued, rather than + /// leaving it to time out. This is what removes the drain loop's latency floor: before the + /// signal existed, an event arriving at an idle queue waited out the loop's whole poll tick. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task WaitForEventsAsync_EnqueueAfterIdle_CompletesWithoutWaitingTheFallback() + { + MxAccessEventQueue queue = new(capacity: 4); + + Task wait = queue.WaitForEventsAsync(WakeFallback, CancellationToken.None); + Assert.False(wait.IsCompleted); + + queue.Enqueue(CreateEvent(MxEventFamily.OnDataChange, itemHandle: 10)); + + Assert.Same(wait, await Task.WhenAny(wait, Task.Delay(WakePatience))); + await wait; + Assert.Equal(1, queue.Count); + } + + /// + /// Verifies a fault recorded while the waiter is parked wakes it too. The drain loop + /// discovers faults by calling DrainFault() at the top of each pass, so without this + /// signal an overflow or conversion fault would not be reported until the loop's fallback + /// tick expired. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task WaitForEventsAsync_RecordFaultWhileParked_WakesTheWaiter() + { + MxAccessEventQueue queue = new(capacity: 4); + + Task wait = queue.WaitForEventsAsync(WakeFallback, CancellationToken.None); + Assert.False(wait.IsCompleted); + + queue.RecordFault(new WorkerFault + { + Category = WorkerFaultCategory.MxaccessEventConversionFailed, + }); + + Assert.Same(wait, await Task.WhenAny(wait, Task.Delay(WakePatience))); + await wait; + Assert.NotNull(queue.DrainFault()); + } + + /// + /// Verifies the one-permit cap loses no wakeups. A burst that lands while nobody is waiting + /// leaves exactly one pending wake — the waiter that consumes it drains the whole burst, so + /// coalescing costs nothing — and that consumed wake is not replayed: the next wait parks + /// until a new enqueue signals it. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task WaitForEventsAsync_BurstWhileNoWaiter_CoalescesToOneWakeThatLosesNothing() + { + MxAccessEventQueue queue = new(capacity: 16); + for (int itemHandle = 0; itemHandle < 5; itemHandle++) + { + queue.Enqueue(CreateEvent(MxEventFamily.OnDataChange, itemHandle)); + } + + Task firstWait = queue.WaitForEventsAsync(WakeFallback, CancellationToken.None); + Assert.Same(firstWait, await Task.WhenAny(firstWait, Task.Delay(WakePatience))); + await firstWait; + + // One wake, the whole burst: the waiter re-drains everything queued, which is why capping + // the signal at a single permit cannot drop an event. + Assert.Equal(5, queue.Drain(maxEvents: 0).Count); + + Task secondWait = queue.WaitForEventsAsync(WakeFallback, CancellationToken.None); + Assert.False(secondWait.IsCompleted); + + queue.Enqueue(CreateEvent(MxEventFamily.OnDataChange, itemHandle: 99)); + + Assert.Same(secondWait, await Task.WhenAny(secondWait, Task.Delay(WakePatience))); + await secondWait; + } + + /// + /// Verifies the timeout still bounds an unsignalled wait: the fallback survives as a ceiling + /// on how long a caller may sleep, so a state change reached by some future path that does + /// not signal is still observed on the next pass rather than never. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task WaitForEventsAsync_WithNoSignal_CompletesAtTheFallbackTimeout() + { + MxAccessEventQueue queue = new(capacity: 4); + + Task wait = queue.WaitForEventsAsync(TimeSpan.FromMilliseconds(25), CancellationToken.None); + + Assert.Same(wait, await Task.WhenAny(wait, Task.Delay(WakePatience))); + await wait; + } + + /// Verifies a cancelled wait unwinds instead of hanging until the fallback expires. + /// A task that represents the asynchronous operation. + [Fact] + public async Task WaitForEventsAsync_WhenCancelled_Throws() + { + MxAccessEventQueue queue = new(capacity: 4); + using CancellationTokenSource cancellation = new(); + + Task wait = queue.WaitForEventsAsync(WakeFallback, cancellation.Token); + cancellation.Cancel(); + + await Assert.ThrowsAnyAsync(async () => await wait); + } + // Mirrors MxAccessEventQueue's per-event repeated-field allowance. Kept local rather than made // public on the queue: the byte-budget tests state their budgets in units of that charge, so a // change to it should surface here as a failing bound instead of silently moving with the code. diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs index a992b12..84edc37 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs @@ -24,6 +24,13 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession private readonly object gate = new(); private readonly Queue events = new(); private readonly List cancelledCorrelationIds = new(); + + // Mirrors MxAccessEventQueue's coalesced wake signal so the drain loop under test is driven the + // same way it is in production: EnqueueEvent(s) releases one permit, WaitForEventsAsync consumes + // it. Never disposed — the drain loop can still be parked on it while Dispose runs, and a + // disposed SemaphoreSlim would turn that shutdown into an ObjectDisposedException. + private readonly SemaphoreSlim eventSignal = new(0, 1); + private TimeSpan? lastWaitForEventsTimeout; private WorkerRuntimeHeartbeatSnapshot snapshot = new( DateTimeOffset.UtcNow, pendingCommandCount: 0, @@ -263,6 +270,52 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession } } + /// + /// When set, honours only the wake signal and cancellation, + /// never the fallback timeout. A drain loop that ships an event while this is set can only + /// have been woken by the enqueue signal, which is what makes "the drain is signal-driven, + /// not poll-driven" assertable without racing the 25 ms fallback tick. + /// + public bool WaitForEventsOnSignalOnly { get; set; } + + /// + /// The timeout argument of the most recent call, so + /// a test can assert the drain loop still passes its fallback ceiling. + /// + public TimeSpan? LastWaitForEventsTimeout + { + get + { + lock (gate) + { + return lastWaitForEventsTimeout; + } + } + } + + /// + public Task WaitForEventsAsync(TimeSpan timeout, CancellationToken cancellationToken) + { + lock (gate) + { + lastWaitForEventsTimeout = timeout; + } + + if (BackingQueue is not null) + { + // Tests that drive a real queue enqueue into it directly, so the real queue owns the + // wake signal too. + return BackingQueue.WaitForEventsAsync(timeout, cancellationToken); + } + + if (WaitForEventsOnSignalOnly) + { + return eventSignal.WaitAsync(cancellationToken); + } + + return eventSignal.WaitAsync(timeout, cancellationToken); + } + /// public WorkerFault? DrainFault() { @@ -370,6 +423,8 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession { events.Enqueue(workerEvent); } + + SignalWake(); } /// @@ -387,6 +442,26 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession events.Enqueue(workerEvent); } } + + SignalWake(); + } + + // Coalesced wake, released outside the gate exactly as MxAccessEventQueue does. + private void SignalWake() + { + if (eventSignal.CurrentCount > 0) + { + return; + } + + try + { + eventSignal.Release(); + } + catch (SemaphoreFullException) + { + // A concurrent enqueue already published the pending wake this call wanted. + } } /// diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs index e91442a..8934946 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs @@ -15,6 +15,11 @@ namespace ZB.MOM.WW.MxGateway.Worker.Ipc; public sealed class WorkerPipeSession { + // Fallback ceiling for the event drain loop's wait — not a poll period. MxAccessEventQueue + // signals on enqueue (and on a recorded fault), so the loop wakes as soon as there is something + // to ship instead of paying up to this interval of latency on every burst from idle, and an + // idle worker parks instead of waking 40x/s. The interval survives only as the bound on how + // long the loop may sleep unsignalled, which keeps its DrainFault() poll on a known cadence. private static readonly TimeSpan EventDrainInterval = TimeSpan.FromMilliseconds(25); private static readonly TimeSpan BackgroundTaskStopTimeout = TimeSpan.FromSeconds(1); private const uint EventDrainBatchSize = 128; @@ -367,7 +372,15 @@ public sealed class WorkerPipeSession IReadOnlyList events = runtimeSession.DrainEvents(EventDrainBatchSize); if (events.Count == 0) { - await Task.Delay(EventDrainInterval, cancellationToken).ConfigureAwait(false); + // Wait on the queue's wake signal rather than sleeping a fixed tick: an event + // enqueued by the STA completes this immediately, so the first event of a burst is + // framed at signal latency instead of waiting out EventDrainInterval, and a session + // with no traffic stops waking at all. The wait's outcome is intentionally ignored — + // whether a signal or the fallback ended it, the next pass re-checks DrainFault() + // and re-drains, which is also why one coalesced wake for many enqueues is safe. + await runtimeSession + .WaitForEventsAsync(EventDrainInterval, cancellationToken) + .ConfigureAwait(false); continue; } diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/IWorkerRuntimeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/IWorkerRuntimeSession.cs index bf1f252..cabfd82 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/IWorkerRuntimeSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/IWorkerRuntimeSession.cs @@ -57,6 +57,23 @@ public interface IWorkerRuntimeSession : IDisposable /// The drained events and the truncation facts describing what stayed queued. WorkerEventDrainResult DrainEvents(uint maxEvents, int maxTotalBytes); + /// + /// Waits until the outbound event queue has something for the caller to look at (an + /// enqueued event or a recorded fault), the fallback timeout elapses, or the token is + /// cancelled. Lets a drain loop be signal-driven instead of polling. + /// + /// + /// Declared on the interface because the pipe session only ever sees an + /// , never the queue behind it; .NET Framework 4.8 has + /// no default interface members, so every implementation supplies it. The wait's outcome is + /// not surfaced: the caller re-drains and re-checks after every + /// wait, because the signal is coalesced and the timeout is a ceiling, not a poll period. + /// + /// Maximum time to wait before the wait completes unsignalled. + /// Token cancelling the wait. + /// A task that completes when the queue is signalled or the timeout elapses. + Task WaitForEventsAsync(TimeSpan timeout, CancellationToken cancellationToken); + /// /// Drains a pending fault from the queue, if any. /// diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs index ed27a67..17f1d67 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; using Google.Protobuf.WellKnownTypes; using ZB.MOM.WW.MxGateway.Contracts.Proto; @@ -38,6 +40,17 @@ public sealed class MxAccessEventQueue private readonly int capacity; private readonly Queue events; private readonly object syncRoot = new(); + + // Wake signal for the worker's event drain loop (see WorkerPipeSession.RunEventDrainLoopAsync). + // Capped at one permit, so it is a level ("there is something to look at"), never a count: a + // burst of enqueues coalesces into one wake and the waiter re-drains everything that arrived. + // Never released while holding syncRoot — the STA thread enqueues here, so its path must stay a + // lock acquire plus a non-blocking Release. Deliberately not disposed: the queue outlives its + // waiter, no wait handle is ever materialized (nothing touches AvailableWaitHandle), and making + // the queue IDisposable would only add a lifecycle race between the STA's Enqueue and the + // drain loop's wait. WorkerFrameWriter's _writeLock is held the same way. + private readonly SemaphoreSlim eventSignal = new SemaphoreSlim(initialCount: 0, maxCount: 1); + private ulong lastEventSequence; private WorkerFault? fault; private bool faultDrained; @@ -142,33 +155,67 @@ public sealed class MxAccessEventQueue throw new ArgumentNullException(nameof(mxEvent)); } - lock (syncRoot) + try { - if (fault is not null) + lock (syncRoot) { - throw new InvalidOperationException("MXAccess outbound event queue is faulted."); + if (fault is not null) + { + throw new InvalidOperationException("MXAccess outbound event queue is faulted."); + } + + if (events.Count >= capacity) + { + fault = CreateOverflowFault(); + throw new MxAccessEventQueueOverflowException(capacity); + } + + // WRK-11: stamp the sequence/timestamp on the caller's own event and + // enqueue that same instance under the lock instead of cloning. See + // the ownership invariant on the class summary — the caller hands the + // event over exclusively, so a defensive Clone() here is pure + // overhead on the hottest path. + mxEvent.WorkerSequence = ++lastEventSequence; + mxEvent.WorkerTimestamp = Timestamp.FromDateTime(DateTime.UtcNow); + + WorkerEvent workerEvent = new() + { + Event = mxEvent, + }; + events.Enqueue(workerEvent); } - - if (events.Count >= capacity) - { - fault = CreateOverflowFault(); - throw new MxAccessEventQueueOverflowException(capacity); - } - - // WRK-11: stamp the sequence/timestamp on the caller's own event and - // enqueue that same instance under the lock instead of cloning. See - // the ownership invariant on the class summary — the caller hands the - // event over exclusively, so a defensive Clone() here is pure - // overhead on the hottest path. - mxEvent.WorkerSequence = ++lastEventSequence; - mxEvent.WorkerTimestamp = Timestamp.FromDateTime(DateTime.UtcNow); - - WorkerEvent workerEvent = new() - { - Event = mxEvent, - }; - events.Enqueue(workerEvent); } + finally + { + // Wake the drain loop outside the lock, on every exit path: the success path has an + // event to ship, and the overflow path has just self-recorded the fault the loop + // reports through DrainFault(). Signalling in a finally also keeps the already-faulted + // rejection harmless — a spurious wake only costs the waiter one empty re-drain. + SignalWake(); + } + } + + /// + /// Waits for a wake signal, the fallback timeout, or cancellation, so the worker's event + /// drain loop is signal-driven instead of polling. and + /// both signal, so an event arriving at an idle queue is picked + /// up immediately rather than at the caller's next tick, and an idle queue costs no wakeups + /// at all. + /// + /// + /// The wait's outcome is deliberately not surfaced: the caller must re-drain (and re-check + /// ) after every wait regardless of whether a signal or the timeout + /// ended it, because the signal is coalesced — one pending wake can stand for any number of + /// enqueues. The timeout is therefore a ceiling on how long the caller may sleep, not a + /// poll period: it bounds how late a state change reaches the caller if some future path + /// mutates the queue without signalling. + /// + /// Maximum time to wait before the wait completes unsignalled. + /// Token cancelling the wait. + /// A task that completes when the queue is signalled or the timeout elapses. + public Task WaitForEventsAsync(TimeSpan timeout, CancellationToken cancellationToken) + { + return eventSignal.WaitAsync(timeout, cancellationToken); } /// @@ -291,6 +338,11 @@ public sealed class MxAccessEventQueue { fault ??= workerFault.Clone(); } + + // The drain loop discovers faults by polling DrainFault() at the top of each pass, so a + // fault recorded while it is parked must wake it — otherwise the fault would be reported + // no sooner than the loop's fallback tick. Signalled outside the lock, as Enqueue does. + SignalWake(); } /// @@ -311,6 +363,28 @@ public sealed class MxAccessEventQueue } } + private void SignalWake() + { + // Fast path: a wake is already pending and no waiter has consumed it yet, so releasing + // again would only throw. This keeps a burst of enqueues on the STA at a single volatile + // read once the first event has signalled. + if (eventSignal.CurrentCount > 0) + { + return; + } + + try + { + eventSignal.Release(); + } + catch (SemaphoreFullException) + { + // A concurrent producer won the race between the check above and this Release. The + // pending wake it published is exactly the wake this call wanted, so there is nothing + // to do — the signal is a level, not a count. + } + } + private WorkerFault CreateOverflowFault() { string message = $"MXAccess outbound event queue reached capacity {capacity}."; diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs index 90e622f..03ddf16 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs @@ -455,6 +455,12 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession return eventQueue.Drain(maxEvents, maxTotalBytes); } + /// + public Task WaitForEventsAsync(TimeSpan timeout, CancellationToken cancellationToken) + { + return eventQueue.WaitForEventsAsync(timeout, cancellationToken); + } + /// public WorkerFault? DrainFault() { From 94fdc18c3cc7f91e1827e8cb984b208480831c9c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 16:59:52 -0400 Subject: [PATCH 29/41] perf(worker): exact-format timestamp parse and compiled status-field accessors on the event path --- .../Conversion/MxStatusProxyConverterTests.cs | 117 ++++++++++++++ .../MxAccess/MxAccessEventMapperTests.cs | 91 +++++++++++ .../Conversion/MxStatusProxyConverter.cs | 148 +++++++++++++----- .../MxAccess/MxAccessEventMapper.cs | 66 +++++++- 4 files changed, 386 insertions(+), 36 deletions(-) diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Conversion/MxStatusProxyConverterTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Conversion/MxStatusProxyConverterTests.cs index 4224573..8539640 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Conversion/MxStatusProxyConverterTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Conversion/MxStatusProxyConverterTests.cs @@ -135,6 +135,90 @@ public sealed class MxStatusProxyConverterTests Assert.Equal("Invalid reference", second.DiagnosticText); } + /// + /// Verifies every accessor strategy converts the same status values + /// identically. PERF-20 reads a public field of a publicly visible type + /// through a delegate compiled from an expression tree instead of + /// FieldInfo.GetValue; a field type whose conversion to int is not + /// a lossless widening () and a type the + /// compiled delegate may not touch (, not + /// visible outside this assembly) keep the reflection read. All four + /// doubles below carry the same logical status, so all four messages must + /// be equal. + /// + [Fact] + public void Convert_AcrossAccessorStrategies_ProducesIdenticalMessages() + { + // Struct, widening field types: the compiled accessor unboxes in place. + MxStatusProxy compiledStruct = _converter.Convert(new FakeMxStatusProxy + { + success = 1, + category = 5, + detectedBy = 3, + detail = 21, + }); + + // Reference type, widening field types: the compiled accessor casts. + MxStatusProxy compiledClass = _converter.Convert(new FakeStatusProxyClass + { + success = 1, + category = 5, + detectedBy = 3, + detail = 21, + }); + + // Non-widening field types: reflection + Convert.ToInt32 is kept. + MxStatusProxy reflectedWide = _converter.Convert(new FakeWideStatusProxy + { + success = 1L, + category = 5L, + detectedBy = 3L, + detail = 21L, + }); + + // Type not visible outside the assembly: reflection is kept. + MxStatusProxy reflectedHidden = _converter.Convert(new HiddenStatusProxy + { + success = 1, + category = 5, + detectedBy = 3, + detail = 21, + }); + + Assert.Equal(compiledStruct, compiledClass); + Assert.Equal(compiledStruct, reflectedWide); + Assert.Equal(compiledStruct, reflectedHidden); + Assert.Equal(MxStatusCategory.OperationalError, compiledStruct.Category); + Assert.Equal(MxStatusSource.RespondingNmx, compiledStruct.DetectedBy); + Assert.Equal("Invalid reference", compiledStruct.DiagnosticText); + } + + /// + /// Verifies the compiled accessor widens a negative 16-bit field the way + /// Convert.ToInt32 did — sign-extended, not reinterpreted. The + /// interop MXSTATUS_PROXY declares 16-bit fields, so a sign-extension + /// mistake here would silently change every failing status the gateway + /// reports. + /// + [Fact] + public void Convert_WithExtremeInt16FieldValues_SignExtendsLikeConvertToInt32() + { + FakeMxStatusProxy status = new() + { + success = short.MinValue, + category = 2, + detectedBy = 0, + detail = short.MaxValue, + }; + + MxStatusProxy converted = _converter.Convert(status); + + Assert.Equal((int)short.MinValue, converted.Success); + Assert.Equal((int)short.MaxValue, converted.Detail); + Assert.Equal(MxStatusCategory.Warning, converted.Category); + Assert.Equal(MxStatusSource.RequestingLmx, converted.DetectedBy); + } + public struct FakeMxStatusProxy { public short success; @@ -146,6 +230,39 @@ public sealed class MxStatusProxyConverterTests public short detail; } + public struct FakeWideStatusProxy + { + public long success; + + public long category; + + public long detectedBy; + + public long detail; + } + + public sealed class FakeStatusProxyClass + { + public int success; + + public int category; + + public int detectedBy; + + public int detail; + } + + private struct HiddenStatusProxy + { + public short success; + + public int category; + + public int detectedBy; + + public short detail; + } + private sealed class MissingFields { } diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventMapperTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventMapperTests.cs index 9e1c09d..fba52ec 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventMapperTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventMapperTests.cs @@ -238,6 +238,97 @@ public sealed class MxAccessEventMapperTests Assert.False(MxAccessEventMapper.TryParseSourceTimestamp(text, out _)); } + /// + /// Verifies the exact-format fast path PERF-20 added in front of the + /// general parse chain returns exactly what that chain returned. The + /// cases cover the captured MXAccess shape (which the fast path is meant + /// to catch on a US-shaped host), its zero-padded and second-precision + /// siblings, and strings the derived formats cannot match — an ISO + /// round-trip form, a long-date form, and a day-first European form — + /// which must fall through to the unchanged two-stage chain. The + /// expectation is computed with the pre-PERF-20 chain in this test, so it + /// holds on any host culture: the fast path is a shortcut, never a + /// different answer. + /// + /// Timestamp string to parse. + [Theory] + [InlineData("3/26/2026 1:38:22.907 PM")] + [InlineData("3/26/2026 1:38:22 PM")] + [InlineData("03/26/2026 01:38:22 PM")] + [InlineData("12/31/2026 11:59:59.999 PM")] + [InlineData("2026-03-26T13:38:22.9070000")] + [InlineData("Thursday, March 26, 2026 1:38:22 PM")] + [InlineData("26.03.2026 13:38:22")] + [InlineData("not a timestamp")] + public void TryParseSourceTimestamp_MatchesPreFastPathChain(string text) + { + bool expectedParsed = TryParseWithGeneralChainOnly(text, out DateTime expectedUtc); + + bool parsed = MxAccessEventMapper.TryParseSourceTimestamp(text, out DateTime utc); + + Assert.Equal(expectedParsed, parsed); + Assert.Equal(expectedUtc, utc); + if (parsed) + { + Assert.Equal(DateTimeKind.Utc, utc.Kind); + } + } + + /// + /// Verifies a timestamp written in the host culture's own patterns — + /// including the millisecond fraction MXAccess appends, which no culture + /// publishes in its long-time pattern — parses as local wall-clock time + /// and comes back as UTC. This is the string shape the derived exact + /// formats are built for, so it exercises the fast path on the worker's + /// own host regardless of which culture that host runs. + /// + [Fact] + public void TryParseSourceTimestamp_WithCultureShapedMillisecondTimestamp_ReturnsLocalWallClockAsUtc() + { + CultureInfo culture = CultureInfo.CurrentCulture; + string longTime = culture.DateTimeFormat.LongTimePattern; + int secondsIndex = longTime.IndexOf("ss", StringComparison.Ordinal); + + // Every Windows culture publishes seconds in its long-time pattern; the + // guard keeps the test honest rather than green-by-accident if one does not. + Assert.True(secondsIndex >= 0, $"Long-time pattern '{longTime}' has no seconds specifier."); + + DateTime localWall = new(2026, 3, 26, 13, 38, 22, 907, DateTimeKind.Unspecified); + string text = localWall.ToString( + culture.DateTimeFormat.ShortDatePattern + " " + longTime.Insert(secondsIndex + 2, ".fff"), + culture); + + Assert.True(MxAccessEventMapper.TryParseSourceTimestamp(text, out DateTime utc)); + Assert.Equal(DateTimeKind.Utc, utc.Kind); + Assert.Equal(DateTime.SpecifyKind(localWall, DateTimeKind.Local).ToUniversalTime(), utc); + } + + /// + /// The parse chain exactly as it stood before PERF-20 added the + /// exact-format stage, used as the parity oracle above. + /// + /// Timestamp string to parse. + /// The parsed UTC timestamp on success. + /// when the string parsed successfully. + private static bool TryParseWithGeneralChainOnly(string? text, out DateTime utc) + { + utc = default; + if (string.IsNullOrWhiteSpace(text)) + { + return false; + } + + const DateTimeStyles styles = DateTimeStyles.AssumeLocal | DateTimeStyles.AdjustToUniversal; + if (DateTime.TryParse(text, CultureInfo.CurrentCulture, styles, out DateTime parsed) + || DateTime.TryParse(text, CultureInfo.InvariantCulture, styles, out parsed)) + { + utc = DateTime.SpecifyKind(parsed, DateTimeKind.Utc); + return true; + } + + return false; + } + private sealed class FakeStatus { public int success; diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Conversion/MxStatusProxyConverter.cs b/src/ZB.MOM.WW.MxGateway.Worker/Conversion/MxStatusProxyConverter.cs index 036cd9c..9aad71f 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/Conversion/MxStatusProxyConverter.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/Conversion/MxStatusProxyConverter.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; +using System.Linq.Expressions; using System.Reflection; using ZB.MOM.WW.MxGateway.Contracts.Proto; @@ -11,13 +12,15 @@ namespace ZB.MOM.WW.MxGateway.Worker.Conversion; public sealed class MxStatusProxyConverter { /// - /// Per-type cache of the four resolved objects a - /// status conversion needs. The status type is stable (the interop - /// MXSTATUS_PROXY struct in production; a fixed test double in - /// tests), so the expensive - /// metadata scan is resolved once per type and reused. Keyed by - /// so a plain-CLR test double and the real interop - /// struct each get their own entry, keeping the converter interop-agnostic. + /// Per-type cache of the four field accessors a status conversion needs. + /// The status type is stable (the interop MXSTATUS_PROXY struct in + /// production; a fixed test double in tests), so the expensive + /// metadata scan — and, + /// for a visible type, the one-time expression compile that replaces + /// on the event path — happens + /// once per type and is reused. Keyed by so a + /// plain-CLR test double and the real interop struct each get their own + /// entry, keeping the converter interop-agnostic. /// private static readonly ConcurrentDictionary FieldCache = new(); @@ -31,12 +34,11 @@ public sealed class MxStatusProxyConverter throw new ArgumentNullException(nameof(status)); } - Type statusType = status.GetType(); - StatusFields fields = GetFields(statusType); - int success = ReadInt32Field(status, statusType, fields.Success); - int rawCategory = ReadInt32Field(status, statusType, fields.Category); - int rawDetectedBy = ReadInt32Field(status, statusType, fields.DetectedBy); - int detail = ReadInt32Field(status, statusType, fields.Detail); + StatusFields fields = GetFields(status.GetType()); + int success = fields.Success(status); + int rawCategory = fields.Category(status); + int rawDetectedBy = fields.DetectedBy(status); + int detail = fields.Detail(status); return new MxStatusProxy { @@ -109,26 +111,102 @@ public sealed class MxStatusProxyConverter } /// - /// Resolves (and caches) the four objects for the - /// given status type. The first resolution for a type performs the - /// reflection scan; every subsequent conversion of that type reuses the - /// cached entry. A type missing a required field throws the same - /// the per-field lookup used to - /// throw — and, because + /// Resolves (and caches) the four field accessors for the given status + /// type. The first resolution for a type performs the reflection scan and + /// the expression compile; every subsequent conversion of that type + /// reuses the cached delegates. A type missing a required field throws the + /// same the per-field lookup used + /// to throw — and, because /// does not store a value when the factory throws, a bad type keeps /// failing identically on every call rather than being cached. /// /// Runtime type of the status object being converted. - /// The resolved field set for . + /// The resolved field accessors for . private static StatusFields GetFields(Type statusType) { return FieldCache.GetOrAdd( statusType, type => new StatusFields( - ResolveField(type, "success"), - ResolveField(type, "category"), - ResolveField(type, "detectedBy"), - ResolveField(type, "detail"))); + BuildAccessor(type, "success"), + BuildAccessor(type, "category"), + BuildAccessor(type, "detectedBy"), + BuildAccessor(type, "detail"))); + } + + /// + /// Builds the accessor for one status field. Every OnDataChange carries a + /// status array, so this read is on the hot event path: a public field on + /// a publicly visible type is read through a delegate compiled once from + /// an expression tree, which drops the boxed that + /// allocates per field per status + /// entry, plus the dispatch behind + /// . + /// + /// The value is required to be identical, so the compiled path is + /// taken only for field types whose conversion to + /// is a lossless widening (see ) — + /// the interop MXSTATUS_PROXY declares its four fields as + /// 16/32-bit integers. Field types where the CLR conversion and + /// Convert.ToInt32 can disagree (bool, char, floating point, + /// unsigned 32/64-bit, or a reference type that could be null) keep + /// the reflection read verbatim, as does a non-visible type, whose + /// members a compiled delegate is not permitted to touch. + /// + /// + /// Runtime type of the status object. + /// Name of the status field to read. + /// A delegate reading the field as an . + private static Func BuildAccessor( + Type valueType, + string fieldName) + { + FieldInfo field = ResolveField(valueType, fieldName); + if (valueType.IsVisible && IsWideningToInt32(field.FieldType)) + { + try + { + ParameterExpression parameter = Expression.Parameter(typeof(object), "status"); + + // Unbox (not Convert) for a struct: it addresses the boxed + // instance in place rather than copying it out before the load. + Expression instance = valueType.IsValueType + ? Expression.Unbox(parameter, valueType) + : Expression.Convert(parameter, valueType); + Expression widened = Expression.Convert(Expression.Field(instance, field), typeof(int)); + return Expression.Lambda>(widened, parameter).Compile(); + } + catch (Exception exception) when (exception is not OutOfMemoryException) + { + // Compiling the accessor is an optimization, never a requirement: + // if the runtime refuses it the reflection read below still + // produces the same value. + } + } + + return status => ReadInt32Field(status, valueType, field); + } + + /// + /// Reports whether reading a field of this type as an + /// via the CLR's own conversion is lossless and total, and therefore + /// indistinguishable from . + /// Enums answer with their underlying type code, which is the intent. + /// + /// Declared type of the status field. + /// when the compiled accessor is safe to use. + private static bool IsWideningToInt32(Type fieldType) + { + switch (Type.GetTypeCode(fieldType)) + { + case TypeCode.SByte: + case TypeCode.Byte: + case TypeCode.Int16: + case TypeCode.UInt16: + case TypeCode.Int32: + return true; + default: + return false; + } } private static FieldInfo ResolveField( @@ -179,17 +257,17 @@ public sealed class MxStatusProxyConverter } /// - /// The four resolved status fields cached per type. Plain readonly struct - /// (not a record) so it compiles under the worker's net48 target, which - /// lacks IsExternalInit. + /// The four resolved status-field accessors cached per type. Plain + /// readonly struct (not a record) so it compiles under the worker's net48 + /// target, which lacks IsExternalInit. /// private readonly struct StatusFields { public StatusFields( - FieldInfo success, - FieldInfo category, - FieldInfo detectedBy, - FieldInfo detail) + Func success, + Func category, + Func detectedBy, + Func detail) { Success = success; Category = category; @@ -197,12 +275,12 @@ public sealed class MxStatusProxyConverter Detail = detail; } - public FieldInfo Success { get; } + public Func Success { get; } - public FieldInfo Category { get; } + public Func Category { get; } - public FieldInfo DetectedBy { get; } + public Func DetectedBy { get; } - public FieldInfo Detail { get; } + public Func Detail { get; } } } diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventMapper.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventMapper.cs index c16b984..d1958cf 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventMapper.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventMapper.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Concurrent; using System.Globalization; using Google.Protobuf.WellKnownTypes; using ZB.MOM.WW.MxGateway.Contracts.Proto; @@ -9,6 +10,18 @@ namespace ZB.MOM.WW.MxGateway.Worker.MxAccess; /// Maps MXAccess COM events to protobuf MxEvent messages. public sealed class MxAccessEventMapper { + /// + /// Per-culture cache of the explicit timestamp formats + /// tries before the general parse + /// chain. Keyed by because the format set is + /// derived from the culture's own patterns and the worker host's culture + /// is fixed in practice, so the derivation runs once. + /// + private static readonly ConcurrentDictionary TimestampFormatCache = new(); + + /// Cached factory delegate so the dictionary lookup allocates nothing on the event path. + private static readonly Func TimestampFormatFactory = BuildTimestampFormats; + private readonly VariantConverter variantConverter; private readonly MxStatusProxyConverter statusProxyConverter; @@ -353,6 +366,23 @@ public sealed class MxAccessEventMapper /// wall-clock UTC). The parsed value is therefore taken as local time /// and converted to UTC. Tries the worker host's culture first /// (MXAccess formats with the host locale), then the invariant culture. + /// + /// PERF: every OnDataChange carries one of these strings, and the + /// general + /// has to walk the culture's whole pattern set to recognize it. The + /// shapes MXAccess actually emits are tried first with + /// . + /// The formats are derived from the current culture's own + /// short-date and long-time patterns (see + /// ) rather than hard-coded to the + /// US shapes seen in captures: a hard-coded M/d/yyyy would + /// reorder month and day on a day-first host relative to what the + /// general parse below returns. Deriving them means any string the + /// exact parse accepts is one the culture-aware general parse would + /// have accepted with the same value, so the fast path is a shortcut + /// and never a different answer; anything it does not match falls + /// through to the unchanged two-stage chain. + /// /// /// The MXAccess timestamp string. /// The parsed UTC timestamp on success. @@ -366,7 +396,10 @@ public sealed class MxAccessEventMapper } const DateTimeStyles styles = DateTimeStyles.AssumeLocal | DateTimeStyles.AdjustToUniversal; - if (DateTime.TryParse(text, CultureInfo.CurrentCulture, styles, out DateTime parsed) + CultureInfo culture = CultureInfo.CurrentCulture; + string[] formats = TimestampFormatCache.GetOrAdd(culture, TimestampFormatFactory); + if (DateTime.TryParseExact(text, formats, culture, styles, out DateTime parsed) + || DateTime.TryParse(text, culture, styles, out parsed) || DateTime.TryParse(text, CultureInfo.InvariantCulture, styles, out parsed)) { utc = DateTime.SpecifyKind(parsed, DateTimeKind.Utc); @@ -376,6 +409,37 @@ public sealed class MxAccessEventMapper return false; } + /// + /// Builds the explicit timestamp formats for a culture: the culture's + /// short date plus its long time, and — first, because it is the shape + /// MXAccess actually fires — the same pattern with the millisecond + /// fraction MXAccess appends to the seconds ("3/26/2026 1:38:22.907 PM" + /// under en-US). No culture publishes a long-time pattern with + /// milliseconds, so the fractional sibling is derived by inserting + /// .fff after the seconds specifier; a pattern without a + /// two-digit seconds specifier, or one that already carries a fraction, + /// gets only the published pattern. + /// + /// Culture whose patterns the formats are derived from. + /// The formats to try, most likely first. + private static string[] BuildTimestampFormats(CultureInfo culture) + { + DateTimeFormatInfo formatInfo = culture.DateTimeFormat; + string shortDate = formatInfo.ShortDatePattern; + string longTime = formatInfo.LongTimePattern; + int secondsIndex = longTime.IndexOf("ss", StringComparison.Ordinal); + if (secondsIndex >= 0 && longTime.IndexOf('f') < 0 && longTime.IndexOf('F') < 0) + { + return + [ + shortDate + " " + longTime.Insert(secondsIndex + 2, ".fff"), + shortDate + " " + longTime, + ]; + } + + return [shortDate + " " + longTime]; + } + private MxArray ConvertBufferedArray( object? value, MxDataType expectedElementDataType) From f3e1de5f37b2d19afde7b0452cf32bf770f18ebd Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 17:04:06 -0400 Subject: [PATCH 30/41] fix(alarms): truncation-safe transitions; perf: single-pass alarm parse; configurable poll cadence --- docs/DesignDecisions.md | 50 +++ docs/GatewayConfiguration.md | 2 + .../Configuration/AlarmsOptions.cs | 23 ++ .../Configuration/GatewayOptionsValidator.cs | 18 + .../Workers/WorkerProcessLauncher.cs | 21 ++ .../GatewayOptionsValidatorTests.cs | 97 +++++ .../Workers/WorkerProcessLauncherTests.cs | 11 + .../MxAccess/WnWrapAlarmConsumerXmlTests.cs | 340 ++++++++++++++++++ .../MxAccess/MxAccessStaSession.cs | 48 ++- .../MxAccess/WnWrapAlarmConsumer.cs | 333 +++++++++++++++-- 10 files changed, 910 insertions(+), 33 deletions(-) diff --git a/docs/DesignDecisions.md b/docs/DesignDecisions.md index 2068527..6a1d8f3 100644 --- a/docs/DesignDecisions.md +++ b/docs/DesignDecisions.md @@ -135,6 +135,56 @@ alarm state is gateway-wide, not session-scoped — every client wants the same current set plus updates, and forcing each to own a worker would multiply AVEVA polling load for no benefit. +### Alarms — a capped snapshot fetch never implies a clear + +Decision (2026-08-15): when the worker's `GetXmlCurrentAlarms2` fetch comes back +holding exactly `MxGateway:Alarms:MaxAlarmsPerFetch` records, the worker treats +the snapshot as **truncated** and merges it into the retained snapshot instead +of replacing it. Alarms the capped reply did carry update normally; alarms it +had no room to mention are retained untouched. + +The COM API caps its reply at `maxAlmCnt` and exposes no "more available" flag, +so a reply sitting exactly on the cap is indistinguishable from a galaxy that +happens to hold exactly that many active alarms. Both are treated as truncated, +because the two error directions are not symmetric. + +Nothing in the worker emits a Clear transition. The clear is an **inference**: +`WnWrapAlarmConsumer.ComputeTransitions` produces no transition for an alarm +that disappears from the snapshot, and `GatewayAlarmMonitor.ApplyReconcile` +later diffs its cache against `SnapshotActiveAlarms()` and broadcasts a Clear +for every cached alarm the worker no longer reports. Before this decision, a +capped fetch shrank that snapshot, so every alarm past the cap was broadcast as +cleared while still standing — a silent, galaxy-wide false clear on exactly the +alarm floods where the cap is reached. + +Consequences, and how this sits with the existing failover/reconcile design: + +- **The suppression is an eviction guard, not a transition filter.** It lives in + the snapshot update inside `PollOnce`, not in `ComputeTransitions`, which was + never going to emit anything for a disappearance. The reconcile/dedup + machinery (`_clearedByReconcile` tombstones, the NEXT-03 duplicate-Clear + suppression) is untouched: it still sees the same shape of snapshot, only + with the truncated poll's unmentionable alarms still present. +- **It preserves at-least-once, idempotent application.** The failure mode + becomes bounded staleness — a genuinely cleared alarm can linger until the + first sub-cap fetch evicts it, and the reconcile then broadcasts its Clear + late. A late Clear is repaired by the next complete poll; a Clear that never + happened is broadcast to every `StreamAlarms` subscriber and cannot be taken + back. Consumers already apply transitions as "set this alarm to this state", + so a repeated or delayed Clear is absorbed. +- **It does not synthesize anything.** Suppressing an inference is the opposite + of inventing an event; no transition is fabricated on a truncated poll. +- **Failover is unaffected.** `FailoverAlarmConsumer` selects which + `IMxAccessAlarmConsumer` is live; the guard is internal to the wnwrap + consumer's own snapshot bookkeeping and changes neither the failure counting + that triggers failover nor the subtag standby's snapshot, which is built from + a bounded watch-list and has no per-fetch cap to hit. +- **Operators get told.** A truncated poll logs a rate-limited (once per + minute) `AlarmSnapshotTruncated` warning carrying the cap, the record counts, + and the running truncated-fetch total — identifiers and counts only, never + tag names, values, limits, or comments. A galaxy that truncates persistently + is a configuration problem: raise `MxGateway:Alarms:MaxAlarmsPerFetch`. + ## Session-Resilience Epic Scope Decision (2026-07-09, archreview TST-04; migrated here 2026-08-07 from the retired diff --git a/docs/GatewayConfiguration.md b/docs/GatewayConfiguration.md index 9d269e1..4231dfe 100644 --- a/docs/GatewayConfiguration.md +++ b/docs/GatewayConfiguration.md @@ -417,6 +417,8 @@ behavior. | `MxGateway:Alarms:SubscriptionExpression` | _(empty)_ | AVEVA alarm-subscription expression the monitor subscribes on startup, in canonical `\\\Galaxy!` form. The literal `Galaxy` provider is correct regardless of the Galaxy database name. When empty and `Enabled` is `true`, the gateway falls back to `\\\Galaxy!` if `DefaultArea` is set. | | `MxGateway:Alarms:DefaultArea` | _(empty)_ | Area name used to compose a default subscription when `SubscriptionExpression` is empty. If both are empty while `Enabled` is `true`, the monitor faults with a configuration diagnostic. | | `MxGateway:Alarms:ReconcileIntervalSeconds` | `30` | How often the monitor reconciles its in-process alarm cache against the worker's authoritative active-alarm snapshot, catching transitions the live poll-and-diff feed missed. Floored at 5 seconds. | +| `MxGateway:Alarms:PollIntervalMilliseconds` | `500` | Cadence at which the worker's STA polls the AVEVA alarm consumer (`GetXmlCurrentAlarms2`) for the active-alarm snapshot the live feed diffs. Must be `>= 100`: every poll is a COM call plus an XML parse on the same STA that serves reads and writes, so a tighter cadence starves the command path. The gateway conveys the value to the worker via the `MXGATEWAY_ALARM_POLL_INTERVAL_MS` environment variable; a missing or unusable value leaves the worker on the 500 ms default rather than failing the session. | +| `MxGateway:Alarms:MaxAlarmsPerFetch` | `1024` | Cap the worker passes to `GetXmlCurrentAlarms2`'s `maxAlmCnt`. Must be `>= 64`. It doubles as the **truncation threshold**: a fetch returning exactly this many records is treated as truncated, because the COM API caps its reply with no "more available" flag. On a truncated poll the worker retains the alarms the capped reply could not mention instead of letting their absence read as a clear, and logs a rate-limited `AlarmSnapshotTruncated` warning (identifiers and counts only). Raise this on galaxies whose steady-state active-alarm count approaches the cap — a galaxy permanently above it holds stale entries in the snapshot until a sub-cap poll. Conveyed to the worker via the `MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH` environment variable; a missing or unusable value leaves the worker on the 1024 default. | The alarm monitor is independent of client sessions: `AcknowledgeAlarm` and `StreamAlarms` are session-less RPCs served by the monitor. diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/AlarmsOptions.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/AlarmsOptions.cs index 565eac8..98259f5 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/AlarmsOptions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/AlarmsOptions.cs @@ -46,6 +46,29 @@ public sealed class AlarmsOptions /// public int ReconcileIntervalSeconds { get; init; } = 30; + /// + /// Cadence at which the worker's STA polls the AVEVA alarm consumer + /// (GetXmlCurrentAlarms2) for the current active-alarm snapshot. + /// Default 500 ms; must be at least 100 ms. Every poll is a COM call + /// plus an XML parse on the STA that also serves reads and writes, so + /// driving it below 100 ms starves the command path. Conveyed to the + /// worker through the MXGATEWAY_ALARM_POLL_INTERVAL_MS + /// environment variable. + /// + public int PollIntervalMilliseconds { get; init; } = 500; + + /// + /// Cap the worker passes to GetXmlCurrentAlarms2's + /// maxAlmCnt argument. Default 1024; must be at least 64. A + /// fetch that comes back holding exactly this many records is treated + /// as truncated: the worker keeps the alarms the capped fetch could + /// not mention in its snapshot rather than letting their absence read + /// as a clear. Raise it on galaxies whose steady-state active-alarm + /// count approaches the cap. Conveyed to the worker through the + /// MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH environment variable. + /// + public int MaxAlarmsPerFetch { get; init; } = 1024; + /// /// Configuration for the alarm-manager ↔ subtag fallback mechanism: /// operating mode, failure-detection thresholds, discovery, and subtag diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs index 3f98691..29a3aa6 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs @@ -414,8 +414,26 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase + /// Conveys MxGateway:Alarms:PollIntervalMilliseconds to the worker: the + /// cadence at which the worker's STA polls the AVEVA alarm consumer. + /// + public const string WorkerAlarmPollIntervalEnvironmentVariableName = + "MXGATEWAY_ALARM_POLL_INTERVAL_MS"; + + /// + /// Conveys MxGateway:Alarms:MaxAlarmsPerFetch to the worker: the cap + /// passed to GetXmlCurrentAlarms2, which is also the record count at + /// which the worker treats a fetch as truncated. + /// + public const string WorkerMaxAlarmsPerFetchEnvironmentVariableName = + "MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH"; + private readonly IWorkerProcessFactory _processFactory; private readonly IWorkerStartupProbe _startupProbe; private readonly GatewayMetrics _metrics; private readonly TimeProvider _timeProvider; private readonly WorkerOptions _workerOptions; + private readonly AlarmsOptions _alarmsOptions; private readonly ILogger _logger; /// @@ -59,6 +75,7 @@ public sealed class WorkerProcessLauncher : IWorkerProcessLauncher ArgumentNullException.ThrowIfNull(metrics); _workerOptions = gatewayOptions.Value.Worker; + _alarmsOptions = gatewayOptions.Value.Alarms; _processFactory = processFactory; _startupProbe = startupProbe; _metrics = metrics; @@ -185,6 +202,10 @@ public sealed class WorkerProcessLauncher : IWorkerProcessLauncher _workerOptions.PipeConnectAttemptTimeoutMilliseconds.ToString(System.Globalization.CultureInfo.InvariantCulture); startInfo.Environment[WorkerWriteCompletionWaitEnvironmentVariableName] = _workerOptions.WriteCompletionWaitMilliseconds.ToString(System.Globalization.CultureInfo.InvariantCulture); + startInfo.Environment[WorkerAlarmPollIntervalEnvironmentVariableName] = + _alarmsOptions.PollIntervalMilliseconds.ToString(System.Globalization.CultureInfo.InvariantCulture); + startInfo.Environment[WorkerMaxAlarmsPerFetchEnvironmentVariableName] = + _alarmsOptions.MaxAlarmsPerFetch.ToString(System.Globalization.CultureInfo.InvariantCulture); commandLine = new WorkerProcessCommandLine(executablePath, arguments); diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs index f5f2120..40a62b9 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs @@ -163,6 +163,103 @@ public sealed class GatewayOptionsValidatorTests Tls = source.Tls, }; + /// Verifies the alarm poll cadence and per-fetch cap defaults pass validation. + [Fact] + public void Validate_Succeeds_WithDefaultAlarmPollCadenceAndFetchCap() + { + AlarmsOptions alarms = new(); + Assert.Equal(500, alarms.PollIntervalMilliseconds); + Assert.Equal(1024, alarms.MaxAlarmsPerFetch); + + ValidateOptionsResult result = new GatewayOptionsValidator() + .Validate(null, CloneWithAlarms(ValidOptions(), alarms)); + Assert.True(result.Succeeded); + } + + /// + /// A poll cadence below the 100 ms floor must fail validation. Both + /// values are stamped onto every worker launch environment, so they + /// are validated whether or not the central alarm monitor is enabled. + /// + /// Cadence under test. + /// Whether the central alarm monitor is on. + [Theory] + [InlineData(99, false)] + [InlineData(0, false)] + [InlineData(-1, false)] + [InlineData(99, true)] + public void Validate_Fails_WhenAlarmPollIntervalBelowFloor( + int pollIntervalMilliseconds, + bool alarmsEnabled) + { + GatewayOptions options = CloneWithAlarms( + ValidOptions(), + new AlarmsOptions + { + Enabled = alarmsEnabled, + DefaultArea = "Galaxy", + PollIntervalMilliseconds = pollIntervalMilliseconds, + }); + + ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, options); + + Assert.False(result.Succeeded); + Assert.Contains( + result.Failures!, + f => f.Contains("MxGateway:Alarms:PollIntervalMilliseconds", StringComparison.Ordinal)); + } + + /// + /// A per-fetch cap below the 64-record floor must fail validation. The + /// cap doubles as the truncation-detection threshold in the worker, so + /// a tiny cap would make almost every fetch read as truncated. + /// + /// Cap under test. + /// Whether the central alarm monitor is on. + [Theory] + [InlineData(63, false)] + [InlineData(0, false)] + [InlineData(-1, false)] + [InlineData(63, true)] + public void Validate_Fails_WhenMaxAlarmsPerFetchBelowFloor( + int maxAlarmsPerFetch, + bool alarmsEnabled) + { + GatewayOptions options = CloneWithAlarms( + ValidOptions(), + new AlarmsOptions + { + Enabled = alarmsEnabled, + DefaultArea = "Galaxy", + MaxAlarmsPerFetch = maxAlarmsPerFetch, + }); + + ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, options); + + Assert.False(result.Succeeded); + Assert.Contains( + result.Failures!, + f => f.Contains("MxGateway:Alarms:MaxAlarmsPerFetch", StringComparison.Ordinal)); + } + + /// Verifies the floor values themselves are accepted. + [Fact] + public void Validate_Succeeds_AtAlarmPollCadenceAndFetchCapFloors() + { + GatewayOptions options = CloneWithAlarms( + ValidOptions(), + new AlarmsOptions + { + Enabled = true, + DefaultArea = "Galaxy", + PollIntervalMilliseconds = 100, + MaxAlarmsPerFetch = 64, + }); + + ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, options); + Assert.True(result.Succeeded); + } + /// Verifies an invalid fallback mode is not validated when alarms are disabled. [Fact] public void Validate_Succeeds_WhenAlarmsDisabled_FallbackNotValidated() diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerProcessLauncherTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerProcessLauncherTests.cs index 52b3756..6acef16 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerProcessLauncherTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerProcessLauncherTests.cs @@ -47,6 +47,17 @@ public sealed class WorkerProcessLauncherTests "1500", processFactory.LastStartInfo.Environment[ WorkerProcessLauncher.WorkerWriteCompletionWaitEnvironmentVariableName]); + // MxGateway:Alarms defaults reach the worker's alarm poll loop and its + // GetXmlCurrentAlarms2 cap (which is also its truncation threshold) + // through the launch environment, not the command line. + Assert.Equal( + "500", + processFactory.LastStartInfo.Environment[ + WorkerProcessLauncher.WorkerAlarmPollIntervalEnvironmentVariableName]); + Assert.Equal( + "1024", + processFactory.LastStartInfo.Environment[ + WorkerProcessLauncher.WorkerMaxAlarmsPerFetchEnvironmentVariableName]); Assert.DoesNotContain(Nonce, handle.CommandLine.ToString(), StringComparison.Ordinal); Assert.DoesNotContain(Nonce, string.Join(" ", handle.CommandLine.Arguments), StringComparison.Ordinal); Assert.False(pipeReservation.DisposeCalled); diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/WnWrapAlarmConsumerXmlTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/WnWrapAlarmConsumerXmlTests.cs index 7a455c2..682365d 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/WnWrapAlarmConsumerXmlTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/WnWrapAlarmConsumerXmlTests.cs @@ -317,6 +317,346 @@ public sealed class WnWrapAlarmConsumerXmlTests Assert.Equal(MxAlarmStateKind.AckAlm, changedTransition.Record.State); } + // ------------------------------------------------------------------------- + // Truncation cliff (Task 23). GetXmlCurrentAlarms2 caps its reply at + // maxAlmCnt with no "more available" flag. Before the guard, a capped fetch + // shrank the retained snapshot, and every alarm past the cap vanished from + // SnapshotActiveAlarms — which the gateway's reconcile pass reads as a + // Clear and broadcasts to every StreamAlarms subscriber. + // ------------------------------------------------------------------------- + + /// + /// A fetch that comes back holding exactly the cap must be treated as + /// truncated; anything below the cap must not. + /// + /// Records the reply carried. + /// The cap passed to the fetch. + /// Whether the reply must read as truncated. + [Theory] + [InlineData(1024, 1024, true)] + [InlineData(1023, 1024, false)] + [InlineData(0, 1024, false)] + [InlineData(64, 64, true)] + public void IsTruncatedFetch_AtOrAboveCap_IsTruncated( + int fetchedRecordCount, + int maxAlarmsPerFetch, + bool expected) + { + Assert.Equal( + expected, + WnWrapAlarmConsumer.IsTruncatedFetch(fetchedRecordCount, maxAlarmsPerFetch)); + } + + /// + /// THE truncation-correctness test. A 1024-record fetch against a + /// 1024 cap that does not mention a known-active alarm must NOT evict + /// that alarm from the retained snapshot: the snapshot is what + /// returns and + /// what the gateway's reconcile diffs its cache against, so an + /// eviction here is a Clear broadcast for an alarm that never cleared. + /// + [Fact] + public void ApplySnapshotUpdate_WhenFetchTruncated_RetainsAlarmMissingFromCappedFetch() + { + const int Cap = 1024; + Guid missingGuid = new Guid("11111111-1111-1111-1111-111111111111"); + + string xml = BuildAlarmXml(Cap); + Dictionary next = + WnWrapAlarmConsumer.ParseSnapshotXml(xml, out int fetchedRecordCount); + + Assert.Equal(Cap, fetchedRecordCount); + Assert.Equal(Cap, next.Count); + Assert.False(next.ContainsKey(missingGuid)); + Assert.True(WnWrapAlarmConsumer.IsTruncatedFetch(fetchedRecordCount, Cap)); + + Dictionary snapshot = new() + { + [missingGuid] = NewRecord(missingGuid, MxAlarmStateKind.UnackAlm), + }; + + // The diff itself never emits a Clear for a disappearance — the Clear + // is the eviction, one level up. Pin both halves. + IReadOnlyList transitions = + WnWrapAlarmConsumer.ComputeTransitions(snapshot, next); + Assert.DoesNotContain(transitions, t => t.Record.AlarmGuid == missingGuid); + + WnWrapAlarmConsumer.ApplySnapshotUpdate(snapshot, next, truncated: true); + + // NO Clear: the alarm the capped fetch had no room to mention survives, + // so the gateway's reconcile still sees it active. + Assert.True(snapshot.ContainsKey(missingGuid)); + Assert.Equal(MxAlarmStateKind.UnackAlm, snapshot[missingGuid].State); + Assert.Equal(Cap + 1, snapshot.Count); + } + + /// + /// The control case: the same missing alarm against a sub-cap fetch + /// must still be evicted, because a complete fetch IS authoritative + /// about absence. Without this, the truncation guard would have + /// silently disabled clears altogether. + /// + [Fact] + public void ApplySnapshotUpdate_WhenFetchBelowCap_EvictsAlarmMissingFromFetch() + { + const int Cap = 1024; + Guid missingGuid = new Guid("11111111-1111-1111-1111-111111111111"); + + string xml = BuildAlarmXml(Cap - 1); + Dictionary next = + WnWrapAlarmConsumer.ParseSnapshotXml(xml, out int fetchedRecordCount); + + Assert.Equal(Cap - 1, fetchedRecordCount); + Assert.False(WnWrapAlarmConsumer.IsTruncatedFetch(fetchedRecordCount, Cap)); + + Dictionary snapshot = new() + { + [missingGuid] = NewRecord(missingGuid, MxAlarmStateKind.UnackAlm), + }; + + WnWrapAlarmConsumer.ApplySnapshotUpdate(snapshot, next, truncated: false); + + // Clear as today: the alarm drops out of the snapshot, and the + // gateway's reconcile turns that absence into a Clear. + Assert.False(snapshot.ContainsKey(missingGuid)); + Assert.Equal(Cap - 1, snapshot.Count); + } + + /// + /// A truncated fetch must still apply the alarms it DID carry — the + /// guard suppresses eviction, not the update. An alarm whose state + /// changed inside a capped reply still transitions. + /// + [Fact] + public void ApplySnapshotUpdate_WhenFetchTruncated_StillAppliesPresentAlarms() + { + Guid presentGuid = new Guid("22222222-2222-2222-2222-222222222222"); + Guid missingGuid = new Guid("11111111-1111-1111-1111-111111111111"); + + Dictionary snapshot = new() + { + [presentGuid] = NewRecord(presentGuid, MxAlarmStateKind.UnackAlm), + [missingGuid] = NewRecord(missingGuid, MxAlarmStateKind.UnackAlm), + }; + Dictionary next = new() + { + [presentGuid] = NewRecord(presentGuid, MxAlarmStateKind.AckAlm), + }; + + IReadOnlyList transitions = + WnWrapAlarmConsumer.ComputeTransitions(snapshot, next); + + MxAlarmTransitionEvent single = Assert.Single(transitions); + Assert.Equal(presentGuid, single.Record.AlarmGuid); + Assert.Equal(MxAlarmStateKind.UnackAlm, single.PreviousState); + Assert.Equal(MxAlarmStateKind.AckAlm, single.Record.State); + + WnWrapAlarmConsumer.ApplySnapshotUpdate(snapshot, next, truncated: true); + + Assert.Equal(MxAlarmStateKind.AckAlm, snapshot[presentGuid].State); + Assert.True(snapshot.ContainsKey(missingGuid)); + } + + /// + /// Truncation is detected from the raw ALARM-element count, not the + /// parsed dictionary size. A record dropped for a malformed GUID still + /// consumed a slot in the capped reply, so counting only survivors + /// would let a truncated fetch pass as complete and re-open the cliff. + /// + [Fact] + public void ParseSnapshotXml_CountsRawAlarmElements_IncludingRecordsDroppedForBadGuid() + { + const int Cap = 64; + string xml = BuildAlarmXml(Cap).Replace( + "00000000000000000000000000000001", + "not-a-guid"); + + Dictionary records = + WnWrapAlarmConsumer.ParseSnapshotXml(xml, out int fetchedRecordCount); + + Assert.Equal(Cap, fetchedRecordCount); + Assert.Equal(Cap - 1, records.Count); + Assert.True(WnWrapAlarmConsumer.IsTruncatedFetch(fetchedRecordCount, Cap)); + } + + /// + /// The per-fetch cap comes from the launcher-set environment variable; + /// a missing, unparseable, or below-floor value must fall back to the + /// 1024 default rather than throw. A bad environment value must never + /// stop the alarm consumer from starting. + /// + /// Raw environment value under test. + /// Expected resolved cap. + [Theory] + [InlineData(null, 1024)] + [InlineData("", 1024)] + [InlineData("not-a-number", 1024)] + [InlineData("0", 1024)] + [InlineData("-5", 1024)] + [InlineData("63", 1024)] + [InlineData("64", 64)] + [InlineData("4096", 4096)] + public void ResolveMaxAlarmsPerFetch_WithEnvironmentValue_FallsBackToDefaultWhenUnusable( + string? environmentValue, + int expected) + { + string? original = Environment.GetEnvironmentVariable( + WnWrapAlarmConsumer.MaxAlarmsPerFetchEnvironmentVariableName); + try + { + Environment.SetEnvironmentVariable( + WnWrapAlarmConsumer.MaxAlarmsPerFetchEnvironmentVariableName, + environmentValue); + + Assert.Equal(expected, WnWrapAlarmConsumer.ResolveMaxAlarmsPerFetch()); + } + finally + { + Environment.SetEnvironmentVariable( + WnWrapAlarmConsumer.MaxAlarmsPerFetchEnvironmentVariableName, + original); + } + } + + /// + /// Builds a well-formed ALARM_RECORDS payload with + /// distinct alarms. GUIDs are the dashless + /// 32-char hex form wnwrap actually emits, numbered from 1 so a test + /// can name one (e.g. ...0001) to corrupt. + /// + /// Number of ALARM elements to emit. + /// The XML payload. + private static string BuildAlarmXml(int count) + { + System.Text.StringBuilder sb = new System.Text.StringBuilder(); + sb.Append(""); + for (int index = 1; index <= count; index++) + { + sb.Append("") + .Append(index.ToString("X32", System.Globalization.CultureInfo.InvariantCulture)) + .Append("") + .Append("2026/5/1") + .Append("2400") + .Append("TEST-NODE") + .Append("Galaxy") + .Append("TestArea") + .Append("TestMachine_") + .Append(index.ToString(System.Globalization.CultureInfo.InvariantCulture)) + .Append(".TestAlarm") + .Append("DSCtruetrue") + .Append("500UNACK_ALM") + .Append("") + .Append("Test alarm"); + } + sb.Append(""); + return sb.ToString(); + } + + // ------------------------------------------------------------------------- + // Single-pass parse parity (Task 23). The per-field SelectSingleNode calls + // were replaced by one walk over ChildNodes; these pin the semantics that + // walk has to reproduce. + // ------------------------------------------------------------------------- + + /// + /// An absent child element and a present-but-empty one must both yield + /// , exactly as + /// SelectSingleNode(name)?.InnerText ?? string.Empty did. + /// + [Fact] + public void ParseSnapshotXml_WithAbsentAndEmptyChildren_YieldsEmptyStrings() + { + string xml = + "" + + "BCC4705395424D65BDAABCDEA6A32A73" + + "" + + "UNACK_ALM" + + ""; + + Dictionary records = WnWrapAlarmConsumer.ParseSnapshotXml(xml); + MxAlarmSnapshotRecord record = records[new Guid("BCC47053-9542-4D65-BDAA-BCDEA6A32A73")]; + + Assert.Equal(string.Empty, record.TagName); // present but empty + Assert.Equal(string.Empty, record.ProviderNode); // absent + Assert.Equal(string.Empty, record.Value); // absent + Assert.Equal(string.Empty, record.AlarmComment); // absent + Assert.Equal(0, record.Priority); // absent → ParseInt("") → 0 + Assert.Equal(MxAlarmStateKind.UnackAlm, record.State); + } + + /// + /// Duplicate child elements must resolve to the FIRST occurrence — + /// SelectSingleNode returned the first match, so a last-wins + /// walk would silently change which value a malformed payload yields. + /// + [Fact] + public void ParseSnapshotXml_WithDuplicateChildElements_TakesFirstOccurrence() + { + string xml = + "" + + "BCC4705395424D65BDAABCDEA6A32A73" + + "FirstSecond" + + "100900" + + "UNACK_ALM" + + ""; + + Dictionary records = WnWrapAlarmConsumer.ParseSnapshotXml(xml); + MxAlarmSnapshotRecord record = records[new Guid("BCC47053-9542-4D65-BDAA-BCDEA6A32A73")]; + + Assert.Equal("First", record.TagName); + Assert.Equal(100, record.Priority); + } + + /// + /// Field names are matched ordinally and case-sensitively, as the + /// XPath node test was. A lowercase element must not populate the + /// field its uppercase counterpart owns. + /// + [Fact] + public void ParseSnapshotXml_WithWrongCaseChildElement_DoesNotPopulateField() + { + string xml = + "" + + "BCC4705395424D65BDAABCDEA6A32A73" + + "ShouldBeIgnored" + + "UNACK_ALM" + + ""; + + Dictionary records = WnWrapAlarmConsumer.ParseSnapshotXml(xml); + MxAlarmSnapshotRecord record = records[new Guid("BCC47053-9542-4D65-BDAA-BCDEA6A32A73")]; + + Assert.Equal(string.Empty, record.TagName); + } + + /// + /// Non-element children (whitespace text, comments, CDATA) must be + /// skipped rather than matched by name, and a nested element must + /// contribute its InnerText exactly as InnerText always did. + /// + [Fact] + public void ParseSnapshotXml_WithCommentsAndWhitespace_IgnoresNonElementChildren() + { + string xml = + "\n" + + " \n" + + " \n" + + " BCC4705395424D65BDAABCDEA6A32A73\n" + + " TestMachine.TestAlarm\n" + + " UNACK_ALM\n" + + " \n" + + ""; + + Dictionary records = + WnWrapAlarmConsumer.ParseSnapshotXml(xml, out int fetchedRecordCount); + + Assert.Equal(1, fetchedRecordCount); + MxAlarmSnapshotRecord record = Assert.Single(records).Value; + Assert.Equal("TestMachine.TestAlarm", record.TagName); + Assert.Equal(MxAlarmStateKind.UnackAlm, record.State); + } + private static MxAlarmSnapshotRecord NewRecord(Guid guid, MxAlarmStateKind state) { return new MxAlarmSnapshotRecord diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs index 03ddf16..20741b6 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs @@ -19,7 +19,26 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession internal const string WriteCompletionWaitEnvironmentVariableName = "MXGATEWAY_WORKER_WRITE_COMPLETION_WAIT_MS"; - private static readonly TimeSpan AlarmPollInterval = TimeSpan.FromMilliseconds(500); + /// + /// Environment variable the gateway's WorkerProcessLauncher sets from + /// MxGateway:Alarms:PollIntervalMilliseconds. A missing or invalid + /// value falls back to . + /// + internal const string AlarmPollIntervalEnvironmentVariableName = + "MXGATEWAY_ALARM_POLL_INTERVAL_MS"; + + /// + /// Floor on the resolved alarm poll cadence. Mirrors the gateway-side + /// MxGateway:Alarms:PollIntervalMilliseconds minimum so a value that + /// slipped past validation (or an environment set by hand) still can + /// not starve the STA's command path. + /// + internal static readonly TimeSpan MinimumAlarmPollInterval = TimeSpan.FromMilliseconds(100); + + /// Default alarm poll cadence when the environment says nothing usable. + internal static readonly TimeSpan DefaultAlarmPollInterval = TimeSpan.FromMilliseconds(500); + + private readonly TimeSpan alarmPollInterval = ResolveAlarmPollInterval(); private readonly IMxAccessComObjectFactory factory; private readonly IMxAccessEventSink eventSink; @@ -191,6 +210,31 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession : MxAccessCommandExecutor.DefaultWriteCompletionTimeout; } + /// + /// Resolves the alarm poll cadence from the launcher-provided + /// environment variable. A missing, unparseable, or out-of-range value + /// falls back to rather than + /// faulting the worker: an alarm poll that runs at the default cadence + /// is always safe, and a session that refuses to start over a + /// mistyped environment variable is not. + /// + /// The cadence the alarm poll loop waits between polls. + internal static TimeSpan ResolveAlarmPollInterval() + { + string? value = Environment.GetEnvironmentVariable(AlarmPollIntervalEnvironmentVariableName); + if (!int.TryParse( + value, + System.Globalization.NumberStyles.Integer, + System.Globalization.CultureInfo.InvariantCulture, + out int milliseconds)) + { + return DefaultAlarmPollInterval; + } + + TimeSpan resolved = TimeSpan.FromMilliseconds(milliseconds); + return resolved < MinimumAlarmPollInterval ? DefaultAlarmPollInterval : resolved; + } + /// /// Starts the MXAccess COM session asynchronously. /// @@ -274,7 +318,7 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession { try { - await Task.Delay(AlarmPollInterval, cancellationToken).ConfigureAwait(false); + await Task.Delay(alarmPollInterval, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/WnWrapAlarmConsumer.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/WnWrapAlarmConsumer.cs index 8dd78b9..18085e1 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/WnWrapAlarmConsumer.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/WnWrapAlarmConsumer.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.Runtime.InteropServices; using System.Xml; @@ -49,11 +50,39 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer private const string DefaultVersion = "1.0"; private const int DefaultMaxAlarmsPerFetch = 1024; + /// + /// Floor on the resolved per-fetch cap. Mirrors the gateway-side + /// MxGateway:Alarms:MaxAlarmsPerFetch minimum so a value that + /// slipped past startup validation still leaves a usable snapshot + /// window. + /// + internal const int MinimumMaxAlarmsPerFetch = 64; + + /// + /// Environment variable the gateway's WorkerProcessLauncher + /// sets from MxGateway:Alarms:MaxAlarmsPerFetch. A missing, + /// unparseable, or below-floor value falls back to + /// — a bad environment value + /// must never keep the alarm consumer from starting. + /// + internal const string MaxAlarmsPerFetchEnvironmentVariableName = + "MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH"; + + /// + /// Minimum gap between truncated-fetch warnings. A galaxy parked above + /// the cap truncates on *every* poll, so an unthrottled warning would + /// be two log lines a second forever. + /// + private const long TruncationWarningIntervalMilliseconds = 60_000; + private readonly object syncRoot = new object(); private readonly Dictionary latestSnapshot = new Dictionary(); private readonly int maxAlarmsPerFetch; + private readonly Stopwatch truncationWarningClock = Stopwatch.StartNew(); + private long lastTruncationWarningMilliseconds = -TruncationWarningIntervalMilliseconds; + private long truncatedFetchCount; private wwAlarmConsumerClass? client; private wwAlarmConsumerClass? ackClient; private bool subscribed; @@ -67,7 +96,7 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer /// every COM call stays on the STA that owns the apartment. /// public WnWrapAlarmConsumer() - : this(new wwAlarmConsumerClass(), DefaultMaxAlarmsPerFetch) + : this(new wwAlarmConsumerClass(), ResolveMaxAlarmsPerFetch()) { } @@ -86,6 +115,27 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer : DefaultMaxAlarmsPerFetch; } + /// + /// Resolves the per-fetch cap from the launcher-provided environment + /// variable. A missing, unparseable, or below-floor value falls back + /// to rather than throwing: + /// polling at the default cap is always safe, and a session that + /// refuses to subscribe over a mistyped environment variable is not. + /// + /// The cap passed to GetXmlCurrentAlarms2. + internal static int ResolveMaxAlarmsPerFetch() + { + string? value = Environment.GetEnvironmentVariable(MaxAlarmsPerFetchEnvironmentVariableName); + return int.TryParse( + value, + NumberStyles.Integer, + CultureInfo.InvariantCulture, + out int cap) + && cap >= MinimumMaxAlarmsPerFetch + ? cap + : DefaultMaxAlarmsPerFetch; + } + /// /// Fires once per detected alarm-state transition (raise, acknowledge, /// clear, or new-alarm-already-acked-on-arrival), dispatched from @@ -282,6 +332,15 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer } /// + /// + /// This is the set the gateway's reconcile pass diffs its cache + /// against, so an alarm missing here is what the gateway reads as a + /// Clear. After a truncated fetch the snapshot deliberately still + /// carries alarms the capped reply could not mention — see + /// . Erring toward "still active" keeps the feed + /// at-least-once: a stale entry is repaired by the next sub-cap poll, + /// whereas a dropped one is broadcast as a Clear that never happened. + /// public IReadOnlyList SnapshotActiveAlarms() { if (disposed) throw new ObjectDisposedException(nameof(WnWrapAlarmConsumer)); @@ -300,6 +359,16 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer } } + /// + /// Sink for the rate-limited truncated-fetch warning. Defaults to + /// , the stream + /// WorkerConsoleLogger already writes to, so the line lands in + /// the worker's captured stderr alongside the rest of its log. Tests + /// substitute a collector. The message carries identifiers and counts + /// only — never alarm values, tag names, or comments. + /// + internal Action? TruncationWarningSink { get; set; } + /// public void PollOnce() { @@ -316,17 +385,28 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer string xml = xmlObj?.ToString() ?? string.Empty; if (xml.Length == 0) return; - Dictionary next = ParseSnapshotXml(xml); + Dictionary next = ParseSnapshotXml(xml, out int fetchedRecordCount); + + // TRUNCATION CLIFF. GetXmlCurrentAlarms2 caps its reply at maxAlmCnt + // and gives no "there is more" flag, so a reply holding exactly the + // cap is indistinguishable from a galaxy that happens to have exactly + // that many active alarms. Treat the ambiguous case as truncated: the + // false-positive cost is a snapshot that stays stale for one poll, the + // false-negative cost is every alarm past the cap reading as cleared. + bool truncated = IsTruncatedFetch(fetchedRecordCount, maxAlarmsPerFetch); IReadOnlyList transitions; + int retainedCount; lock (syncRoot) { transitions = ComputeTransitions(latestSnapshot, next); - latestSnapshot.Clear(); - foreach (KeyValuePair kv in next) - { - latestSnapshot[kv.Key] = kv.Value; - } + ApplySnapshotUpdate(latestSnapshot, next, truncated); + retainedCount = latestSnapshot.Count; + } + + if (truncated) + { + WarnTruncatedFetch(fetchedRecordCount, next.Count, retainedCount); } if (transitions.Count == 0) return; @@ -338,6 +418,111 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer } } + /// + /// Decides whether a fetch that came back holding + /// records hit the cap. + /// GetXmlCurrentAlarms2 caps its reply at maxAlmCnt and + /// offers no "more available" flag, so a reply at exactly the cap is + /// indistinguishable from a galaxy that happens to hold exactly that + /// many active alarms; both are treated as truncated. Exposed as + /// internal static so the rule is unit-testable without the + /// wnwrapConsumer COM object. + /// + /// ALARM records the reply carried. + /// The cap that was passed to the fetch. + /// when the reply must be treated as truncated. + internal static bool IsTruncatedFetch(int fetchedRecordCount, int maxAlarmsPerFetch) + { + return fetchedRecordCount >= maxAlarmsPerFetch; + } + + /// + /// Folds a freshly fetched snapshot into the retained one. + /// + /// + /// + /// A complete fetch is authoritative about absence: + /// replacing the snapshot wholesale is exactly what lets a cleared + /// alarm drop out of , which is + /// the evidence the gateway's reconcile pass turns into a Clear. + /// + /// + /// A truncated fetch is authoritative about presence + /// only. Merging rather than replacing means alarms the capped + /// reply did carry update normally, while alarms it had no room to + /// mention are retained untouched — so the gateway infers no Clear + /// from their absence. The cost is bounded staleness: a genuinely + /// cleared alarm can linger in the snapshot until the first + /// sub-cap fetch evicts it. That direction is the safe one for an + /// at-least-once alarm feed — a late Clear is repaired by the next + /// complete poll, whereas a Clear that never happened is broadcast + /// to every subscriber and cannot be taken back. + /// + /// + /// The retained snapshot, updated in place. + /// The snapshot just parsed from the fetch. + /// Whether the fetch hit the per-fetch cap. + internal static void ApplySnapshotUpdate( + Dictionary snapshot, + Dictionary next, + bool truncated) + { + if (snapshot is null) throw new ArgumentNullException(nameof(snapshot)); + if (next is null) throw new ArgumentNullException(nameof(next)); + + if (!truncated) + { + snapshot.Clear(); + } + + foreach (KeyValuePair kv in next) + { + snapshot[kv.Key] = kv.Value; + } + } + + /// + /// Emits the truncated-fetch warning at most once per + /// , counting every + /// truncated poll in between so the operator can tell a one-off burst + /// from a galaxy permanently parked above the cap. + /// + /// ALARM records the capped reply carried. + /// Records that survived GUID parsing. + /// Snapshot size after the merge. + private void WarnTruncatedFetch(int fetchedRecordCount, int parsedRecordCount, int retainedCount) + { + long total; + long elapsed; + lock (syncRoot) + { + total = ++truncatedFetchCount; + elapsed = truncationWarningClock.ElapsedMilliseconds; + if (elapsed - lastTruncationWarningMilliseconds < TruncationWarningIntervalMilliseconds) + { + return; + } + + lastTruncationWarningMilliseconds = elapsed; + } + + // Identifiers and counts only — no tag names, values, limits, or + // comments, per the gateway's "don't log tag values by default" rule. + string message = string.Format( + CultureInfo.InvariantCulture, + "level=Warning event=AlarmSnapshotTruncated maxAlarmsPerFetch={0} fetchedRecords={1} " + + "parsedRecords={2} retainedSnapshotSize={3} truncatedFetchesSinceStart={4} " + + "note=absence-implies-clear suppressed for this poll; raise MxGateway:Alarms:MaxAlarmsPerFetch", + maxAlarmsPerFetch, + fetchedRecordCount, + parsedRecordCount, + retainedCount, + total); + + Action sink = TruncationWarningSink ?? Console.Error.WriteLine; + sink(message); + } + /// /// Pure snapshot-to-transitions diff. Compares the previous polled /// snapshot to the next snapshot and produces one @@ -355,6 +540,21 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer /// A GUID present in both with a different produces a transition carrying the prior state. /// A GUID present in but absent from produces no transition. AVEVA drops cleared alarms from the active set; the snapshot simply stops mentioning them. /// + /// + /// The absence rule is deliberately inert here: this diff never + /// turns a disappearance into a Clear. The clear is inferred one + /// level up, by the gateway's reconcile pass diffing its cache + /// against , which reads the + /// snapshot maintains from these same + /// dictionaries. That is why the truncation guard lives in + /// 's snapshot update rather than in this + /// method: suppressing "absence implies Clear" means declining to + /// *evict* alarms a capped fetch could not mention, not declining + /// to emit a transition this method was never going to emit. + /// Passing a truncated straight through + /// stays correct for the alarms it does carry — first sightings + /// and state changes are computed from presence alone. + /// /// /// The snapshot from the previous poll (or empty on first call). /// The snapshot just parsed from GetXmlCurrentAlarms2. @@ -393,9 +593,29 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer /// The XML snapshot payload. /// The parsed alarm snapshot records, keyed by alarm GUID. public static Dictionary ParseSnapshotXml(string xml) + { + return ParseSnapshotXml(xml, out _); + } + + /// + /// plus the raw + /// ALARM-element count, which compares + /// against the per-fetch cap to detect a truncated reply. The raw + /// count — not the dictionary size — is the right signal: records + /// dropped for a malformed GUID still consumed a slot in the capped + /// reply, so counting only the survivors would under-report and let a + /// truncated fetch pass as complete. + /// + /// The XML snapshot payload. + /// Number of ALARM elements in the payload. + /// The parsed alarm snapshot records, keyed by alarm GUID. + internal static Dictionary ParseSnapshotXml( + string xml, + out int alarmRecordCount) { Dictionary records = new Dictionary(); + alarmRecordCount = 0; if (string.IsNullOrWhiteSpace(xml)) return records; @@ -404,45 +624,96 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer XmlNodeList? alarmNodes = doc.SelectNodes("/ALARM_RECORDS/ALARM"); if (alarmNodes is null) return records; + alarmRecordCount = alarmNodes.Count; + foreach (XmlNode alarmNode in alarmNodes) { - string guidHex = TextOf(alarmNode, "GUID"); + // One pass over the child elements instead of 17 SelectSingleNode + // XPath evaluations per record. At the 1024-record cap that is the + // difference between ~17k XPath expression compilations plus child + // walks and one walk per record, on the STA that also serves reads + // and writes. + // + // Parity with the XPath it replaces: SelectSingleNode(name) picks + // the FIRST child ELEMENT of that name (ordinal, case-sensitive) + // and yields its InnerText; an absent child yields string.Empty, + // and so does a present-but-empty one. Hence: element nodes only, + // first-match-wins (null means "not seen yet"), and a single + // coalesce to string.Empty at the end — which keeps the absent and + // empty cases indistinguishable exactly as before. + string? guidHex = null; + string? xmlDate = null; + string? xmlTime = null; + string? gmtOffset = null; + string? dstAdjust = null; + string? providerNode = null; + string? providerName = null; + string? group = null; + string? tagName = null; + string? type = null; + string? value = null; + string? limit = null; + string? priority = null; + string? state = null; + string? operatorNode = null; + string? operatorName = null; + string? alarmComment = null; + + foreach (XmlNode child in alarmNode.ChildNodes) + { + if (child.NodeType != XmlNodeType.Element) continue; + + switch (child.Name) + { + case "GUID": guidHex ??= child.InnerText; break; + case "DATE": xmlDate ??= child.InnerText; break; + case "TIME": xmlTime ??= child.InnerText; break; + case "GMTOFFSET": gmtOffset ??= child.InnerText; break; + case "DSTADJUST": dstAdjust ??= child.InnerText; break; + case "PROVIDER_NODE": providerNode ??= child.InnerText; break; + case "PROVIDER_NAME": providerName ??= child.InnerText; break; + case "GROUP": group ??= child.InnerText; break; + case "TAGNAME": tagName ??= child.InnerText; break; + case "TYPE": type ??= child.InnerText; break; + case "VALUE": value ??= child.InnerText; break; + case "LIMIT": limit ??= child.InnerText; break; + case "PRIORITY": priority ??= child.InnerText; break; + case "STATE": state ??= child.InnerText; break; + case "OPERATOR_NODE": operatorNode ??= child.InnerText; break; + case "OPERATOR_NAME": operatorName ??= child.InnerText; break; + case "ALARM_COMMENT": alarmComment ??= child.InnerText; break; + } + } + if (!TryParseHexGuid(guidHex, out Guid guid)) continue; - string xmlDate = TextOf(alarmNode, "DATE"); - string xmlTime = TextOf(alarmNode, "TIME"); - int gmtOffset = ParseInt(TextOf(alarmNode, "GMTOFFSET")); - int dstAdjust = ParseInt(TextOf(alarmNode, "DSTADJUST")); DateTime tsUtc = AlarmRecordTransitionMapper.ParseTransitionTimestampUtc( - xmlDate, xmlTime, gmtOffset, dstAdjust); + xmlDate ?? string.Empty, + xmlTime ?? string.Empty, + ParseInt(gmtOffset ?? string.Empty), + ParseInt(dstAdjust ?? string.Empty)); records[guid] = new MxAlarmSnapshotRecord { AlarmGuid = guid, TransitionTimestampUtc = tsUtc, - ProviderNode = TextOf(alarmNode, "PROVIDER_NODE"), - ProviderName = TextOf(alarmNode, "PROVIDER_NAME"), - Group = TextOf(alarmNode, "GROUP"), - TagName = TextOf(alarmNode, "TAGNAME"), - Type = TextOf(alarmNode, "TYPE"), - Value = TextOf(alarmNode, "VALUE"), - Limit = TextOf(alarmNode, "LIMIT"), - Priority = ParseInt(TextOf(alarmNode, "PRIORITY")), - State = AlarmRecordTransitionMapper.ParseStateKind(TextOf(alarmNode, "STATE")), - OperatorNode = TextOf(alarmNode, "OPERATOR_NODE"), - OperatorName = TextOf(alarmNode, "OPERATOR_NAME"), - AlarmComment = TextOf(alarmNode, "ALARM_COMMENT"), + ProviderNode = providerNode ?? string.Empty, + ProviderName = providerName ?? string.Empty, + Group = group ?? string.Empty, + TagName = tagName ?? string.Empty, + Type = type ?? string.Empty, + Value = value ?? string.Empty, + Limit = limit ?? string.Empty, + Priority = ParseInt(priority ?? string.Empty), + State = AlarmRecordTransitionMapper.ParseStateKind(state ?? string.Empty), + OperatorNode = operatorNode ?? string.Empty, + OperatorName = operatorName ?? string.Empty, + AlarmComment = alarmComment ?? string.Empty, }; } return records; } - private static string TextOf(XmlNode parent, string childName) - { - XmlNode? node = parent.SelectSingleNode(childName); - return node?.InnerText ?? string.Empty; - } - private static int ParseInt(string text) { return int.TryParse(text, NumberStyles.Integer, CultureInfo.InvariantCulture, out int n) From 13583322b53cfdabc385dc1493929f55508396c4 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 17:10:54 -0400 Subject: [PATCH 31/41] perf(worker): launcher-configurable event queue capacity --- docs/GatewayConfiguration.md | 1 + docs/MxAccessWorkerInstanceDesign.md | 6 +- .../Configuration/GatewayOptionsValidator.cs | 12 ++++ .../Configuration/WorkerOptions.cs | 12 ++++ .../Workers/WorkerProcessLauncher.cs | 9 +++ .../GatewayOptionsValidatorTests.cs | 55 +++++++++++++++ .../Workers/WorkerProcessLauncherTests.cs | 37 +++++++++- .../MxAccess/MxAccessEventQueueTests.cs | 69 +++++++++++++++++++ .../MxAccess/MxAccessEventQueue.cs | 48 +++++++++++++ .../MxAccess/MxAccessStaSession.cs | 9 ++- 10 files changed, 253 insertions(+), 5 deletions(-) diff --git a/docs/GatewayConfiguration.md b/docs/GatewayConfiguration.md index 4231dfe..7d73c9e 100644 --- a/docs/GatewayConfiguration.md +++ b/docs/GatewayConfiguration.md @@ -115,6 +115,7 @@ launch CWD (SEC-01, SEC-33). | `MxGateway:Worker:StartupProbeRetryDelayMilliseconds` | `250` | Delay between transient startup probe retry attempts. | | `MxGateway:Worker:PipeConnectAttemptTimeoutMilliseconds` | `2000` | Per-attempt timeout used by the worker named-pipe connect retry path. The overall pipe connection still stays under the startup budget. | | `MxGateway:Worker:WriteCompletionWaitMilliseconds` | `1500` | Bounded wait the worker holds a unary write reply (`Write`/`Write2`/`WriteSecured`/`WriteSecured2`; bulk writes excluded) for the matching MXAccess `OnWriteComplete` callback, so the reply's `statuses` carry the real commit outcome. `0` disables the wait (pure fire-and-forget replies). Must be `>= 0`. The gateway conveys the value to the worker via the `MXGATEWAY_WORKER_WRITE_COMPLETION_WAIT_MS` environment variable. Consumers that time their own writes must budget above this wait: OtOpcUa's GalaxyDriver wraps gateway writes in a 2 s Tier A resilience timeout, so a deployment raising this option past ~2000 must raise that driver `ResilienceConfig` write timeout in step or slow-but-successful commits surface as consumer-side failures. | +| `MxGateway:Worker:EventQueueCapacity` | `10000` | Capacity, in events, of the worker's outbound MXAccess event queue. Must be between `1000` and `1000000`. This is burst headroom, not a throttle: the queue has no drop policy, so filling it records a `QueueOverflow` worker fault and faults the session. Raise it for sessions whose subscription set can outrun the drain loop (large advise sets, slow event consumers); the backing queue pre-allocates its slots, so the ceiling keeps a mistyped value from committing the 32-bit worker to an outsized allocation. The gateway conveys the value to the worker via the `MXGATEWAY_EVENT_QUEUE_CAPACITY` environment variable; a missing or unusable value leaves the worker on the 10000 default rather than failing the session. | | `MxGateway:Worker:ShutdownTimeoutSeconds` | `10` | Grace period for worker shutdown before the gateway treats shutdown as failed and may kill the worker process tree. | | `MxGateway:Worker:HeartbeatIntervalSeconds` | `5` | Worker heartbeat send interval and gateway heartbeat check cadence input. | | `MxGateway:Worker:HeartbeatGraceSeconds` | `15` | Maximum age of the last worker heartbeat before the gateway faults the worker. This must be greater than or equal to `HeartbeatIntervalSeconds`. | diff --git a/docs/MxAccessWorkerInstanceDesign.md b/docs/MxAccessWorkerInstanceDesign.md index 4372eba..c043110 100644 --- a/docs/MxAccessWorkerInstanceDesign.md +++ b/docs/MxAccessWorkerInstanceDesign.md @@ -416,7 +416,11 @@ type on buffered events. `OperationComplete` is only emitted from the native `MxAccessEventQueue` is the bounded outbound event queue for one worker session. It assigns the monotonic `WorkerSequence` and `WorkerTimestamp` when an event is accepted, preserving the order in which MXAccess handlers enqueue -events. The default capacity is `10000`. When the queue reaches capacity it +events. The capacity is `10000` by default and comes from +`MxGateway:Worker:EventQueueCapacity`, which the gateway stamps onto the worker +launch environment as `MXGATEWAY_EVENT_QUEUE_CAPACITY`; a missing, unparseable, +or out-of-range value (outside `1000`–`1000000`) leaves the worker on the +default rather than failing the session. When the queue reaches capacity it records a `WorkerFaultCategory.QueueOverflow` fault and rejects further events. The event handler catches conversion and enqueue failures, records the first fault on the queue, and returns to the STA message pump instead of writing to diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs index 29a3aa6..55e31c1 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs @@ -10,6 +10,12 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase MaximumWorkerEventQueueCapacity) + { + builder.Add( + $"MxGateway:Worker:EventQueueCapacity must be between {MinimumWorkerEventQueueCapacity} and {MaximumWorkerEventQueueCapacity}."); + } + if (options.MaxMessageBytes is < MinimumMaxMessageBytes or > MaximumMaxMessageBytes) { builder.Add( diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/WorkerOptions.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/WorkerOptions.cs index 78b1586..699a0e2 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/WorkerOptions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/WorkerOptions.cs @@ -33,6 +33,18 @@ public sealed class WorkerOptions /// public int WriteCompletionWaitMilliseconds { get; init; } = 1500; + /// + /// Capacity of the worker's outbound MXAccess event queue, in events. + /// Default 10,000; must be between 1,000 and 1,000,000. This is + /// headroom, not a throttle: the queue has no drop policy, so a burst + /// that fills it faults the session with a QueueOverflow worker + /// fault. Raise it for sessions whose subscription set can outrun the + /// drain loop (large advise sets, slow event consumers). Conveyed to + /// the worker through the MXGATEWAY_EVENT_QUEUE_CAPACITY + /// environment variable. + /// + public int EventQueueCapacity { get; init; } = 10000; + /// The maximum time in seconds for graceful shutdown. public int ShutdownTimeoutSeconds { get; init; } = 10; diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerProcessLauncher.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerProcessLauncher.cs index e36fec8..96d07c6 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerProcessLauncher.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerProcessLauncher.cs @@ -44,6 +44,13 @@ public sealed class WorkerProcessLauncher : IWorkerProcessLauncher public const string WorkerMaxAlarmsPerFetchEnvironmentVariableName = "MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH"; + /// + /// Conveys MxGateway:Worker:EventQueueCapacity to the worker: the capacity + /// of the outbound MXAccess event queue, whose overflow faults the session. + /// + public const string WorkerEventQueueCapacityEnvironmentVariableName = + "MXGATEWAY_EVENT_QUEUE_CAPACITY"; + private readonly IWorkerProcessFactory _processFactory; private readonly IWorkerStartupProbe _startupProbe; private readonly GatewayMetrics _metrics; @@ -202,6 +209,8 @@ public sealed class WorkerProcessLauncher : IWorkerProcessLauncher _workerOptions.PipeConnectAttemptTimeoutMilliseconds.ToString(System.Globalization.CultureInfo.InvariantCulture); startInfo.Environment[WorkerWriteCompletionWaitEnvironmentVariableName] = _workerOptions.WriteCompletionWaitMilliseconds.ToString(System.Globalization.CultureInfo.InvariantCulture); + startInfo.Environment[WorkerEventQueueCapacityEnvironmentVariableName] = + _workerOptions.EventQueueCapacity.ToString(System.Globalization.CultureInfo.InvariantCulture); startInfo.Environment[WorkerAlarmPollIntervalEnvironmentVariableName] = _alarmsOptions.PollIntervalMilliseconds.ToString(System.Globalization.CultureInfo.InvariantCulture); startInfo.Environment[WorkerMaxAlarmsPerFetchEnvironmentVariableName] = diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs index 40a62b9..a5cb35a 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs @@ -1099,4 +1099,59 @@ public sealed class GatewayOptionsValidatorTests Assert.True(result.Failed); Assert.Contains(result.Failures!, f => f.Contains("MaxMessageBytes") && f.Contains("reserve")); } + + /// + /// Verifies the shipped worker event-queue capacity default passes validation and still matches + /// the worker-side MxAccessEventQueue.DefaultCapacity the environment variable falls back + /// to when the launcher value is unusable. + /// + [Fact] + public void Validate_Succeeds_WithDefaultEventQueueCapacity() + { + Assert.Equal(10000, new WorkerOptions().EventQueueCapacity); + + ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, ValidOptions()); + Assert.True(result.Succeeded); + } + + /// + /// Verifies a worker event-queue capacity outside the supported range fails validation. Too + /// small leaves no burst headroom (an overflow faults the whole session); too large commits the + /// 32-bit worker to an outsized pre-allocation. + /// + /// Capacity under test. + [Theory] + [InlineData(0)] + [InlineData(-1)] + [InlineData(999)] + [InlineData(1_000_001)] + public void Validate_Fails_WhenEventQueueCapacityOutOfRange(int eventQueueCapacity) + { + ValidateOptionsResult result = new GatewayOptionsValidator().Validate( + null, + WithWorkerAndProtocol( + new WorkerOptions { EventQueueCapacity = eventQueueCapacity }, + new ProtocolOptions())); + + Assert.True(result.Failed); + Assert.Contains( + result.Failures!, + f => f.Contains("MxGateway:Worker:EventQueueCapacity", StringComparison.Ordinal)); + } + + /// Verifies the worker event-queue capacity bounds themselves are accepted. + /// Capacity under test. + [Theory] + [InlineData(1000)] + [InlineData(1_000_000)] + public void Validate_Succeeds_AtEventQueueCapacityBounds(int eventQueueCapacity) + { + ValidateOptionsResult result = new GatewayOptionsValidator().Validate( + null, + WithWorkerAndProtocol( + new WorkerOptions { EventQueueCapacity = eventQueueCapacity }, + new ProtocolOptions())); + + Assert.True(result.Succeeded); + } } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerProcessLauncherTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerProcessLauncherTests.cs index 6acef16..fb97c47 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerProcessLauncherTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerProcessLauncherTests.cs @@ -47,6 +47,13 @@ public sealed class WorkerProcessLauncherTests "1500", processFactory.LastStartInfo.Environment[ WorkerProcessLauncher.WorkerWriteCompletionWaitEnvironmentVariableName]); + // The worker sizes its outbound event queue from the launch environment; + // the queue has no drop policy, so this capacity is the session's burst + // headroom rather than a throttle. + Assert.Equal( + "10000", + processFactory.LastStartInfo.Environment[ + WorkerProcessLauncher.WorkerEventQueueCapacityEnvironmentVariableName]); // MxGateway:Alarms defaults reach the worker's alarm poll loop and its // GetXmlCurrentAlarms2 cap (which is also its truncation threshold) // through the launch environment, not the command line. @@ -64,6 +71,32 @@ public sealed class WorkerProcessLauncherTests Assert.Equal(0, metrics.GetSnapshot().WorkersRunning); } + /// + /// Verifies that a configured — not the shipped + /// default — is what reaches the worker, so the option is deployable without a worker rebuild. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task LaunchAsync_WithConfiguredEventQueueCapacity_ExportsItToTheWorkerEnvironment() + { + using TestDirectory directory = TestDirectory.Create(); + string executablePath = directory.CreateWorkerExecutable(machine: 0x014c); + FakeWorkerProcessFactory processFactory = new(new FakeWorkerProcess(processId: 1234)); + WorkerProcessLauncher launcher = CreateLauncher( + executablePath, + processFactory, + new SucceedingStartupProbe(), + eventQueueCapacity: 65536); + + using WorkerProcessHandle handle = await launcher.LaunchAsync(CreateRequest()); + + Assert.NotNull(processFactory.LastStartInfo); + Assert.Equal( + "65536", + processFactory.LastStartInfo.Environment[ + WorkerProcessLauncher.WorkerEventQueueCapacityEnvironmentVariableName]); + } + /// Verifies that a failed startup probe kills and disposes the worker process. /// A task that represents the asynchronous operation. [Fact] @@ -216,7 +249,8 @@ public sealed class WorkerProcessLauncherTests GatewayMetrics? metrics = null, int startupTimeoutSeconds = 30, int startupProbeRetryAttempts = 3, - int startupProbeRetryDelayMilliseconds = 250) + int startupProbeRetryDelayMilliseconds = 250, + int eventQueueCapacity = 10000) { GatewayOptions options = new() { @@ -227,6 +261,7 @@ public sealed class WorkerProcessLauncherTests StartupTimeoutSeconds = startupTimeoutSeconds, StartupProbeRetryAttempts = startupProbeRetryAttempts, StartupProbeRetryDelayMilliseconds = startupProbeRetryDelayMilliseconds, + EventQueueCapacity = eventQueueCapacity, }, }; diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventQueueTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventQueueTests.cs index e783fde..82eba17 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventQueueTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventQueueTests.cs @@ -419,6 +419,75 @@ public sealed class MxAccessEventQueueTests await Assert.ThrowsAnyAsync(async () => await wait); } + /// + /// The queue capacity comes from the launcher-set environment variable; + /// a missing, unparseable, below-floor, or above-ceiling value must fall + /// back to the 10,000 default rather than throw. A bad environment value + /// must never stop the worker's session from starting. + /// + /// Raw environment value under test. + /// Expected resolved capacity. + [Theory] + [InlineData(null, MxAccessEventQueue.DefaultCapacity)] + [InlineData("", MxAccessEventQueue.DefaultCapacity)] + [InlineData("not-a-number", MxAccessEventQueue.DefaultCapacity)] + [InlineData("0", MxAccessEventQueue.DefaultCapacity)] + [InlineData("-5", MxAccessEventQueue.DefaultCapacity)] + [InlineData("999", MxAccessEventQueue.DefaultCapacity)] + [InlineData("1000001", MxAccessEventQueue.DefaultCapacity)] + [InlineData("1000", 1000)] + [InlineData("1000000", 1000000)] + [InlineData("50000", 50000)] + public void ResolveCapacity_WithEnvironmentValue_FallsBackToDefaultWhenUnusable( + string? environmentValue, + int expected) + { + string? original = Environment.GetEnvironmentVariable( + MxAccessEventQueue.CapacityEnvironmentVariableName); + try + { + Environment.SetEnvironmentVariable( + MxAccessEventQueue.CapacityEnvironmentVariableName, + environmentValue); + + Assert.Equal(expected, MxAccessEventQueue.ResolveCapacity()); + } + finally + { + Environment.SetEnvironmentVariable( + MxAccessEventQueue.CapacityEnvironmentVariableName, + original); + } + } + + /// + /// A resolved capacity is what the queue is actually built with, so the + /// configured headroom reaches the overflow check rather than only the + /// resolver. + /// + [Fact] + public void Constructor_WithResolvedCapacity_UsesEnvironmentValue() + { + string? original = Environment.GetEnvironmentVariable( + MxAccessEventQueue.CapacityEnvironmentVariableName); + try + { + Environment.SetEnvironmentVariable( + MxAccessEventQueue.CapacityEnvironmentVariableName, + "2500"); + + MxAccessEventQueue queue = new(MxAccessEventQueue.ResolveCapacity()); + + Assert.Equal(2500, queue.Capacity); + } + finally + { + Environment.SetEnvironmentVariable( + MxAccessEventQueue.CapacityEnvironmentVariableName, + original); + } + } + // Mirrors MxAccessEventQueue's per-event repeated-field allowance. Kept local rather than made // public on the queue: the byte-budget tests state their budgets in units of that charge, so a // change to it should surface here as a failing bound instead of silently moving with the code. diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs index 17f1d67..c22288d 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Threading; using System.Threading.Tasks; using Google.Protobuf.WellKnownTypes; @@ -28,6 +29,31 @@ public sealed class MxAccessEventQueue /// public const int DefaultCapacity = 10000; + /// + /// Environment variable the gateway's WorkerProcessLauncher sets + /// from MxGateway:Worker:EventQueueCapacity. A missing, + /// unparseable, or out-of-range value falls back to + /// — a bad environment value must never + /// keep the session from starting. + /// + internal const string CapacityEnvironmentVariableName = "MXGATEWAY_EVENT_QUEUE_CAPACITY"; + + /// + /// Floor on the resolved capacity. Mirrors the gateway-side + /// MxGateway:Worker:EventQueueCapacity minimum so a value that + /// slipped past startup validation still leaves the headroom the queue + /// needs: it has no drop policy, so an overflow faults the session. + /// + internal const int MinimumCapacity = 1000; + + /// + /// Ceiling on the resolved capacity, mirroring the gateway-side maximum. + /// The backing queue pre-allocates its slots, so an absurd environment + /// value would otherwise cost the 32-bit worker that allocation at + /// startup. + /// + internal const int MaximumCapacity = 1_000_000; + // Extra per-event slack added to WorkerEvent.CalculateSize() when charging the byte budget in // Drain(maxEvents, maxTotalBytes). CalculateSize() already accounts for the event's own tag and // length-delimiter (the WorkerEvent wrapper serializes MxEvent as field 1, and the reply packs @@ -80,6 +106,28 @@ public sealed class MxAccessEventQueue events = new Queue(capacity); } + /// + /// Resolves the queue capacity from the launcher-provided environment + /// variable. A missing, unparseable, or out-of-range value falls back to + /// rather than throwing: running on the + /// default capacity is always safe, and a worker that refuses to start + /// over a mistyped environment variable is not. + /// + /// The capacity to construct the outbound event queue with. + internal static int ResolveCapacity() + { + string? value = Environment.GetEnvironmentVariable(CapacityEnvironmentVariableName); + return int.TryParse( + value, + NumberStyles.Integer, + CultureInfo.InvariantCulture, + out int capacity) + && capacity >= MinimumCapacity + && capacity <= MaximumCapacity + ? capacity + : DefaultCapacity; + } + /// /// The queue's maximum capacity. /// diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs index 20741b6..6c2a480 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs @@ -63,12 +63,15 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession /// /// Initializes a new instance of with default dependencies. + /// The outbound event queue is sized from the launcher-provided + /// MXGATEWAY_EVENT_QUEUE_CAPACITY (see ); + /// callers that pass their own queue keep full control of its capacity. /// public MxAccessStaSession() : this( new StaRuntime(), new MxAccessComObjectFactory(), - new MxAccessEventQueue()) + new MxAccessEventQueue(MxAccessEventQueue.ResolveCapacity())) { } @@ -84,7 +87,7 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession : this( new StaRuntime(), new MxAccessComObjectFactory(), - new MxAccessEventQueue(), + new MxAccessEventQueue(MxAccessEventQueue.ResolveCapacity()), alarmCommandHandlerFactory) { } @@ -99,7 +102,7 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession StaRuntime staRuntime, IMxAccessComObjectFactory factory, IMxAccessEventSink eventSink) - : this(staRuntime, factory, eventSink, new MxAccessEventQueue()) + : this(staRuntime, factory, eventSink, new MxAccessEventQueue(MxAccessEventQueue.ResolveCapacity())) { } From 25cbe5cd3ec2e63bd635be76fcb730083d148d90 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 17:10:58 -0400 Subject: [PATCH 32/41] fix(worker): guard timestamp-format derivation against pathological culture patterns --- .../MxAccess/MxAccessEventMapperTests.cs | 49 +++++++++++ .../MxAccess/MxAccessEventMapper.cs | 82 +++++++++++++++++-- 2 files changed, 122 insertions(+), 9 deletions(-) diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventMapperTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventMapperTests.cs index fba52ec..1194d0f 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventMapperTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventMapperTests.cs @@ -303,6 +303,55 @@ public sealed class MxAccessEventMapperTests Assert.Equal(DateTime.SpecifyKind(localWall, DateTimeKind.Local).ToUniversalTime(), utc); } + /// + /// Verifies format derivation survives a culture whose long-time pattern + /// hides an "ss" inside quoted literal text — the case where blindly + /// inserting ".fff" at the first "ss" produces a format that mangles the + /// value or is outright malformed, and a malformed format makes exact + /// parsing throw rather than report failure. Derivation must reject such + /// a candidate at build time, so parsing here stays exception-free and + /// the culture's own published shape still reads as local wall-clock + /// time. Uses a culture no other test touches, because the format cache + /// compares cultures by name. + /// + [Fact] + public void TryParseSourceTimestamp_WithLiteralSecondsInCulturePattern_ParsesWithoutThrowing() + { + CultureInfo pathological = new("en-ZA") + { + DateTimeFormat = + { + ShortDatePattern = "M/d/yyyy", + LongTimePattern = "'sss' HH:mm:ss", + }, + }; + + CultureInfo original = CultureInfo.CurrentCulture; + try + { + CultureInfo.CurrentCulture = pathological; + DateTime localWall = new(2026, 3, 26, 13, 38, 22, DateTimeKind.Unspecified); + + // The culture's published shape: "3/26/2026 sss 13:38:22". + string published = localWall.ToString( + pathological.DateTimeFormat.ShortDatePattern + " " + pathological.DateTimeFormat.LongTimePattern, + pathological); + + Assert.True(MxAccessEventMapper.TryParseSourceTimestamp(published, out DateTime utc)); + Assert.Equal(DateTimeKind.Utc, utc.Kind); + Assert.Equal(DateTime.SpecifyKind(localWall, DateTimeKind.Local).ToUniversalTime(), utc); + + // Shapes this culture cannot describe must still fail quietly rather + // than throw out of the derived formats. + Assert.False(MxAccessEventMapper.TryParseSourceTimestamp("not a timestamp", out _)); + Assert.False(MxAccessEventMapper.TryParseSourceTimestamp("sss sss sss", out _)); + } + finally + { + CultureInfo.CurrentCulture = original; + } + } + /// /// The parse chain exactly as it stood before PERF-20 added the /// exact-format stage, used as the parity oracle above. diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventMapper.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventMapper.cs index d1958cf..a045bfb 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventMapper.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventMapper.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Globalization; using Google.Protobuf.WellKnownTypes; using ZB.MOM.WW.MxGateway.Contracts.Proto; @@ -15,7 +16,13 @@ public sealed class MxAccessEventMapper /// tries before the general parse /// chain. Keyed by because the format set is /// derived from the culture's own patterns and the worker host's culture - /// is fixed in practice, so the derivation runs once. + /// is fixed in practice, so the derivation runs once. The cache assumes + /// patterns are not customized or + /// mutated at runtime — compares by name, so a + /// culture carrying custom patterns is served the formats derived for its + /// stock namesake. That direction is safe: a stale format simply stops + /// matching, and the unchanged general chain below still parses the + /// string. /// private static readonly ConcurrentDictionary TimestampFormatCache = new(); @@ -397,8 +404,12 @@ public sealed class MxAccessEventMapper const DateTimeStyles styles = DateTimeStyles.AssumeLocal | DateTimeStyles.AdjustToUniversal; CultureInfo culture = CultureInfo.CurrentCulture; + + // Empty when the culture's patterns yielded no format that survived + // validation; that culture just gets no fast path. string[] formats = TimestampFormatCache.GetOrAdd(culture, TimestampFormatFactory); - if (DateTime.TryParseExact(text, formats, culture, styles, out DateTime parsed) + DateTime parsed; + if ((formats.Length > 0 && DateTime.TryParseExact(text, formats, culture, styles, out parsed)) || DateTime.TryParse(text, culture, styles, out parsed) || DateTime.TryParse(text, CultureInfo.InvariantCulture, styles, out parsed)) { @@ -419,25 +430,78 @@ public sealed class MxAccessEventMapper /// .fff after the seconds specifier; a pattern without a /// two-digit seconds specifier, or one that already carries a fraction, /// gets only the published pattern. + /// + /// Both candidates are validated by round-tripping a probe instant + /// before they are handed to the event path, because the derivation + /// reads patterns it does not control: the first ss in a + /// pattern could in principle sit inside quoted literal text, and + /// inserting into it would produce a format that either mangles the + /// value or is outright malformed — and a malformed format makes + /// + /// and its siblings throw rather than + /// report failure. Validation happens here, once per culture, so the + /// event path never carries that risk; a candidate that does not + /// round-trip is dropped, and a culture that loses both simply gets + /// no fast path. + /// /// /// Culture whose patterns the formats are derived from. - /// The formats to try, most likely first. + /// The validated formats to try, most likely first; empty when none survived. private static string[] BuildTimestampFormats(CultureInfo culture) { DateTimeFormatInfo formatInfo = culture.DateTimeFormat; string shortDate = formatInfo.ShortDatePattern; string longTime = formatInfo.LongTimePattern; + List formats = new(2); + int secondsIndex = longTime.IndexOf("ss", StringComparison.Ordinal); if (secondsIndex >= 0 && longTime.IndexOf('f') < 0 && longTime.IndexOf('F') < 0) { - return - [ - shortDate + " " + longTime.Insert(secondsIndex + 2, ".fff"), - shortDate + " " + longTime, - ]; + string withMilliseconds = shortDate + " " + longTime.Insert(secondsIndex + 2, ".fff"); + if (RoundTripsProbe(withMilliseconds, culture, new DateTime(2026, 3, 26, 13, 38, 22, 907))) + { + formats.Add(withMilliseconds); + } } - return [shortDate + " " + longTime]; + string published = shortDate + " " + longTime; + if (RoundTripsProbe(published, culture, new DateTime(2026, 3, 26, 13, 38, 22))) + { + formats.Add(published); + } + + return formats.ToArray(); + } + + /// + /// Reports whether a derived format both formats and re-parses a probe + /// instant back to itself under the given culture. A format that throws, + /// fails to parse its own output, or loses part of the instant is + /// rejected — the fast path may only ever be a shortcut. + /// + /// Candidate format to validate. + /// Culture the format was derived from. + /// Instant to round-trip; carries the precision the format must preserve. + /// when the format is safe to use on the event path. + private static bool RoundTripsProbe( + string format, + CultureInfo culture, + DateTime probe) + { + try + { + string probeText = probe.ToString(format, culture); + return DateTime.TryParseExact(probeText, format, culture, DateTimeStyles.None, out DateTime roundTripped) + && roundTripped == probe; + } + catch (FormatException) + { + return false; + } + catch (ArgumentException) + { + return false; + } } private MxArray ConvertBufferedArray( From 896d81e286a35922e92e5e1f0eaeb2997579d91b Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 17:13:17 -0400 Subject: [PATCH 33/41] docs(worker): record the SemaphoreSlim/STA dispatch invariant + single-waiter contract; harden fake --- .../MxAccess/MxAccessEventQueueTests.cs | 21 +++++++++++++++---- .../TestSupport/FakeRuntimeSession.cs | 11 ++++++++++ .../MxAccess/IWorkerRuntimeSession.cs | 5 +++++ .../MxAccess/MxAccessEventQueue.cs | 14 +++++++++++++ 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventQueueTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventQueueTests.cs index 82eba17..f01c42f 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventQueueTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventQueueTests.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using ZB.MOM.WW.MxGateway.Contracts.Proto; @@ -389,20 +390,32 @@ public sealed class MxAccessEventQueueTests } /// - /// Verifies the timeout still bounds an unsignalled wait: the fallback survives as a ceiling - /// on how long a caller may sleep, so a state change reached by some future path that does - /// not signal is still observed on the next pass rather than never. + /// Verifies the timeout still bounds an unsignalled wait, from both sides: the wait ends + /// without a signal (the fallback survives as a ceiling, so a state change reached by some + /// future path that does not signal is still observed on the next pass rather than never) + /// and it does not end early (the wait really is the timeout, not an already-armed permit + /// completing it instantly). A longer-than-production timeout is used so the lower bound + /// carries a wide margin over timer resolution. /// /// A task that represents the asynchronous operation. [Fact] public async Task WaitForEventsAsync_WithNoSignal_CompletesAtTheFallbackTimeout() { MxAccessEventQueue queue = new(capacity: 4); + TimeSpan fallback = TimeSpan.FromMilliseconds(200); + Stopwatch elapsed = Stopwatch.StartNew(); - Task wait = queue.WaitForEventsAsync(TimeSpan.FromMilliseconds(25), CancellationToken.None); + Task wait = queue.WaitForEventsAsync(fallback, CancellationToken.None); Assert.Same(wait, await Task.WhenAny(wait, Task.Delay(WakePatience))); await wait; + elapsed.Stop(); + + // Half the timeout: far enough below it to be immune to timer resolution, far enough above + // zero to fail an implementation that returned a completed task instead of waiting. + Assert.True( + elapsed.Elapsed >= TimeSpan.FromMilliseconds(100), + $"Unsignalled wait returned after {elapsed.ElapsedMilliseconds} ms, well inside its {fallback.TotalMilliseconds} ms fallback."); } /// Verifies a cancelled wait unwinds instead of hanging until the fallback expires. diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs index 84edc37..b5a4252 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs @@ -303,6 +303,17 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession if (BackingQueue is not null) { + if (WaitForEventsOnSignalOnly) + { + // The two are mutually exclusive: a real backing queue owns its own signal and + // always honours the timeout, so silently letting it win would leave a + // signal-only test passing on a fallback tick — exactly the false green the + // option exists to rule out. + throw new InvalidOperationException( + "FakeRuntimeSession cannot combine BackingQueue with WaitForEventsOnSignalOnly: " + + "the backing queue honours the fallback timeout, which defeats the signal-only wait."); + } + // Tests that drive a real queue enqueue into it directly, so the real queue owns the // wake signal too. return BackingQueue.WaitForEventsAsync(timeout, cancellationToken); diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/IWorkerRuntimeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/IWorkerRuntimeSession.cs index cabfd82..e1050c8 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/IWorkerRuntimeSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/IWorkerRuntimeSession.cs @@ -68,6 +68,11 @@ public interface IWorkerRuntimeSession : IDisposable /// no default interface members, so every implementation supplies it. The wait's outcome is /// not surfaced: the caller re-drains and re-checks after every /// wait, because the signal is coalesced and the timeout is a ceiling, not a poll period. + /// + /// Precondition: at most one waiter. The implementation's wake signal carries a + /// single permit, which is sufficient only because a session has exactly one event + /// drain loop; a second concurrent waiter would degrade to the fallback timeout. + /// /// /// Maximum time to wait before the wait completes unsignalled. /// Token cancelling the wait. diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs index c22288d..5bcf9e1 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs @@ -75,6 +75,13 @@ public sealed class MxAccessEventQueue // waiter, no wait handle is ever materialized (nothing touches AvailableWaitHandle), and making // the queue IDisposable would only add a lifecycle race between the STA's Enqueue and the // drain loop's wait. WorkerFrameWriter's _writeLock is held the same way. + // + // It must STAY a SemaphoreSlim: on .NET Framework 4.8, Release() hands an async waiter to the + // thread pool (the waiter is a TaskNode : IThreadPoolWorkItem) rather than completing it + // inline, so the STA thread's Enqueue never runs the drain loop's continuation. Swapping this + // for a TaskCompletionSource — a refactor that looks cosmetic — would complete the waiter + // inline on the caller and put the drain loop's pipe write on the apartment thread, which is a + // real STA/COM reentrancy hazard (and stalls the message pump behind the write). private readonly SemaphoreSlim eventSignal = new SemaphoreSlim(initialCount: 0, maxCount: 1); private ulong lastEventSequence; @@ -257,6 +264,13 @@ public sealed class MxAccessEventQueue /// enqueues. The timeout is therefore a ceiling on how long the caller may sleep, not a /// poll period: it bounds how late a state change reaches the caller if some future path /// mutates the queue without signalling. + /// + /// Precondition: at most one waiter. One permit is sufficient only because the + /// queue has exactly one consumer (the worker's single event drain loop). A second + /// concurrent waiter would find no permit left for it and degrade to the fallback + /// timeout, so a design that needs multiple drainers must widen the signal rather than + /// reuse this one. + /// /// /// Maximum time to wait before the wait completes unsignalled. /// Token cancelling the wait. From 7c9add3d7348c7d228f923b9c4ab502537eba289 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 17:17:10 -0400 Subject: [PATCH 34/41] fix(worker): StaWaitHelper regression tests + full-drain contract notes; consolidate wait P/Invoke --- docs/MxAccessWorkerInstanceDesign.md | 8 + .../MxAccess/MxAccessValueCacheTests.cs | 28 ++- .../MxAccessWriteCompletionCacheTests.cs | 28 ++- .../Sta/StaWaitHelperTests.cs | 235 ++++++++++++++++++ .../MxAccess/MxAccessCommandExecutor.cs | 14 ++ .../MxAccess/MxAccessValueCache.cs | 25 +- .../MxAccess/MxAccessWriteCompletionCache.cs | 25 +- .../Sta/StaMessagePump.cs | 19 +- .../Sta/StaNativeMethods.cs | 58 +++++ .../Sta/StaWaitHelper.cs | 92 +++++-- 10 files changed, 466 insertions(+), 66 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Worker.Tests/Sta/StaWaitHelperTests.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Worker/Sta/StaNativeMethods.cs diff --git a/docs/MxAccessWorkerInstanceDesign.md b/docs/MxAccessWorkerInstanceDesign.md index c043110..a776161 100644 --- a/docs/MxAccessWorkerInstanceDesign.md +++ b/docs/MxAccessWorkerInstanceDesign.md @@ -295,6 +295,14 @@ cache lock) so a cross-thread producer wakes the waiter immediately; in the live worker the update arrives on the STA from inside `pumpStep` itself, and the message wake is what carries it. +`MWMO_INPUTAVAILABLE` makes the drain contract load-bearing: the wait wakes on +input that is merely *present*, including input an earlier `PeekMessage` saw but +did not remove. A `pumpStep` that drains only part of the queue — or a no-op one +— therefore leaves a message that satisfies the wake condition forever, and the +loop spins at 100% CPU until its deadline (deadline and reply shape still hold; +it is a CPU fault, not a correctness one). Every `pumpStep` must drain to empty, +as `StaRuntime.PumpPendingMessages` does. + The wait slice is capped at 50 ms so `pumpStep` runs periodically even when nothing wakes the wait — a process with no STA message queue (unit tests drive these caches from ordinary threads, standing in for the STA by updating the diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessValueCacheTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessValueCacheTests.cs index 6b2ef48..bad8580 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessValueCacheTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessValueCacheTests.cs @@ -179,27 +179,35 @@ public sealed class MxAccessValueCacheTests } /// - /// Verifies the wait is message/signal-driven rather than sleep-polled: - /// with a poll interval far longer than the deadline, a value set from - /// another thread still wakes the wait and is returned well inside the - /// deadline. The sleep-polled loop this replaced would have sat blind for - /// the whole 10 s interval — on the STA that means 10 s with no Windows + /// Verifies the wait slice stays bounded no matter what poll interval the + /// caller asks for: with a 10 s interval and a 3 s deadline, a value set + /// 50 ms in is still returned in time, because every slice is capped at + /// the 50 ms fallback tick. The blind Thread.Sleep this replaced + /// would have slept the full interval — on the STA, 10 s with no Windows /// messages dispatched, so the OnDataChange being waited for could not /// have arrived at all — and then reported a timeout. /// + /// + /// Scope: this proves the cadence bound, not the signal path. The clamp + /// alone would satisfy it, so it stays green if the Set-side + /// signal is deleted. StaWaitHelperTests owns the signal path, + /// where a five-second wait with no clamp in play makes the handle the + /// only thing that can end it early. + /// + /// A task that represents the asynchronous operation. [Fact] - public void TryWaitForUpdate_WakesOnCrossThreadSet_DespiteLongPollInterval() + public async Task TryWaitForUpdate_CompletesWithinDeadline_WhenPollIntervalExceedsTheFallbackTick() { MxAccessValueCache cache = new(); Timestamp sourceTimestamp = Timestamp.FromDateTime(DateTime.UtcNow); using ManualResetEventSlim waitEntered = new(false); - Task setter = Task.Run(() => + Task setter = Task.Run(async () => { // Handshake so the value cannot land before the wait starts — // otherwise the first check would satisfy it and prove nothing. waitEntered.Wait(TimeSpan.FromSeconds(5)); - Thread.Sleep(50); + await Task.Delay(50, CancellationToken.None); cache.Set(7, 21, BuildEvent(7, 21, intValue: 8080, quality: 192, sourceTimestamp)); }); @@ -213,13 +221,13 @@ public sealed class MxAccessValueCacheTests out MxAccessValueCache.CachedValue value, pollIntervalMs: 10_000); elapsed.Stop(); - setter.Wait(TimeSpan.FromSeconds(5)); + await setter; Assert.True(found); Assert.Equal(8080, value.Value.Int32Value); Assert.True( elapsed.Elapsed < TimeSpan.FromSeconds(2), - $"The wait should have woken on the cached value, not slept out the poll interval; took {elapsed.ElapsedMilliseconds} ms."); + $"The wait should have re-checked within the fallback tick, not slept out the poll interval; took {elapsed.ElapsedMilliseconds} ms."); } /// diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessWriteCompletionCacheTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessWriteCompletionCacheTests.cs index fd817f1..8d4b1c8 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessWriteCompletionCacheTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessWriteCompletionCacheTests.cs @@ -113,25 +113,33 @@ public sealed class MxAccessWriteCompletionCacheTests } /// - /// Verifies the wait is message/signal-driven rather than sleep-polled: - /// with a poll interval far longer than the deadline, a completion - /// recorded from another thread still wakes the wait and is returned well - /// inside the deadline. The sleep-polled loop this replaced would have sat - /// blind for the whole 10 s interval — on the STA that means 10 s with no + /// Verifies the wait slice stays bounded no matter what poll interval the + /// caller asks for: with a 10 s interval and a 3 s deadline, a completion + /// recorded 50 ms in is still returned in time, because every slice is + /// capped at the 50 ms fallback tick. The blind Thread.Sleep this + /// replaced would have slept the full interval — on the STA, 10 s with no /// Windows messages dispatched — and then reported a timeout. /// + /// + /// Scope: this proves the cadence bound, not the signal path. The clamp + /// alone would satisfy it, so it stays green if the Record-side + /// Set() is deleted. StaWaitHelperTests owns the signal + /// path, where a five-second wait with no clamp in play makes the handle + /// the only thing that can end it early. + /// + /// A task that represents the asynchronous operation. [Fact] - public void TryWaitForCompletion_WakesOnCrossThreadRecord_DespiteLongPollInterval() + public async Task TryWaitForCompletion_CompletesWithinDeadline_WhenPollIntervalExceedsTheFallbackTick() { MxAccessWriteCompletionCache cache = new(); using ManualResetEventSlim waitEntered = new(false); - Task recorder = Task.Run(() => + Task recorder = Task.Run(async () => { // Handshake so the completion cannot land before the wait starts — // otherwise the first check would satisfy it and prove nothing. waitEntered.Wait(TimeSpan.FromSeconds(5)); - Thread.Sleep(50); + await Task.Delay(50, CancellationToken.None); cache.Record(7, 21, BuildStatuses(detail: 8080)); }); @@ -145,13 +153,13 @@ public sealed class MxAccessWriteCompletionCacheTests out RepeatedField statuses, pollIntervalMs: 10_000); elapsed.Stop(); - recorder.Wait(TimeSpan.FromSeconds(5)); + await recorder; Assert.True(found); Assert.Equal(8080, Assert.Single(statuses).Detail); Assert.True( elapsed.Elapsed < TimeSpan.FromSeconds(2), - $"The wait should have woken on the recorded completion, not slept out the poll interval; took {elapsed.ElapsedMilliseconds} ms."); + $"The wait should have re-checked within the fallback tick, not slept out the poll interval; took {elapsed.ElapsedMilliseconds} ms."); } /// Verifies that Record stores an independent clone of the caller's rows. diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Sta/StaWaitHelperTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Sta/StaWaitHelperTests.cs new file mode 100644 index 0000000..925f86c --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Sta/StaWaitHelperTests.cs @@ -0,0 +1,235 @@ +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; +using ZB.MOM.WW.MxGateway.Worker.Sta; + +namespace ZB.MOM.WW.MxGateway.Worker.Tests.Sta; + +/// +/// Tests for , the message-aware wait behind the +/// write-completion and ReadBulk value waits. +/// +/// +/// +/// These exist because the cache-level tests cannot guard the signal +/// path. Every cache wait clamps its slice to +/// , so a wait +/// woken by the handle and a wait that merely timed out and re-checked +/// are milliseconds apart — no wall-clock assertion up there can tell +/// them apart, and deleting the caches' Set() calls would leave +/// them green. Calling the helper directly with a five-second timeout +/// removes the clamp from the picture: only the handle can end that wait +/// early, and the test asserts the handle was the thing consumed. +/// +/// +/// Boundary: the WAIT_FAILED branch is not exercised, for the same +/// reason gives — forcing +/// MsgWaitForMultipleObjectsEx to fail means handing it a +/// deliberately invalid native handle, which is unsafe to construct in a +/// managed test. +/// +/// +public sealed class StaWaitHelperTests +{ + /// Verifies that a null wait handle is rejected. + [Fact] + public void WaitForSignalOrMessages_NullSignal_ThrowsArgumentNullException() + { + ArgumentNullException exception = Assert.Throws( + () => StaWaitHelper.WaitForSignalOrMessages(null!, 10)); + + Assert.Equal("signal", exception.ParamName); + } + + /// + /// Verifies the wait ends on a cross-thread signal rather than on its + /// timeout: the handle is set ~50 ms into a five-second wait, and the + /// wait must return two orders of magnitude before that timeout. The + /// post-condition that the handle was consumed is what makes this a + /// discriminator — an auto-reset event that our wait did not take would + /// still be signalled afterwards, which would mean something other than + /// the signal ended the wait. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task WaitForSignalOrMessages_SignalSetDuringWait_ReturnsLongBeforeTheTimeout() + { + using AutoResetEvent signal = new(initialState: false); + + // Drain anything already queued for this thread so the wait cannot be + // woken by stale input instead of by the signal. + new StaMessagePump().PumpPendingMessages(); + + Task setter = Task.Run(async () => + { + await Task.Delay(50, CancellationToken.None); + signal.Set(); + }); + + Stopwatch elapsed = Stopwatch.StartNew(); + StaWaitHelper.WaitForSignalOrMessages(signal, 5000); + elapsed.Stop(); + await setter; + + Assert.True( + elapsed.Elapsed < TimeSpan.FromMilliseconds(500), + $"Wait took {elapsed.ElapsedMilliseconds} ms of its 5000 ms timeout; the signal should have ended it."); + Assert.False( + signal.WaitOne(0), + "The wait did not consume the signal, so something other than the signal ended it."); + } + + /// + /// Verifies an already-signalled handle ends the wait immediately, which + /// is the case that keeps a completion recorded between the caller's + /// check and its wait from being missed. + /// + [Fact] + public void WaitForSignalOrMessages_PreSignalledHandle_ReturnsImmediately() + { + using AutoResetEvent signal = new(initialState: true); + + Stopwatch elapsed = Stopwatch.StartNew(); + StaWaitHelper.WaitForSignalOrMessages(signal, 30_000); + elapsed.Stop(); + + Assert.True( + elapsed.Elapsed < TimeSpan.FromSeconds(5), + $"Wait took {elapsed.ElapsedMilliseconds} ms; a pre-signalled handle must return at once."); + Assert.False(signal.WaitOne(0), "The wait should have consumed the signal."); + } + + /// + /// Verifies the wait actually blocks for its timeout when nothing signals + /// it. The lower bound is the point: a wait that returned instantly would + /// turn every caller's poll loop into a spin, which is exactly the + /// failure mode the queue must be drained to avoid. + /// + [Fact] + public void WaitForSignalOrMessages_NeverSignalled_BlocksUntilTheTimeout() + { + using AutoResetEvent signal = new(initialState: false); + + // The wait wakes on input that is merely present, so drain first — + // otherwise a stale message would end the wait early and the lower + // bound below would be measuring the wrong thing. + new StaMessagePump().PumpPendingMessages(); + + Stopwatch elapsed = Stopwatch.StartNew(); + StaWaitHelper.WaitForSignalOrMessages(signal, 200); + elapsed.Stop(); + + // Loose lower bound: the OS may return slightly early on a coarse timer + // tick, so this proves "it blocked", not "it blocked for exactly 200 ms". + Assert.True( + elapsed.Elapsed >= TimeSpan.FromMilliseconds(100), + $"Wait returned after {elapsed.ElapsedMilliseconds} ms; a 200 ms wait must not return instantly."); + Assert.True( + elapsed.Elapsed < TimeSpan.FromSeconds(5), + $"Wait took {elapsed.ElapsedMilliseconds} ms; a 200 ms timeout must end it."); + } + + /// + /// Verifies a non-positive timeout returns without entering the wait at + /// all — proven by the signal still being set afterwards, since a wait + /// that ran would have consumed it. + /// + /// The non-positive timeout under test. + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void WaitForSignalOrMessages_NonPositiveTimeout_DoesNotEnterTheWait(int timeoutMilliseconds) + { + using AutoResetEvent signal = new(initialState: true); + + Stopwatch elapsed = Stopwatch.StartNew(); + StaWaitHelper.WaitForSignalOrMessages(signal, timeoutMilliseconds); + elapsed.Stop(); + + Assert.True( + elapsed.Elapsed < TimeSpan.FromSeconds(2), + $"Wait took {elapsed.ElapsedMilliseconds} ms; a non-positive timeout must not block."); + Assert.True(signal.WaitOne(0), "The signal must be untouched when the wait is skipped."); + } + + /// + /// Boundary table for the slice math. Every caller's wait length comes + /// from here, so each clause is pinned: a poll interval below one clamps + /// up to 1 ms, an expired deadline yields 0 (the caller returns without + /// waiting), a deadline nearer than the tick rounds up so the wait never + /// ends before the deadline, and any interval above the fallback ceiling + /// clamps down to it. + /// + /// Time left before the caller's deadline. + /// The caller's requested pump cadence. + /// The expected wait slice in milliseconds. + [Theory] + // pollInterval < 1 clamps up to 1 ms — never a zero-length spin. + [InlineData(1000d, 0, 1)] + [InlineData(1000d, -5, 1)] + // remaining <= 0 yields 0; the caller has already returned on its deadline. + [InlineData(0d, 5, 0)] + [InlineData(-10d, 5, 0)] + // remaining < tick shortens the wait to the deadline. + [InlineData(3d, 5, 3)] + [InlineData(1d, 50, 1)] + // remaining == tick is not "less than", so the tick stands. + [InlineData(5d, 5, 5)] + // The caller's interval wins while it is under the ceiling. + [InlineData(1000d, 5, 5)] + [InlineData(1000d, 49, 49)] + // Anything at or above the ceiling clamps to it — this is what bounds the + // unconditional pumpStep cadence regardless of what a caller asks for. + [InlineData(1000d, 50, 50)] + [InlineData(1000d, 10_000, 50)] + [InlineData(50d, 10_000, 50)] + public void ClampWaitMilliseconds_BoundaryTable( + double remainingMilliseconds, + int pollIntervalMilliseconds, + int expected) + { + int actual = StaWaitHelper.ClampWaitMilliseconds( + TimeSpan.FromMilliseconds(remainingMilliseconds), + pollIntervalMilliseconds); + + Assert.Equal(expected, actual); + } + + /// + /// Verifies a sub-millisecond remainder rounds up rather than down: a + /// zero-length wait would spin, and truncating would end the wait before + /// the caller's deadline. Built from ticks because + /// rounds its argument to + /// whole milliseconds and so cannot express these values at all. + /// + [Fact] + public void ClampWaitMilliseconds_SubMillisecondRemaining_RoundsUp() + { + // 10_000 ticks == 1 ms. + Assert.Equal(1, StaWaitHelper.ClampWaitMilliseconds(TimeSpan.FromTicks(4_000), 5)); + Assert.Equal(3, StaWaitHelper.ClampWaitMilliseconds(TimeSpan.FromTicks(25_000), 5)); + Assert.Equal(1, StaWaitHelper.ClampWaitMilliseconds(TimeSpan.FromTicks(1), 5)); + } + + /// + /// Verifies the clamp never exceeds the time actually left, which is what + /// keeps a wait from overrunning its caller's deadline. + /// + [Fact] + public void ClampWaitMilliseconds_NeverExceedsRemainingOrTheCeiling() + { + for (int remaining = 1; remaining <= 200; remaining++) + { + int slice = StaWaitHelper.ClampWaitMilliseconds( + TimeSpan.FromMilliseconds(remaining), + pollIntervalMilliseconds: 10_000); + + Assert.True(slice >= 1, $"Slice for {remaining} ms remaining was {slice}; a positive deadline must wait."); + Assert.True(slice <= remaining, $"Slice {slice} overran the {remaining} ms left before the deadline."); + Assert.True( + slice <= StaWaitHelper.MaxFallbackTickMilliseconds, + $"Slice {slice} exceeded the {StaWaitHelper.MaxFallbackTickMilliseconds} ms fallback ceiling."); + } + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessCommandExecutor.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessCommandExecutor.cs index 9c36df2..15019f4 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessCommandExecutor.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessCommandExecutor.cs @@ -100,6 +100,20 @@ public sealed class MxAccessCommandExecutor : IStaCommandExecutor this.variantConverter = variantConverter ?? throw new ArgumentNullException(nameof(variantConverter)); this.statusProxyConverter = new MxStatusProxyConverter(); this.alarmCommandHandler = alarmCommandHandler; + + // The no-op fallback is a TEST affordance only — never pass null on a + // real STA. The cache wait loops this pump step feeds + // (MxAccessValueCache.TryWaitForUpdate, + // MxAccessWriteCompletionCache.TryWaitForCompletion) block in + // StaWaitHelper.WaitForSignalOrMessages, whose contract is that the pump + // step drains the message queue to empty on every call: the wait wakes on + // input that is merely present, so a pump step that removes nothing makes + // every wait return instantly and turns the loop into a busy spin at 100% + // CPU until its deadline (deadline and reply shape still hold — it is a + // CPU fault, not a correctness one). It is harmless here only because the + // fakes that pass null pre-populate the caches and run on threads with no + // windows and no pending input, so there is never a message left behind + // for the wait to keep waking on. this.pumpStep = pumpStep ?? (static () => { }); this.writeCompletionTimeout = writeCompletionTimeout ?? DefaultWriteCompletionTimeout; } diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessValueCache.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessValueCache.cs index abe133f..b6ecd3a 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessValueCache.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessValueCache.cs @@ -32,11 +32,26 @@ public sealed class MxAccessValueCache // the value is recorded from inside the waiter's own pumpStep (COM // dispatches OnDataChange on the STA), so the message wake is what does the // work there; this handle covers the cross-thread producers — the tests, and - // any future off-STA writer. Not IDisposable on purpose: the cache is - // co-owned by MxAccessBaseEventSink and MxAccessSession with no single - // teardown owner, and a late COM callback must never Set a disposed handle. - // One kernel event per session (one session per worker process), released by - // the handle's finalizer. + // any future off-STA writer. + // + // Single-waiter invariant: an AutoResetEvent releases exactly one waiter per + // Set. Today the sole waiter is the STA (one worker per session, and ReadBulk + // holds the apartment while it waits), so no wake is ever lost. A second + // concurrent waiter would not hang — the loser still re-checks the entry + // every fallback tick and still honors its deadline — it would just fall + // back to 50 ms latency for that iteration. + // + // NOT IDisposable, and that is one invariant with the wait's use of + // DangerousGetHandle: the cache is co-owned by MxAccessBaseEventSink and + // MxAccessSession with no single teardown owner, so a late COM callback + // could otherwise Set a disposed handle. StaWaitHelper deliberately does not + // DangerousAddRef the handle; GC.KeepAlive there closes the *finalization* + // race but not a dispose race — if this field were ever disposed while the + // STA sat inside the wait, the OS could recycle that handle value and the + // wait would silently attach to an unrelated object. Making these caches + // IDisposable is therefore not a free refactor: it must add DangerousAddRef/ + // DangerousRelease around the wait in the same change. One kernel event per + // session (one session per worker process), released by the finalizer. private readonly AutoResetEvent updateSignal = new(false); /// Records a fresh OnDataChange payload for the given handle pair. diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessWriteCompletionCache.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessWriteCompletionCache.cs index bfbfeaa..78a4114 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessWriteCompletionCache.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessWriteCompletionCache.cs @@ -31,11 +31,26 @@ public sealed class MxAccessWriteCompletionCache // worker the completion is recorded from inside the waiter's own pumpStep // (COM dispatches OnWriteComplete on the STA), so the message wake is what // does the work there; this handle covers the cross-thread producers — the - // tests, and any future off-STA recorder. Not IDisposable on purpose: the - // cache is co-owned by MxAccessBaseEventSink and MxAccessSession with no - // single teardown owner, and a late COM callback must never Set a disposed - // handle. One kernel event per session (one session per worker process), - // released by the handle's finalizer. + // tests, and any future off-STA recorder. + // + // Single-waiter invariant: an AutoResetEvent releases exactly one waiter per + // Set. Today the sole waiter is the STA (one worker per session, and the + // write path holds the apartment while it waits), so no wake is ever lost. A + // second concurrent waiter would not hang — the loser still re-checks the + // entry every fallback tick and still honors its deadline — it would just + // fall back to 50 ms latency for that iteration. + // + // NOT IDisposable, and that is one invariant with the wait's use of + // DangerousGetHandle: the cache is co-owned by MxAccessBaseEventSink and + // MxAccessSession with no single teardown owner, so a late COM callback + // could otherwise Set a disposed handle. StaWaitHelper deliberately does not + // DangerousAddRef the handle; GC.KeepAlive there closes the *finalization* + // race but not a dispose race — if this field were ever disposed while the + // STA sat inside the wait, the OS could recycle that handle value and the + // wait would silently attach to an unrelated object. Making these caches + // IDisposable is therefore not a free refactor: it must add DangerousAddRef/ + // DangerousRelease around the wait in the same change. One kernel event per + // session (one session per worker process), released by the finalizer. private readonly AutoResetEvent completionSignal = new(false); /// Records the status rows of a fresh OnWriteComplete callback for the given handle pair. diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Sta/StaMessagePump.cs b/src/ZB.MOM.WW.MxGateway.Worker/Sta/StaMessagePump.cs index 74d31b7..535eec3 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/Sta/StaMessagePump.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/Sta/StaMessagePump.cs @@ -9,10 +9,7 @@ namespace ZB.MOM.WW.MxGateway.Worker.Sta; public sealed class StaMessagePump { private const uint Infinite = 0xFFFFFFFF; - private const uint MsgWaitFailed = 0xFFFFFFFF; - private const uint MwmoInputAvailable = 0x0004; private const uint PmRemove = 0x0001; - private const uint QsAllInput = 0x04FF; /// Waits for a command wake event or Windows messages, pumping any pending messages. /// Event to signal when work is available. @@ -28,14 +25,14 @@ public sealed class StaMessagePump SafeWaitHandle safeHandle = commandWakeEvent.SafeWaitHandle; IntPtr[] handles = [safeHandle.DangerousGetHandle()]; - uint result = MsgWaitForMultipleObjectsEx( + uint result = StaNativeMethods.MsgWaitForMultipleObjectsEx( (uint)handles.Length, handles, timeoutMilliseconds, - QsAllInput, - MwmoInputAvailable); + StaNativeMethods.QsAllInput, + StaNativeMethods.MwmoInputAvailable); - if (result == MsgWaitFailed) + if (result == StaNativeMethods.MsgWaitFailed) { throw new InvalidOperationException( "The worker STA message pump failed while waiting for command work or Windows messages."); @@ -75,14 +72,6 @@ public sealed class StaMessagePump : (uint)Math.Ceiling(timeout.TotalMilliseconds); } - [DllImport("user32.dll", SetLastError = true)] - private static extern uint MsgWaitForMultipleObjectsEx( - uint count, - IntPtr[] handles, - uint milliseconds, - uint wakeMask, - uint flags); - [DllImport("user32.dll", SetLastError = true)] private static extern bool PeekMessage( out NativeMessage message, diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Sta/StaNativeMethods.cs b/src/ZB.MOM.WW.MxGateway.Worker/Sta/StaNativeMethods.cs new file mode 100644 index 0000000..032af95 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Worker/Sta/StaNativeMethods.cs @@ -0,0 +1,58 @@ +using System; +using System.Runtime.InteropServices; + +namespace ZB.MOM.WW.MxGateway.Worker.Sta; + +/// +/// The single declaration of the Win32 wait the worker's STA is built on. +/// Both waiters use it: for the runtime's outer +/// idle wait, and for the inner completion waits +/// a command performs while it already holds the apartment. +/// +/// +/// Deliberately one copy. This is the subtlest Win32 contract in the worker — +/// the wake mask decides which events can wake the STA, and +/// decides whether a message already seen by +/// a PeekMessage still counts as a wake. Two copies of it could drift +/// apart silently, and a divergence would show up only as COM events +/// arriving late on a live provider. +/// +internal static class StaNativeMethods +{ + /// Return value of a failed wait (WAIT_FAILED). + internal const uint MsgWaitFailed = 0xFFFFFFFF; + + /// + /// Wake even when the pending input has already been seen (but not + /// removed) by an earlier PeekMessage. Both callers peek before + /// they wait, so without this flag a message that arrived during the peek + /// would not wake the wait that follows it. + /// + internal const uint MwmoInputAvailable = 0x0004; + + /// Wake on any queued input, including sent and posted messages. + internal const uint QsAllInput = 0x04FF; + + /// + /// Waits for the given handles, for queued input matching + /// , or for the timeout — whichever comes + /// first. + /// + /// Number of handles in . + /// Handles to wait on; marshaled as a pointer to the array. + /// Timeout in milliseconds; 0xFFFFFFFF means INFINITE. + /// Input types that may wake the wait (). + /// Wait flags (). + /// + /// WAIT_OBJECT_0 + i for a signaled handle, WAIT_OBJECT_0 + count + /// for available input, WAIT_TIMEOUT (258), or + /// . + /// + [DllImport("user32.dll", SetLastError = true)] + internal static extern uint MsgWaitForMultipleObjectsEx( + uint count, + IntPtr[] handles, + uint milliseconds, + uint wakeMask, + uint flags); +} diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Sta/StaWaitHelper.cs b/src/ZB.MOM.WW.MxGateway.Worker/Sta/StaWaitHelper.cs index 7549d29..1171f7a 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/Sta/StaWaitHelper.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/Sta/StaWaitHelper.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.Runtime.InteropServices; using System.Threading; using Microsoft.Win32.SafeHandles; @@ -17,23 +18,22 @@ namespace ZB.MOM.WW.MxGateway.Worker.Sta; /// Deliberately separate from rather than a new /// method on it: the two caches that use this are plain per-session objects /// with no pump dependency (they take the pump step as an injected delegate -/// so tests can drive them off the STA). Keeping the helper in this folder -/// keeps every user32 wait signature in the worker reviewable in one -/// place; the declaration below mirrors the pump's exactly. +/// so tests can drive them off the STA). The wait itself is not duplicated — +/// both this and the pump call the single declaration in +/// . /// internal static class StaWaitHelper { /// - /// Ceiling on the blind fallback tick. A caller's poll interval is - /// clamped to this so no wait can go longer than 50 ms without calling - /// its pump step, even when the process has no message queue to wake it - /// (unit-test threads) and the wait handle is never signaled. + /// Ceiling on the fallback tick — the longest a wait may block when + /// nothing wakes it. A caller's poll interval is clamped to this, so a + /// caller's pump step keeps running at least this often no matter what + /// interval it asked for, even on a thread that never receives a message + /// and whose signal is never set. /// internal const int MaxFallbackTickMilliseconds = 50; - private const uint MsgWaitFailed = 0xFFFFFFFF; - private const uint MwmoInputAvailable = 0x0004; - private const uint QsAllInput = 0x04FF; + private static int waitFailureReported; /// /// Blocks the calling thread until is set, a @@ -42,6 +42,29 @@ internal static class StaWaitHelper /// never blinds the thread to inbound COM event messages: the wait /// returns the instant one arrives, so the caller can dispatch it. /// + /// + /// + /// Caller contract: the pump step must drain the message queue + /// to empty on every call. The wait uses + /// , which reports + /// input that is merely present — including input an earlier + /// PeekMessage has already seen but not removed. A caller + /// whose pump step drains only part of the queue, or does nothing at + /// all, therefore leaves a message that satisfies the wake condition + /// forever: every wait returns immediately, and the caller's poll + /// loop degenerates into a busy loop that spins the thread at 100% + /// until its deadline. That is a CPU fault, not a correctness one — + /// the loop still honors its deadline and return shape — but it must + /// not ship. Drain with PeekMessage(..., PM_REMOVE) in a loop + /// until it returns false, exactly as + /// does. + /// + /// + /// The wait does not DangerousAddRef the handle. That is + /// sound only because the callers never dispose their wait handles — + /// see the coupled invariant documented on the caches' signal fields. + /// + /// /// Wait handle set by whoever produces the awaited result. /// Maximum wait; values at or below zero return immediately. internal static void WaitForSignalOrMessages( @@ -60,20 +83,28 @@ internal static class StaWaitHelper SafeWaitHandle safeHandle = signal.SafeWaitHandle; IntPtr[] handles = [safeHandle.DangerousGetHandle()]; - uint result = MsgWaitForMultipleObjectsEx( + uint result = StaNativeMethods.MsgWaitForMultipleObjectsEx( (uint)handles.Length, handles, (uint)timeoutMilliseconds, - QsAllInput, - MwmoInputAvailable); + StaNativeMethods.QsAllInput, + StaNativeMethods.MwmoInputAvailable); + + // Read before anything else can run another P/Invoke and overwrite the + // thread's last-error slot. + int lastError = result == StaNativeMethods.MsgWaitFailed + ? Marshal.GetLastWin32Error() + : 0; // The handle is a field of a live caller object, so it cannot be // collected mid-call; KeepAlive states that dependency explicitly // rather than relying on it. GC.KeepAlive(signal); - if (result == MsgWaitFailed) + if (result == StaNativeMethods.MsgWaitFailed) { + ReportWaitFailureOnce(lastError); + // Degrade to the pre-existing blind sleep instead of throwing or // hot-spinning: the caller is a bounded, deadline-driven poll loop, // and a failed wait must not turn a timed-out write into a fault. @@ -108,11 +139,30 @@ internal static class StaWaitHelper : tick; } - [DllImport("user32.dll", SetLastError = true)] - private static extern uint MsgWaitForMultipleObjectsEx( - uint count, - IntPtr[] handles, - uint milliseconds, - uint wakeMask, - uint flags); + /// + /// Traces the first wait failure of the process and stays silent after + /// that. + /// + /// + /// The degrade path is silent by design — a failed wait still honors the + /// caller's deadline — but a permanently failing wait means every + /// completion wait has quietly reverted to the sleep-polled behavior this + /// class exists to replace, and nothing else would ever say so. One trace + /// makes that discoverable; latching it keeps a repeating failure from + /// flooding the trace listener from inside a poll loop. + /// + /// The Win32 error captured immediately after the failed wait. + private static void ReportWaitFailureOnce(int lastError) + { + if (Interlocked.Exchange(ref waitFailureReported, 1) != 0) + { + return; + } + + Trace.TraceWarning( + "MsgWaitForMultipleObjectsEx failed in the worker STA completion wait (Win32 error {0}). " + + "The wait has degraded to blind sleeping: deadlines are still honored, but inbound " + + "MXAccess COM events will only dispatch at poll-tick granularity. Reported once per process.", + lastError); + } } From b5ea6bb461eff05f3d5e109b32edbb7dc5effcfd Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 17:19:41 -0400 Subject: [PATCH 35/41] fix(alarms): fetch/poll ceilings; truncation-semantics docs; log-format conformance --- docs/AlarmClientDiscovery.md | 20 +++++++ docs/DesignDecisions.md | 28 +++++++-- docs/GatewayConfiguration.md | 4 +- .../Configuration/AlarmsOptions.cs | 26 ++++---- .../Configuration/GatewayOptionsValidator.cs | 20 +++++-- .../appsettings.json | 4 +- .../GatewayOptionsValidatorTests.cs | 44 ++++++++++---- .../MxAccess/MxAccessStaSessionTests.cs | 45 ++++++++++++++ .../MxAccess/WnWrapAlarmConsumerXmlTests.cs | 31 ++++++++++ .../MxAccess/MxAccessStaSession.cs | 13 +++- .../MxAccess/WnWrapAlarmConsumer.cs | 59 +++++++++++++++++-- 11 files changed, 254 insertions(+), 40 deletions(-) diff --git a/docs/AlarmClientDiscovery.md b/docs/AlarmClientDiscovery.md index e03ac9d..f43fff8 100644 --- a/docs/AlarmClientDiscovery.md +++ b/docs/AlarmClientDiscovery.md @@ -140,6 +140,26 @@ Two viable A.2 designs given the probe data: poll period; modest CPU floor because the call is cheap. Matches the heartbeat-style WM 0xC275 semantics — AVEVA itself runs a poll loop internally. + + As shipped, this is the chosen design, and the cadence is **no + longer fixed at 500 ms**: it is the 500 ms *default* of + `MxGateway:Alarms:PollIntervalMilliseconds` (range 100 ms – 1 h), + which the gateway hands the worker through the + `MXGATEWAY_ALARM_POLL_INTERVAL_MS` environment variable. The + per-fetch cap is likewise configurable + (`MxGateway:Alarms:MaxAlarmsPerFetch`, default 1024). + + One snapshot rule matters when reading the capture below: a fetch + that returns exactly the cap is treated as **truncated**, and the + worker *merges* it into the retained snapshot instead of replacing + it. `GetXmlCurrentAlarms2` caps its reply with no "more available" + flag, so a capped reply is authoritative about presence only — + alarms it had no room to mention are retained rather than allowed + to vanish, because their disappearance is what the gateway's + reconcile pass reads as a clear. Only a sub-cap fetch replaces the + snapshot wholesale and can therefore clear alarms. See + `docs/DesignDecisions.md`, "Alarms — a capped snapshot fetch never + implies a clear". 2. **Hook AVEVA's internal window.** Discover AVEVA's own window (`hwnd=0x18032E` in the probe), `SetWindowsHookEx` or `SetWindowSubclass` on it, and intercept WM 0xC275 on AVEVA's diff --git a/docs/DesignDecisions.md b/docs/DesignDecisions.md index 6a1d8f3..e315ce0 100644 --- a/docs/DesignDecisions.md +++ b/docs/DesignDecisions.md @@ -172,6 +172,19 @@ Consequences, and how this sits with the existing failover/reconcile design: happened is broadcast to every `StreamAlarms` subscriber and cannot be taken back. Consumers already apply transitions as "set this alarm to this state", so a repeated or delayed Clear is absorbed. +- **Under *sustained* truncation, some intermediate history is lost — end state + is not.** For an alarm that stays outside the fetch window, a full + clear→re-raise cycle that begins and ends between two sightings emits **no + transitions at all**: the retained record is identical before and after, so + the diff sees nothing to report. Consumers that render current state are + correct; consumers that *count occurrences* lose an event. Likewise, an + operator acknowledgement of an out-of-window alarm does not reach the feed + until that alarm re-enters a fetch window, at which point the reconcile + repairs the acked state. This is a strictly better failure than the + pre-guard behaviour (which fabricated a Clear for every out-of-window alarm + on every poll), but it is not lossless, and it is another reason a + persistently truncating deployment is a configuration defect to fix rather + than a mode to run in. - **It does not synthesize anything.** Suppressing an inference is the opposite of inventing an event; no transition is fabricated on a truncated poll. - **Failover is unaffected.** `FailoverAlarmConsumer` selects which @@ -179,10 +192,17 @@ Consequences, and how this sits with the existing failover/reconcile design: consumer's own snapshot bookkeeping and changes neither the failure counting that triggers failover nor the subtag standby's snapshot, which is built from a bounded watch-list and has no per-fetch cap to hit. -- **Operators get told.** A truncated poll logs a rate-limited (once per - minute) `AlarmSnapshotTruncated` warning carrying the cap, the record counts, - and the running truncated-fetch total — identifiers and counts only, never - tag names, values, limits, or comments. A galaxy that truncates persistently +- **Operators get told, weakly.** A truncated poll logs a rate-limited (once + per minute) `AlarmSnapshotTruncated` warning carrying the cap, the record + counts, and the running truncated-fetch total — identifiers and counts only, + never tag names, values, limits, or comments. Be honest about its reach: it + goes to the worker's console/stderr, which is captured on dev hosts but is + not a metric, not a dashboard tile, and not part of any session-status or + alarm-feed payload, so a production deployment can truncate indefinitely + without anyone noticing. Surfacing truncation as a **structural** degraded + status (a field on the alarm-provider mode/status surface the dashboard and + `StreamAlarms` consumers already read) is filed as a follow-up; until it + lands, the log line is the only signal. A galaxy that truncates persistently is a configuration problem: raise `MxGateway:Alarms:MaxAlarmsPerFetch`. ## Session-Resilience Epic Scope diff --git a/docs/GatewayConfiguration.md b/docs/GatewayConfiguration.md index 7d73c9e..5feb37a 100644 --- a/docs/GatewayConfiguration.md +++ b/docs/GatewayConfiguration.md @@ -418,8 +418,8 @@ behavior. | `MxGateway:Alarms:SubscriptionExpression` | _(empty)_ | AVEVA alarm-subscription expression the monitor subscribes on startup, in canonical `\\\Galaxy!` form. The literal `Galaxy` provider is correct regardless of the Galaxy database name. When empty and `Enabled` is `true`, the gateway falls back to `\\\Galaxy!` if `DefaultArea` is set. | | `MxGateway:Alarms:DefaultArea` | _(empty)_ | Area name used to compose a default subscription when `SubscriptionExpression` is empty. If both are empty while `Enabled` is `true`, the monitor faults with a configuration diagnostic. | | `MxGateway:Alarms:ReconcileIntervalSeconds` | `30` | How often the monitor reconciles its in-process alarm cache against the worker's authoritative active-alarm snapshot, catching transitions the live poll-and-diff feed missed. Floored at 5 seconds. | -| `MxGateway:Alarms:PollIntervalMilliseconds` | `500` | Cadence at which the worker's STA polls the AVEVA alarm consumer (`GetXmlCurrentAlarms2`) for the active-alarm snapshot the live feed diffs. Must be `>= 100`: every poll is a COM call plus an XML parse on the same STA that serves reads and writes, so a tighter cadence starves the command path. The gateway conveys the value to the worker via the `MXGATEWAY_ALARM_POLL_INTERVAL_MS` environment variable; a missing or unusable value leaves the worker on the 500 ms default rather than failing the session. | -| `MxGateway:Alarms:MaxAlarmsPerFetch` | `1024` | Cap the worker passes to `GetXmlCurrentAlarms2`'s `maxAlmCnt`. Must be `>= 64`. It doubles as the **truncation threshold**: a fetch returning exactly this many records is treated as truncated, because the COM API caps its reply with no "more available" flag. On a truncated poll the worker retains the alarms the capped reply could not mention instead of letting their absence read as a clear, and logs a rate-limited `AlarmSnapshotTruncated` warning (identifiers and counts only). Raise this on galaxies whose steady-state active-alarm count approaches the cap — a galaxy permanently above it holds stale entries in the snapshot until a sub-cap poll. Conveyed to the worker via the `MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH` environment variable; a missing or unusable value leaves the worker on the 1024 default. | +| `MxGateway:Alarms:PollIntervalMilliseconds` | `500` | Cadence at which the worker's STA polls the AVEVA alarm consumer (`GetXmlCurrentAlarms2`) for the active-alarm snapshot the live feed diffs. Must be between `100` and `3600000` (one hour): every poll is a COM call plus an XML parse on the same STA that serves reads and writes, so a tighter cadence starves the command path, while a value above an hour stops being a cadence and silently disables alarm polling. The gateway conveys the value to the worker via the `MXGATEWAY_ALARM_POLL_INTERVAL_MS` environment variable; a missing or out-of-range value leaves the worker on the 500 ms default rather than failing the session. | +| `MxGateway:Alarms:MaxAlarmsPerFetch` | `1024` | Cap the worker passes to `GetXmlCurrentAlarms2`'s `maxAlmCnt`. Must be between `64` and `65536` — the worker is a 32-bit process that materializes each reply as one BSTR plus a full `XmlDocument`, so an unbounded cap faults the STA with an out-of-memory rather than merely slowing it. It doubles as the **truncation threshold**: a fetch returning exactly this many records is treated as truncated, because the COM API caps its reply with no "more available" flag. On a truncated poll the worker retains the alarms the capped reply could not mention instead of letting their absence read as a clear, and logs a rate-limited `AlarmSnapshotTruncated` warning to its stderr (identifiers and counts only). **Remediation when you see that warning: raise this value** so the steady-state active-alarm count fits inside one fetch. A galaxy permanently above the cap holds stale entries in the snapshot until a sub-cap poll, and loses clear→re-raise cycles that happen entirely out of window (see `docs/DesignDecisions.md`). Conveyed to the worker via the `MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH` environment variable; a missing or out-of-range value leaves the worker on the 1024 default. | The alarm monitor is independent of client sessions: `AcknowledgeAlarm` and `StreamAlarms` are session-less RPCs served by the monitor. diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/AlarmsOptions.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/AlarmsOptions.cs index 98259f5..94363bf 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/AlarmsOptions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/AlarmsOptions.cs @@ -49,22 +49,26 @@ public sealed class AlarmsOptions /// /// Cadence at which the worker's STA polls the AVEVA alarm consumer /// (GetXmlCurrentAlarms2) for the current active-alarm snapshot. - /// Default 500 ms; must be at least 100 ms. Every poll is a COM call - /// plus an XML parse on the STA that also serves reads and writes, so - /// driving it below 100 ms starves the command path. Conveyed to the - /// worker through the MXGATEWAY_ALARM_POLL_INTERVAL_MS - /// environment variable. + /// Default 500 ms; must be between 100 ms and 3,600,000 ms (one hour). + /// Every poll is a COM call plus an XML parse on the STA that also + /// serves reads and writes, so driving it below 100 ms starves the + /// command path; above an hour the cadence stops being a cadence and + /// silently disables alarm polling. Conveyed to the worker through the + /// MXGATEWAY_ALARM_POLL_INTERVAL_MS environment variable. /// public int PollIntervalMilliseconds { get; init; } = 500; /// /// Cap the worker passes to GetXmlCurrentAlarms2's - /// maxAlmCnt argument. Default 1024; must be at least 64. A - /// fetch that comes back holding exactly this many records is treated - /// as truncated: the worker keeps the alarms the capped fetch could - /// not mention in its snapshot rather than letting their absence read - /// as a clear. Raise it on galaxies whose steady-state active-alarm - /// count approaches the cap. Conveyed to the worker through the + /// maxAlmCnt argument. Default 1024; must be between 64 and + /// 65,536 — the worker is a 32-bit process that materializes each + /// fetch as one BSTR plus a full XmlDocument, so an unbounded cap + /// faults the STA rather than merely slowing it. A fetch that comes + /// back holding exactly this many records is treated as truncated: the + /// worker keeps the alarms the capped fetch could not mention in its + /// snapshot rather than letting their absence read as a clear. Raise it + /// on galaxies whose steady-state active-alarm count approaches the + /// cap. Conveyed to the worker through the /// MXGATEWAY_ALARM_MAX_ALARMS_PER_FETCH environment variable. /// public int MaxAlarmsPerFetch { get; init; } = 1024; diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs index 55e31c1..eadeea3 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs @@ -427,23 +427,35 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase MaximumAlarmPollIntervalMilliseconds) { builder.Add( - $"MxGateway:Alarms:PollIntervalMilliseconds must be greater than or equal to {MinimumAlarmPollIntervalMilliseconds}."); + $"MxGateway:Alarms:PollIntervalMilliseconds must be between {MinimumAlarmPollIntervalMilliseconds} and {MaximumAlarmPollIntervalMilliseconds}."); } - if (options.MaxAlarmsPerFetch < MinimumMaxAlarmsPerFetch) + if (options.MaxAlarmsPerFetch is < MinimumMaxAlarmsPerFetch or > MaximumMaxAlarmsPerFetch) { builder.Add( - $"MxGateway:Alarms:MaxAlarmsPerFetch must be greater than or equal to {MinimumMaxAlarmsPerFetch}."); + $"MxGateway:Alarms:MaxAlarmsPerFetch must be between {MinimumMaxAlarmsPerFetch} and {MaximumMaxAlarmsPerFetch}."); } if (!options.Enabled) diff --git a/src/ZB.MOM.WW.MxGateway.Server/appsettings.json b/src/ZB.MOM.WW.MxGateway.Server/appsettings.json index e99f5e0..ad80bcf 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/appsettings.json +++ b/src/ZB.MOM.WW.MxGateway.Server/appsettings.json @@ -86,7 +86,9 @@ "Enabled": true, "SubscriptionExpression": "\\\\DESKTOP-6JL3KKO\\Galaxy!DEV", "DefaultArea": "", - "ReconcileIntervalSeconds": 30 + "ReconcileIntervalSeconds": 30, + "PollIntervalMilliseconds": 500, + "MaxAlarmsPerFetch": 1024 } } } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs index a5cb35a..2878517 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs @@ -177,9 +177,10 @@ public sealed class GatewayOptionsValidatorTests } /// - /// A poll cadence below the 100 ms floor must fail validation. Both - /// values are stamped onto every worker launch environment, so they - /// are validated whether or not the central alarm monitor is enabled. + /// A poll cadence outside the 100 ms – 1 h range must fail validation. + /// Both values are stamped onto every worker launch environment, so + /// they are validated whether or not the central alarm monitor is + /// enabled. /// /// Cadence under test. /// Whether the central alarm monitor is on. @@ -188,7 +189,12 @@ public sealed class GatewayOptionsValidatorTests [InlineData(0, false)] [InlineData(-1, false)] [InlineData(99, true)] - public void Validate_Fails_WhenAlarmPollIntervalBelowFloor( + // Above the one-hour ceiling the cadence stops being a cadence: int.MaxValue + // milliseconds is ~24 days, which silently disables alarm polling. + [InlineData(3_600_001, false)] + [InlineData(int.MaxValue, false)] + [InlineData(int.MaxValue, true)] + public void Validate_Fails_WhenAlarmPollIntervalOutOfRange( int pollIntervalMilliseconds, bool alarmsEnabled) { @@ -210,9 +216,12 @@ public sealed class GatewayOptionsValidatorTests } /// - /// A per-fetch cap below the 64-record floor must fail validation. The - /// cap doubles as the truncation-detection threshold in the worker, so - /// a tiny cap would make almost every fetch read as truncated. + /// A per-fetch cap outside the 64 – 65,536 range must fail validation. + /// The cap doubles as the truncation-detection threshold in the worker, + /// so a tiny cap would make almost every fetch read as truncated; and + /// the worker is a 32-bit process that materializes each reply as one + /// BSTR plus a full XmlDocument, so an unbounded cap is an + /// out-of-memory fault on the STA rather than a slow poll. /// /// Cap under test. /// Whether the central alarm monitor is on. @@ -221,7 +230,10 @@ public sealed class GatewayOptionsValidatorTests [InlineData(0, false)] [InlineData(-1, false)] [InlineData(63, true)] - public void Validate_Fails_WhenMaxAlarmsPerFetchBelowFloor( + [InlineData(65_537, false)] + [InlineData(int.MaxValue, false)] + [InlineData(int.MaxValue, true)] + public void Validate_Fails_WhenMaxAlarmsPerFetchOutOfRange( int maxAlarmsPerFetch, bool alarmsEnabled) { @@ -242,9 +254,15 @@ public sealed class GatewayOptionsValidatorTests f => f.Contains("MxGateway:Alarms:MaxAlarmsPerFetch", StringComparison.Ordinal)); } - /// Verifies the floor values themselves are accepted. - [Fact] - public void Validate_Succeeds_AtAlarmPollCadenceAndFetchCapFloors() + /// Verifies the boundary values themselves are accepted at both ends. + /// Cadence under test. + /// Cap under test. + [Theory] + [InlineData(100, 64)] // floors + [InlineData(3_600_000, 65_536)] // ceilings + public void Validate_Succeeds_AtAlarmPollCadenceAndFetchCapBoundaries( + int pollIntervalMilliseconds, + int maxAlarmsPerFetch) { GatewayOptions options = CloneWithAlarms( ValidOptions(), @@ -252,8 +270,8 @@ public sealed class GatewayOptionsValidatorTests { Enabled = true, DefaultArea = "Galaxy", - PollIntervalMilliseconds = 100, - MaxAlarmsPerFetch = 64, + PollIntervalMilliseconds = pollIntervalMilliseconds, + MaxAlarmsPerFetch = maxAlarmsPerFetch, }); ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, options); diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessStaSessionTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessStaSessionTests.cs index 44894ea..aace1d7 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessStaSessionTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessStaSessionTests.cs @@ -57,6 +57,51 @@ public sealed class MxAccessStaSessionTests } } + /// + /// The alarm poll cadence comes from the launcher-set environment + /// variable; a missing, unparseable, or out-of-range value must fall + /// back to the 500 ms default rather than throw. The ceiling matters as + /// much as the floor: milliseconds is ~24 + /// days, which silently disables alarm polling altogether. + /// + /// Raw environment value under test. + /// Expected resolved cadence, in milliseconds. + [Theory] + [InlineData(null, 500)] + [InlineData("", 500)] + [InlineData("not-a-number", 500)] + [InlineData("0", 500)] + [InlineData("-1", 500)] + [InlineData("99", 500)] + [InlineData("100", 100)] + [InlineData("250", 250)] + [InlineData("3600000", 3600000)] + [InlineData("3600001", 500)] + [InlineData("2147483647", 500)] + public void ResolveAlarmPollInterval_WithEnvironmentValue_FallsBackToDefaultWhenOutOfRange( + string? environmentValue, + int expectedMilliseconds) + { + string? original = Environment.GetEnvironmentVariable( + MxAccessStaSession.AlarmPollIntervalEnvironmentVariableName); + try + { + Environment.SetEnvironmentVariable( + MxAccessStaSession.AlarmPollIntervalEnvironmentVariableName, + environmentValue); + + Assert.Equal( + TimeSpan.FromMilliseconds(expectedMilliseconds), + MxAccessStaSession.ResolveAlarmPollInterval()); + } + finally + { + Environment.SetEnvironmentVariable( + MxAccessStaSession.AlarmPollIntervalEnvironmentVariableName, + original); + } + } + /// /// Verifies that StartAsync creates the MXAccess COM object and attaches the event sink on the STA thread. /// diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/WnWrapAlarmConsumerXmlTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/WnWrapAlarmConsumerXmlTests.cs index 682365d..7dd9448 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/WnWrapAlarmConsumerXmlTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/WnWrapAlarmConsumerXmlTests.cs @@ -496,6 +496,11 @@ public sealed class WnWrapAlarmConsumerXmlTests [InlineData("63", 1024)] [InlineData("64", 64)] [InlineData("4096", 4096)] + [InlineData("65536", 65536)] + // Above the ceiling: the x86 worker materializes the whole reply as one + // BSTR plus an XmlDocument, so an unbounded cap is an OOM on the STA. + [InlineData("65537", 1024)] + [InlineData("2147483647", 1024)] public void ResolveMaxAlarmsPerFetch_WithEnvironmentValue_FallsBackToDefaultWhenUnusable( string? environmentValue, int expected) @@ -518,6 +523,32 @@ public sealed class WnWrapAlarmConsumerXmlTests } } + /// + /// A galaxy parked above the cap truncates on every poll, so the + /// warning must be throttled: two truncated polls inside one interval + /// produce exactly one line, and the next one only after the full + /// interval has elapsed. + /// + [Fact] + public void ShouldWarnTruncation_ThrottlesConsecutiveTruncatedPollsToOneWarningPerInterval() + { + // Seeded so the very first truncated poll always warns. + const long NeverWarned = -60_000; + + // Poll 1 at t=0: warns, and records t=0 as the last warning. + Assert.True(WnWrapAlarmConsumer.ShouldWarnTruncation(0, NeverWarned)); + + // Poll 2 half a second later (the default cadence): suppressed. + Assert.False(WnWrapAlarmConsumer.ShouldWarnTruncation(500, 0)); + + // Still suppressed just shy of the interval... + Assert.False(WnWrapAlarmConsumer.ShouldWarnTruncation(59_999, 0)); + + // ...and allowed again exactly on it. + Assert.True(WnWrapAlarmConsumer.ShouldWarnTruncation(60_000, 0)); + Assert.True(WnWrapAlarmConsumer.ShouldWarnTruncation(120_000, 60_000)); + } + /// /// Builds a well-formed ALARM_RECORDS payload with /// distinct alarms. GUIDs are the dashless diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs index 6c2a480..6050993 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessStaSession.cs @@ -35,6 +35,15 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession /// internal static readonly TimeSpan MinimumAlarmPollInterval = TimeSpan.FromMilliseconds(100); + /// + /// Ceiling on the resolved alarm poll cadence (one hour). Mirrors the + /// gateway-side MxGateway:Alarms:PollIntervalMilliseconds maximum. + /// Without it, int.MaxValue milliseconds — which passed startup + /// validation before this existed — silently disables alarm polling + /// for ~24 days rather than reporting a misconfiguration. + /// + internal static readonly TimeSpan MaximumAlarmPollInterval = TimeSpan.FromMilliseconds(3_600_000); + /// Default alarm poll cadence when the environment says nothing usable. internal static readonly TimeSpan DefaultAlarmPollInterval = TimeSpan.FromMilliseconds(500); @@ -235,7 +244,9 @@ public sealed class MxAccessStaSession : IWorkerRuntimeSession } TimeSpan resolved = TimeSpan.FromMilliseconds(milliseconds); - return resolved < MinimumAlarmPollInterval ? DefaultAlarmPollInterval : resolved; + return resolved < MinimumAlarmPollInterval || resolved > MaximumAlarmPollInterval + ? DefaultAlarmPollInterval + : resolved; } /// diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/WnWrapAlarmConsumer.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/WnWrapAlarmConsumer.cs index 18085e1..b9f44f4 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/WnWrapAlarmConsumer.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/WnWrapAlarmConsumer.cs @@ -58,10 +58,20 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer /// internal const int MinimumMaxAlarmsPerFetch = 64; + /// + /// Ceiling on the resolved per-fetch cap. Mirrors the gateway-side + /// MxGateway:Alarms:MaxAlarmsPerFetch maximum. The worker is a + /// 32-bit process: every poll materializes the whole reply as one BSTR + /// and then a full over it, so an unbounded + /// cap ( passed startup validation before + /// this existed) is an out-of-memory fault on the STA, not a slow poll. + /// + internal const int MaximumMaxAlarmsPerFetch = 65_536; + /// /// Environment variable the gateway's WorkerProcessLauncher /// sets from MxGateway:Alarms:MaxAlarmsPerFetch. A missing, - /// unparseable, or below-floor value falls back to + /// unparseable, or out-of-range value falls back to /// — a bad environment value /// must never keep the alarm consumer from starting. /// @@ -117,10 +127,13 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer /// /// Resolves the per-fetch cap from the launcher-provided environment - /// variable. A missing, unparseable, or below-floor value falls back + /// variable. A missing, unparseable, or out-of-range value falls back /// to rather than throwing: /// polling at the default cap is always safe, and a session that /// refuses to subscribe over a mistyped environment variable is not. + /// The ceiling is enforced here as well as in the gateway validator so + /// a hand-set environment (or a future launcher bug) cannot hand the + /// x86 worker a cap that faults it on the first poll. /// /// The cap passed to GetXmlCurrentAlarms2. internal static int ResolveMaxAlarmsPerFetch() @@ -132,6 +145,7 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer CultureInfo.InvariantCulture, out int cap) && cap >= MinimumMaxAlarmsPerFetch + && cap <= MaximumMaxAlarmsPerFetch ? cap : DefaultMaxAlarmsPerFetch; } @@ -498,7 +512,7 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer { total = ++truncatedFetchCount; elapsed = truncationWarningClock.ElapsedMilliseconds; - if (elapsed - lastTruncationWarningMilliseconds < TruncationWarningIntervalMilliseconds) + if (!ShouldWarnTruncation(elapsed, lastTruncationWarningMilliseconds)) { return; } @@ -506,13 +520,24 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer lastTruncationWarningMilliseconds = elapsed; } + // Format matches WorkerConsoleLogger's "level= event= k=v k=v" + // line shape so the worker's stderr parses uniformly. Deviation, on + // purpose: IWorkerLogger exposes only Information and Error, so + // "Warning" is a third level string no other worker line emits. A + // truncated snapshot is not an error (the poll succeeded and the + // snapshot is safe) but it is not routine either, so downgrading it to + // Information would bury it. Revisit if the truncation signal becomes + // structural — see docs/DesignDecisions.md. + // // Identifiers and counts only — no tag names, values, limits, or // comments, per the gateway's "don't log tag values by default" rule. + // The trailing note stays a single bare k=v token (no spaces, no + // semicolons); remediation prose lives in docs/GatewayConfiguration.md. string message = string.Format( CultureInfo.InvariantCulture, "level=Warning event=AlarmSnapshotTruncated maxAlarmsPerFetch={0} fetchedRecords={1} " + "parsedRecords={2} retainedSnapshotSize={3} truncatedFetchesSinceStart={4} " - + "note=absence-implies-clear suppressed for this poll; raise MxGateway:Alarms:MaxAlarmsPerFetch", + + "note=truncated-snapshot-retained", maxAlarmsPerFetch, fetchedRecordCount, parsedRecordCount, @@ -523,6 +548,25 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer sink(message); } + /// + /// Rate-limit decision for the truncated-fetch warning: emit only when + /// a full has + /// elapsed since the last one. A galaxy parked above the cap truncates + /// on every poll, so consecutive truncated polls inside one interval + /// must produce exactly one line. Exposed as internal static + /// so the throttle is unit-testable without the wnwrapConsumer COM + /// object. + /// + /// Consumer-lifetime clock reading for this poll. + /// Clock reading when the last warning was emitted. + /// when this poll may emit a warning. + internal static bool ShouldWarnTruncation( + long elapsedMilliseconds, + long lastWarningMilliseconds) + { + return elapsedMilliseconds - lastWarningMilliseconds >= TruncationWarningIntervalMilliseconds; + } + /// /// Pure snapshot-to-transitions diff. Compares the previous polled /// snapshot to the next snapshot and produces one @@ -641,6 +685,13 @@ public sealed class WnWrapAlarmConsumer : IMxAccessAlarmConsumer // first-match-wins (null means "not seen yet"), and a single // coalesce to string.Empty at the end — which keeps the absent and // empty cases indistinguishable exactly as before. + // + // One theoretical divergence: XmlNode.Name is the QName, so a child + // carrying its own default-namespace declaration () + // matches here where the unprefixed XPath name test would not. It is + // theoretical — wnwrap emits no namespaces, and a namespaced ALARM + // wrapper makes the outer SelectNodes return nothing either way — and + // it errs permissive (field populated rather than silently dropped). string? guidHex = null; string? xmlDate = null; string? xmlTime = null; From 58d97ad4e809f6e4ecc51bcf648706f38100ca7d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 17:27:47 -0400 Subject: [PATCH 36/41] perf(worker): memoize event frame size at enqueue; drain stops sizing under the STA's lock --- docs/MxAccessWorkerInstanceDesign.md | 6 +- .../MxAccess/MxAccessEventQueueTests.cs | 166 ++++++++++++++++++ .../MxAccess/MxAccessEventQueue.cs | 106 +++++++++-- 3 files changed, 263 insertions(+), 15 deletions(-) diff --git a/docs/MxAccessWorkerInstanceDesign.md b/docs/MxAccessWorkerInstanceDesign.md index a776161..c194340 100644 --- a/docs/MxAccessWorkerInstanceDesign.md +++ b/docs/MxAccessWorkerInstanceDesign.md @@ -541,7 +541,11 @@ is bounded on **two** axes because no diagnostics command may be session-fatal: maximum less a 64 KiB envelope/reply-wrapper reserve, and the size decision happens inside the event queue's lock, so an event is dequeued only once it is known to fit. An event that does not fit stays at the head of the queue and is - never lost. + never lost. Each event's serialized size is *measured* once at enqueue, outside + that lock, and stored beside it: the drain only compares memoized numbers, so a + large drain never walks messages under the lock the STA needs to enqueue the + next COM callback. The memoized size cannot go stale because an enqueued event + is never mutated again (WRK-11). Truncation is reported in the reply's existing `DiagnosticMessage` ("N events returned, M remain; repeat DrainEvents for the rest") rather than in a diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventQueueTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventQueueTests.cs index f01c42f..f2a5f1e 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventQueueTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessEventQueueTests.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Threading; using System.Threading.Tasks; +using Google.Protobuf.WellKnownTypes; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Worker.MxAccess; @@ -252,6 +253,158 @@ public sealed class MxAccessEventQueueTests Assert.Equal(3, result.RemainingCount); } + /// + /// Brackets the byte budget's per-event charge to the exact serialized size of the event. + /// The queue measures that size once at enqueue rather than during the drain, so this pins + /// the two ends of the charge against the very instance the queue holds: a budget one byte + /// short must refuse the head (and leave it queued, WRK-21), and a budget of exactly the + /// charge must ship it. An undercharge — sizing the event before Enqueue stamps its worker + /// sequence and timestamp, say — passes the first probe and breaks the frame guarantee; an + /// overcharge of even one byte fails the second. The preStamped case covers an event + /// that arrives with those two fields already filled, which Enqueue overwrites. + /// + /// Whether the event carries a stale sequence/timestamp on arrival. + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Drain_ByteBudget_ChargesTheEventsExactSerializedSize(bool preStamped) + { + MxAccessEventQueue queue = new(capacity: 4); + MxEvent mxEvent = CreateEventWithPayload(itemHandle: 7, payloadLength: 300); + if (preStamped) + { + mxEvent.WorkerSequence = ulong.MaxValue; + mxEvent.WorkerTimestamp = Timestamp.FromDateTime( + new DateTime(1990, 1, 1, 0, 0, 0, DateTimeKind.Utc)); + } + + queue.Enqueue(mxEvent); + + // Enqueue takes ownership but never mutates the event afterwards, so the retained reference + // measures exactly what the queue holds — no second drain is needed to learn the cost. + Assert.Equal(1UL, mxEvent.WorkerSequence); + int exactCost = DrainCostOf(new WorkerEvent { Event = mxEvent }); + + WorkerEventDrainResult refused = queue.Drain(maxEvents: 0, maxTotalBytes: exactCost - 1); + Assert.Empty(refused.Events); + Assert.True(refused.TruncatedBySize); + Assert.Equal(1UL, refused.OversizedHeadSequence); + Assert.Equal(1, queue.Count); + + WorkerEventDrainResult drained = queue.Drain(maxEvents: 0, maxTotalBytes: exactCost); + Assert.Single(drained.Events); + Assert.Same(mxEvent, drained.Events[0].Event); + Assert.False(drained.TruncatedBySize); + Assert.Equal(0, queue.Count); + } + + /// + /// Pins the batch boundaries of a byte-budgeted walk over a mixed-size queue. Each batch is + /// checked against the true serialized sizes of the events it returned: no batch may exceed + /// the budget (an undercharge would build a reply past the frame maximum) and no batch may + /// stop early — the next event, at its real size, must not have fit (an overcharge would + /// ship frames smaller than the negotiated maximum allows). Both bounds come from the + /// drained events themselves, so any drift between the size memoized at enqueue and the real + /// one moves a boundary and fails here. + /// + [Fact] + public void Drain_ByteBudget_MixedSizes_KeepsBatchBoundariesOnTheRealSizes() + { + int[] payloadLengths = { 8, 512, 40, 2_048, 96, 1_200, 16, 700 }; + const int eventCount = 240; + + // Comfortably above the largest single event's cost, so the walk is never blocked by an + // oversized head and every stop is a genuine budget boundary. + const int budget = 4096; + + MxAccessEventQueue queue = new(eventCount); + for (int index = 0; index < eventCount; index++) + { + queue.Enqueue(CreateEventWithPayload(index, payloadLengths[index % payloadLengths.Length])); + } + + List> batches = new(); + while (true) + { + WorkerEventDrainResult result = queue.Drain(maxEvents: 0, maxTotalBytes: budget); + if (result.Events.Count == 0) + { + Assert.Equal(0UL, result.OversizedHeadSequence); + break; + } + + batches.Add(result.Events); + Assert.True(batches.Count <= eventCount, "Drain made no progress."); + } + + Assert.Equal(0, queue.Count); + Assert.True( + batches.Count > 5, + $"Expected the byte budget to split the drain, saw {batches.Count} batches."); + + ulong expectedSequence = 0; + for (int batchIndex = 0; batchIndex < batches.Count; batchIndex++) + { + IReadOnlyList batch = batches[batchIndex]; + int charged = 0; + foreach (WorkerEvent drained in batch) + { + charged += DrainCostOf(drained); + Assert.Equal(++expectedSequence, drained.Event.WorkerSequence); + } + + Assert.True( + charged <= budget, + $"Batch {batchIndex} shipped {charged} bytes against a {budget} byte budget."); + + if (batchIndex + 1 < batches.Count) + { + int nextCost = DrainCostOf(batches[batchIndex + 1][0]); + Assert.True( + charged + nextCost > budget, + $"Batch {batchIndex} stopped at {charged} bytes although the next event's {nextCost} still fit the {budget} byte budget."); + } + } + + Assert.Equal((ulong)eventCount, expectedSequence); + } + + /// + /// The other half of the WRK-21 head guarantee: a head refused for its size is drained + /// unchanged by a later call whose budget fits it. The size the queue charges lives with the + /// event across calls, so a refused attempt must neither consume nor alter it — and the + /// refusal itself is justified by the event's real serialized size. + /// + [Fact] + public void Drain_ByteBudget_RefusedHead_IsDrainedUnchangedOnceTheBudgetFitsIt() + { + MxAccessEventQueue queue = new(capacity: 8); + queue.Enqueue(CreateEventWithPayload(itemHandle: 0, payloadLength: 4096)); + queue.Enqueue(CreateEventWithPayload(itemHandle: 1, payloadLength: 8)); + + WorkerEventDrainResult refused = queue.Drain(maxEvents: 0, maxTotalBytes: 1024); + Assert.Empty(refused.Events); + Assert.True(refused.TruncatedBySize); + Assert.Equal(1UL, refused.OversizedHeadSequence); + Assert.Equal(2, queue.Count); + + WorkerEventDrainResult retried = queue.Drain(maxEvents: 0, maxTotalBytes: 64 * 1024); + + Assert.Equal(2, retried.Events.Count); + Assert.Equal(1UL, retried.Events[0].Event.WorkerSequence); + Assert.Equal(2UL, retried.Events[1].Event.WorkerSequence); + Assert.False(retried.TruncatedBySize); + Assert.Equal(0, retried.RemainingCount); + Assert.Equal(0, queue.Count); + + Assert.True( + DrainCostOf(retried.Events[0]) > 1024, + "The head was refused although its real cost fits the 1024-byte budget it was refused under."); + Assert.True( + DrainCostOf(retried.Events[0]) + DrainCostOf(retried.Events[1]) <= 64 * 1024, + "The retried batch exceeded the budget it was drained under."); + } + /// Verifies that Enqueue is rejected after a fault is recorded manually. [Fact] public void Enqueue_AfterRecordFault_ThrowsInvalidOperationException() @@ -515,6 +668,19 @@ public sealed class MxAccessEventQueueTests /// /// Length of the event's raw-status payload string. /// The per-event byte cost. + /// + /// What the byte budget must charge for an already-stamped event: its true serialized size plus + /// the repeated-field allowance. Measured from the generated CalculateSize() so the + /// budget tests bound the queue's memoized size against the real one rather than against a + /// second copy of the queue's own arithmetic. + /// + /// Event as the queue holds it, sequence and timestamp stamped. + /// The per-event byte cost. + private static int DrainCostOf(WorkerEvent workerEvent) + { + return workerEvent.CalculateSize() + RepeatedFieldOverheadBytes; + } + private static int MeasureDrainCost(int payloadLength) { MxAccessEventQueue probe = new(capacity: 1); diff --git a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs index 5bcf9e1..dd07094 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/MxAccess/MxAccessEventQueue.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Globalization; using System.Threading; using System.Threading.Tasks; +using Google.Protobuf; using Google.Protobuf.WellKnownTypes; using ZB.MOM.WW.MxGateway.Contracts.Proto; @@ -21,6 +22,11 @@ namespace ZB.MOM.WW.MxGateway.Worker.MxAccess; /// MxAccessAlarmEventSink, AlarmCommandHandler) build a new event per /// Enqueue via the mapper and satisfy this; the value cache stores its own /// independent snapshot (see ). +/// +/// The byte-budgeted relies on the same invariant: each +/// event's serialized size is measured once at enqueue and stored beside it, which is +/// only sound because an enqueued event is never mutated again. +/// /// public sealed class MxAccessEventQueue { @@ -63,8 +69,16 @@ public sealed class MxAccessEventQueue // edge can never push the packed reply past the frame maximum. private const int RepeatedFieldOverheadBytes = 8; + // Every field the enqueue-time size reconstruction charges by hand — WorkerEvent.event (field + // 1), MxEvent.worker_sequence (9) and MxEvent.worker_timestamp (10) — has a field number below + // 16, so protobuf encodes each of their tags in a single byte. Renumbering any of them above 15 + // would widen its tag and make this estimate one byte light per field, which the + // RepeatedFieldOverheadBytes slack absorbs — so the frame stays safe — but the constant should + // be revisited with the .proto. + private const int SingleByteFieldTagBytes = 1; + private readonly int capacity; - private readonly Queue events; + private readonly Queue events; private readonly object syncRoot = new(); // Wake signal for the worker's event drain loop (see WorkerPipeSession.RunEventDrainLoopAsync). @@ -110,7 +124,7 @@ public sealed class MxAccessEventQueue } this.capacity = capacity; - events = new Queue(capacity); + events = new Queue(capacity); } /// @@ -210,6 +224,28 @@ public sealed class MxAccessEventQueue throw new ArgumentNullException(nameof(mxEvent)); } + // Size the event here, outside syncRoot, and carry the result into the queue. The + // byte-budgeted Drain used to call CalculateSize() on every candidate while holding the + // lock, so a large gateway-pulled drain kept the STA's next Enqueue blocked for the whole + // sizing walk — exactly when a burst of COM callbacks needs the lock. Measuring on the + // enqueuing thread is safe on both counts: that thread owns this event exclusively at this + // point, and WRK-11 (see the class remarks) makes the hand-off final — nothing mutates an + // event after Enqueue, so a memoized size can never go stale. + // + // Enqueue itself stamps the worker sequence and worker timestamp, and those must stay under + // the lock (the sequence has to be assigned in queue order), so they are excluded from this + // measurement and charged back by CalculateWorkerEventSize with O(1) arithmetic instead of a + // second walk of the message. Clearing them first is observationally a no-op — both are + // overwritten unconditionally below — and it makes this measurement exactly "the event minus + // the two stamped fields" even if a caller pre-filled them. + // + // CalculateSize() on a valid protobuf message does not throw, so moving it ahead of the + // fault and overflow checks introduces no new failure before them; an event those checks + // reject simply discards the size computed here. + mxEvent.WorkerSequence = 0UL; + mxEvent.WorkerTimestamp = null; + int unstampedEventSize = mxEvent.CalculateSize(); + try { lock (syncRoot) @@ -237,7 +273,9 @@ public sealed class MxAccessEventQueue { Event = mxEvent, }; - events.Enqueue(workerEvent); + events.Enqueue(new QueuedEvent( + workerEvent, + CalculateWorkerEventSize(mxEvent, unstampedEventSize))); } } finally @@ -295,7 +333,7 @@ public sealed class MxAccessEventQueue return false; } - workerEvent = events.Dequeue(); + workerEvent = events.Dequeue().Event; return true; } } @@ -320,7 +358,7 @@ public sealed class MxAccessEventQueue List drained = new(drainCount); for (int index = 0; index < drainCount; index++) { - drained.Add(events.Dequeue()); + drained.Add(events.Dequeue().Event); } return drained; @@ -334,11 +372,13 @@ public sealed class MxAccessEventQueue /// /// The size decision happens inside the queue lock, so an event is dequeued only once it is /// known to fit: an event that does not fit stays at the head for the next call and is never - /// lost (WRK-21). Per-event cost is WorkerEvent.CalculateSize() — which already - /// includes the event's own tag and length prefix, the same shape the reply's - /// events repeated field packs it into — plus - /// of pure slack, so the running total stays strictly ahead of the true serialized size and - /// the estimate errs on the safe side. + /// lost (WRK-21). Per-event cost is the serialized WorkerEvent size measured once at + /// enqueue time — which already includes the event's own tag and length prefix, the same + /// shape the reply's events repeated field packs it into — plus + /// of pure slack, so the running total stays + /// strictly ahead of the true serialized size and the estimate errs on the safe side. Only + /// the memoized number is read here: this loop no longer walks a message under the lock the + /// STA needs for , so a large drain cannot stall COM callbacks. /// /// Maximum number of events to drain; 0 means "no count limit". /// Byte budget for the drained events' estimated serialized size. @@ -357,8 +397,8 @@ public sealed class MxAccessEventQueue while (drained.Count < countLimit && events.Count > 0) { - WorkerEvent head = events.Peek(); - int cost = head.CalculateSize() + RepeatedFieldOverheadBytes; + QueuedEvent head = events.Peek(); + int cost = head.Size + RepeatedFieldOverheadBytes; if (cost > remainingBudget) { truncatedBySize = true; @@ -367,14 +407,14 @@ public sealed class MxAccessEventQueue // The head alone cannot fit this budget, so repeating the call will not // move it either. Report its sequence instead of silently stalling; the // events that did fit are still returned. - oversizedHeadSequence = head.Event?.WorkerSequence ?? 0; + oversizedHeadSequence = head.Event.Event?.WorkerSequence ?? 0; } break; } remainingBudget -= cost; - drained.Add(events.Dequeue()); + drained.Add(events.Dequeue().Event); } return new WorkerEventDrainResult( @@ -447,6 +487,24 @@ public sealed class MxAccessEventQueue } } + // Serialized size of the WorkerEvent wrapping mxEvent, rebuilt from the size measured before the + // lock plus the exact cost of the two fields Enqueue stamps inside it. It mirrors the generated + // CalculateSize() instead of calling it, because calling it again under the lock is the cost + // this memoization exists to remove: WorkerEvent carries the single length-delimited `event` + // field, MxEvent charges worker_sequence as tag + varint and worker_timestamp as tag + + // length-delimited message, and both are always serialized here — the sequence is + // pre-incremented, so it is never the proto3 default that generated code would skip, and the + // timestamp is never null. Exactness matters: Drain must charge the same bytes it charged when + // it sized events itself, so an unchanged budget still yields unchanged batches. + private static int CalculateWorkerEventSize(MxEvent mxEvent, int unstampedEventSize) + { + int eventSize = unstampedEventSize + + SingleByteFieldTagBytes + CodedOutputStream.ComputeUInt64Size(mxEvent.WorkerSequence) + + SingleByteFieldTagBytes + CodedOutputStream.ComputeMessageSize(mxEvent.WorkerTimestamp); + + return SingleByteFieldTagBytes + CodedOutputStream.ComputeLengthSize(eventSize) + eventSize; + } + private WorkerFault CreateOverflowFault() { string message = $"MXAccess outbound event queue reached capacity {capacity}."; @@ -461,4 +519,24 @@ public sealed class MxAccessEventQueue }, }; } + + // A queued event and the serialized size measured for it at enqueue time. A struct, so carrying + // the size costs the queue one extra field per slot rather than an allocation per event on the + // STA's path. Nothing outside the queue sees it: every drain hands back the WorkerEvent alone, + // exactly as before. + private readonly struct QueuedEvent + { + public QueuedEvent(WorkerEvent workerEvent, int size) + { + Event = workerEvent; + Size = size; + } + + // The queued event, returned unchanged by TryDequeue and both Drain overloads. + public WorkerEvent Event { get; } + + // Serialized size of Event, valid for the queue's whole lifetime because an enqueued event + // is never mutated again (WRK-11). + public int Size { get; } + } } From b2d8dd70ed57dfae047e90fe5970c4cb455c9ae8 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 17:35:05 -0400 Subject: [PATCH 37/41] chore(plans): Phase B implementation complete; as-built note for Task 17 --- .../2026-08-15-perf-review-remediation.md | 2 + ...8-15-perf-review-remediation.md.tasks.json | 211 +++++++++++++++--- 2 files changed, 186 insertions(+), 27 deletions(-) diff --git a/docs/plans/2026-08-15-perf-review-remediation.md b/docs/plans/2026-08-15-perf-review-remediation.md index 71e8f89..269625e 100644 --- a/docs/plans/2026-08-15-perf-review-remediation.md +++ b/docs/plans/2026-08-15-perf-review-remediation.md @@ -438,6 +438,8 @@ Commit: `perf(worker): signal-driven event drain — removes the 25 ms latency f **Implement:** copy the `WriteCompletionWaitMilliseconds` plumbing end to end under a new name (`MXGW_EVENT_QUEUE_CAPACITY` shaped like the existing variable's naming). Absent/invalid env value → default 10000 (never crash the worker on a bad value; log and default). +> **As-built note (1358332):** shipped as silent default without logging, matching the alarm-resolver precedent — no `ILogger` is reachable from the static resolve site without new plumbing; the silent fallback is disclosed in `GatewayConfiguration.md`. The Bootstrap parser files listed above were correctly NOT touched — the established env-var pattern reads `Environment.GetEnvironmentVariable` at the resolve site. + Note the gateway-side files here don't overlap Phase A tasks — safe after Task 15. Commit: `perf(worker): launcher-configurable event queue capacity` diff --git a/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json b/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json index 1a2be11..e8c6d7b 100644 --- a/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json +++ b/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json @@ -1,31 +1,188 @@ { "planPath": "docs/plans/2026-08-15-perf-review-remediation.md", "tasks": [ - {"id": 1, "subject": "Task 1: Named-pipe buffer sizes", "status": "completed"}, - {"id": 2, "subject": "Task 2: Metrics pull-gauge + Interlocked counters", "status": "completed"}, - {"id": 3, "subject": "Task 3: Distributor copy-on-write subscriber snapshot", "status": "completed"}, - {"id": 4, "subject": "Task 4: Dashboard event mirror viewer gating", "status": "completed"}, - {"id": 5, "subject": "Task 5: Snapshot pipeline idle gating + cached config", "status": "completed"}, - {"id": 6, "subject": "Task 6: Reply ownership transfer in MapCommandReply", "status": "completed"}, - {"id": 7, "subject": "Task 7: WorkerClient WaitAsync timeout + WorkerCancel", "status": "completed", "blockedBy": [2]}, - {"id": 8, "subject": "Task 8: Audit pipeline background writer + retention", "status": "completed"}, - {"id": 9, "subject": "Task 9: Parallel session teardown", "status": "completed"}, - {"id": 10, "subject": "Task 10: Dashboard live-data subscription cap", "status": "completed"}, - {"id": 11, "subject": "Task 11: Alarm monitor cached CurrentAlarms", "status": "completed"}, - {"id": 12, "subject": "Task 12: Logging middleware hoist + bearer redaction fix", "status": "completed"}, - {"id": 13, "subject": "Task 13: Auth-path span parsing + limiter keys", "status": "completed"}, - {"id": 14, "subject": "Task 14: Bulk constraint loops, caches, hygiene", "status": "completed"}, - {"id": 15, "subject": "Task 15: Phase A gate — full gateway suite", "status": "in_progress", "blockedBy": [1,2,3,4,5,6,7,8,9,10,11,12,13,14]}, - {"id": 16, "subject": "Task 16: Event drain wake signal", "status": "pending", "blockedBy": [15]}, - {"id": 17, "subject": "Task 17: Event queue capacity env plumbing", "status": "pending", "blockedBy": [16]}, - {"id": 18, "subject": "Task 18: STA message-driven completion waits", "status": "pending", "blockedBy": [15]}, - {"id": 19, "subject": "Task 19: Handle registry reverse index + O(1) removals", "status": "pending", "blockedBy": [15]}, - {"id": 20, "subject": "Task 20: Event conversion TryParseExact + compiled accessors", "status": "pending", "blockedBy": [15]}, - {"id": 21, "subject": "Task 21: Drain size memoized at enqueue", "status": "pending", "blockedBy": [16, 17]}, - {"id": 22, "subject": "Task 22: Worker frame writer/reader pooled buffers", "status": "pending", "blockedBy": [15]}, - {"id": 23, "subject": "Task 23: Alarm consumer parse + truncation + cadence", "status": "pending", "blockedBy": [15]}, - {"id": 24, "subject": "Task 24: Phase B verification on windev (psbridge)", "status": "pending", "blockedBy": [16,17,18,19,20,21,22,23]}, - {"id": 25, "subject": "Task 25: Wrap-up docs sweep + follow-ups", "status": "pending", "blockedBy": [24]} + { + "id": 1, + "subject": "Task 1: Named-pipe buffer sizes", + "status": "completed" + }, + { + "id": 2, + "subject": "Task 2: Metrics pull-gauge + Interlocked counters", + "status": "completed" + }, + { + "id": 3, + "subject": "Task 3: Distributor copy-on-write subscriber snapshot", + "status": "completed" + }, + { + "id": 4, + "subject": "Task 4: Dashboard event mirror viewer gating", + "status": "completed" + }, + { + "id": 5, + "subject": "Task 5: Snapshot pipeline idle gating + cached config", + "status": "completed" + }, + { + "id": 6, + "subject": "Task 6: Reply ownership transfer in MapCommandReply", + "status": "completed" + }, + { + "id": 7, + "subject": "Task 7: WorkerClient WaitAsync timeout + WorkerCancel", + "status": "completed", + "blockedBy": [ + 2 + ] + }, + { + "id": 8, + "subject": "Task 8: Audit pipeline background writer + retention", + "status": "completed" + }, + { + "id": 9, + "subject": "Task 9: Parallel session teardown", + "status": "completed" + }, + { + "id": 10, + "subject": "Task 10: Dashboard live-data subscription cap", + "status": "completed" + }, + { + "id": 11, + "subject": "Task 11: Alarm monitor cached CurrentAlarms", + "status": "completed" + }, + { + "id": 12, + "subject": "Task 12: Logging middleware hoist + bearer redaction fix", + "status": "completed" + }, + { + "id": 13, + "subject": "Task 13: Auth-path span parsing + limiter keys", + "status": "completed" + }, + { + "id": 14, + "subject": "Task 14: Bulk constraint loops, caches, hygiene", + "status": "completed" + }, + { + "id": 15, + "subject": "Task 15: Phase A gate \u2014 full gateway suite", + "status": "completed", + "blockedBy": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14 + ] + }, + { + "id": 16, + "subject": "Task 16: Event drain wake signal", + "status": "completed", + "blockedBy": [ + 15 + ] + }, + { + "id": 17, + "subject": "Task 17: Event queue capacity env plumbing", + "status": "completed", + "blockedBy": [ + 16 + ] + }, + { + "id": 18, + "subject": "Task 18: STA message-driven completion waits", + "status": "completed", + "blockedBy": [ + 15 + ] + }, + { + "id": 19, + "subject": "Task 19: Handle registry reverse index + O(1) removals", + "status": "completed", + "blockedBy": [ + 15 + ] + }, + { + "id": 20, + "subject": "Task 20: Event conversion TryParseExact + compiled accessors", + "status": "completed", + "blockedBy": [ + 15 + ] + }, + { + "id": 21, + "subject": "Task 21: Drain size memoized at enqueue", + "status": "completed", + "blockedBy": [ + 16, + 17 + ] + }, + { + "id": 22, + "subject": "Task 22: Worker frame writer/reader pooled buffers", + "status": "completed", + "blockedBy": [ + 15 + ] + }, + { + "id": 23, + "subject": "Task 23: Alarm consumer parse + truncation + cadence", + "status": "completed", + "blockedBy": [ + 15 + ] + }, + { + "id": 24, + "subject": "Task 24: Phase B verification on windev (psbridge)", + "status": "in_progress", + "blockedBy": [ + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23 + ] + }, + { + "id": 25, + "subject": "Task 25: Wrap-up docs sweep + follow-ups", + "status": "pending", + "blockedBy": [ + 24 + ] + } ], - "lastUpdated": "2026-08-15T13:30:00Z" -} + "lastUpdated": "2026-08-15T18:05:00Z" +} \ No newline at end of file From 7755745f2f898b2a3027d7b0b9c6816efaed1553 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 17:44:27 -0400 Subject: [PATCH 38/41] =?UTF-8?q?chore(plans):=20Task=2024=20windev=20gate?= =?UTF-8?q?=20green=20=E2=80=94=20slnx+x86=20builds=20clean,=20worker=2049?= =?UTF-8?q?9/499,=20gateway=201039/1040=20(1=20pre-existing=20main=20failu?= =?UTF-8?q?re,=201=20isolated-pass=20load=20flake)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plans/2026-08-15-perf-review-remediation.md.tasks.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json b/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json index e8c6d7b..33b93bf 100644 --- a/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json +++ b/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json @@ -163,7 +163,7 @@ { "id": 24, "subject": "Task 24: Phase B verification on windev (psbridge)", - "status": "in_progress", + "status": "completed", "blockedBy": [ 16, 17, @@ -178,11 +178,11 @@ { "id": 25, "subject": "Task 25: Wrap-up docs sweep + follow-ups", - "status": "pending", + "status": "in_progress", "blockedBy": [ 24 ] } ], - "lastUpdated": "2026-08-15T18:05:00Z" + "lastUpdated": "2026-08-15T18:40:00Z" } \ No newline at end of file From dc2df628e3678b0bbe53dbf007f22c784c473ed2 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 17:54:31 -0400 Subject: [PATCH 39/41] chore(followups): reviewer-recommended tests, comments, and hardening from the remediation reviews The remediation reviews approved every task but left a tail of small notes. This lands the gateway-side half of them. Hardening (behavior changes, all narrow): - BuildFilteredWriteBulkCommand's unreachable default case failed OPEN: a fifth bulk-write kind added upstream without a filter case here would have shipped the DENIED entries to the worker while reporting them denied to the caller. It now throws UnreachableException. - SqliteCanonicalAuditStore.ListRecentAsync no longer throws on a row it cannot date. The retention sweep deliberately preserves such rows (SQLite's datetime() yields NULL, so the DELETE never matches), which guaranteed the dashboard's recent-audit view would meet one eventually and lose the whole page to it. The row is now reported at DateTimeOffset.MinValue with every other column intact, behind an optional logger. - The audit drain loop's finally now completes the channel writer alongside detaching the drain, so a producer that raced past the attached check takes the write-through branch instead of stranding its event in a buffer nobody reads until shutdown. TryComplete is idempotent, so StopAsync is unaffected. Tests: - MapCommandReply ownership (Assert.Same on the inner reply), mirroring the existing MapEvent ownership test. - Redactor key-id length boundary at exactly 64 and 65 characters, pinning which way it fails. Nothing validates key-id length at creation, so docs/Diagnostics.md's "which no issued key id does" is now stated as the heuristic it is. - ApiKeyFailureLimiter.Reset with a PartitionResolution whose partition was evicted between the Check and the Reset: inert, and clears nobody else's block. - Constraint-cache concurrency stress: the cap is enforced by the inserting thread, so overshoot must be transient and proportional to the in-flight inserters, and the cache must settle at or under the cap. - ListRecentAsync against a raw-SQL undateable row. Comment/doc accuracy: - EventsHubViewerRegistry.ReleaseConnection records that it relies on SignalR's default sequential per-connection dispatch (MaximumParallelInvocationsPerClient = 1). - A PERF(followup) note on Invoke's double session resolve and why removing it needs a SessionManager overload. - SessionEventDistributor: the volatile-field comment named the pump as the lock-free reader, but the pump's single capture point is inside _replayLock; the genuinely lock-free reader is SubscriberCount. OnSubscriberOverflow's "cannot be observed here" now excepts the DisposeAsync abandon path. The churn test names its ConcurrentDictionary bucket-order assumption and that a violation surfaces as a read timeout, not a silent pass. - The two "restores the sequential drain's behavior" claims (SessionManager, docs/Sessions.md) were wrong: the sequential drain leaked too, because KillWorkerAsync's entry ThrowIfCancellationRequested aborted the whole loop on the first session for zero kills. Reworded to "fixes a leak the sequential drain also had", with the sweep-bound/shutdown-unbound ParallelOptions asymmetry explained. - ISessionManager.ShutdownAsync's token doc: it degrades the drain to a kill sweep rather than cancelling it, with the bounded overrun stated. SessionShutdownHostedService.StopAsync records that its cancellation-logging branch is now unreachable. --- docs/Diagnostics.md | 8 ++- docs/Sessions.md | 4 +- .../2026-08-15-perf-review-remediation.md | 1 + .../Dashboard/Hubs/EventsHubViewerRegistry.cs | 5 ++ .../Grpc/MxAccessGatewayService.cs | 16 ++++-- .../Security/Audit/AuditDrainService.cs | 7 +++ .../Security/Audit/ChannelAuditWriter.cs | 7 ++- .../Audit/SqliteCanonicalAuditStore.cs | 35 +++++++++++-- .../AuthStoreServiceCollectionExtensions.cs | 7 ++- .../Sessions/ISessionManager.cs | 9 +++- .../Sessions/SessionEventDistributor.cs | 14 ++++-- .../Sessions/SessionManager.cs | 18 +++++-- .../Sessions/SessionShutdownHostedService.cs | 9 +++- .../Diagnostics/GatewayLogRedactorTests.cs | 27 ++++++++++ .../Gateway/Grpc/MxAccessGrpcMapperTests.cs | 24 +++++++++ .../Sessions/SessionEventDistributorTests.cs | 8 +++ .../Security/Audit/ChannelAuditWriterTests.cs | 30 +++++++++++- .../CachingApiKeyVerifierTests.cs | 49 +++++++++++++++++++ .../ApiKeyFailureLimiterTests.cs | 40 +++++++++++++++ 19 files changed, 293 insertions(+), 25 deletions(-) diff --git a/docs/Diagnostics.md b/docs/Diagnostics.md index bec160c..0e886b0 100644 --- a/docs/Diagnostics.md +++ b/docs/Diagnostics.md @@ -101,8 +101,12 @@ correlate a log entry to a specific principal. A scheme word survives only when it is one of the recognized authorization schemes (`Bearer`, `Basic`, `Digest`, `Negotiate`, `NTLM`, `ApiKey`, `Token`). An unrecognized leading word is as likely to be credential material as it is to be a scheme, so it is dropped along with the rest. The key id is -also dropped when it runs longer than 64 characters, which no issued key id does — a long run before -the first `_` is secret material, not an identifier. +also dropped when it runs longer than 64 characters — a long run before the first `_` is more likely to +be secret material than an identifier. Neither key-creation path (`ApiKeyAdminCommandLineParser.IsValidKeyId`, +`DashboardApiKeyManagementService.ValidateKeyId`) enforces a length, so this is a redaction heuristic +rather than a guarantee: operators should keep key ids under 64 characters, or the id stops appearing +in logs and only the `mxgw_[redacted]` shape survives. The direction of the failure is deliberate — +losing an identifier is cheap, logging a secret is not. The parse is span-based (no regex, no `Split` allocation): the value is split once at the first space, and the key id is read up to the first `_` of the remainder. diff --git a/docs/Sessions.md b/docs/Sessions.md index 418a368..072110b 100644 --- a/docs/Sessions.md +++ b/docs/Sessions.md @@ -215,7 +215,9 @@ Splitting the phases moves the TOCTOU re-check earlier, and that is an accepted A close that throws no longer abandons the rest of the selected set: the sweep attempts every selected session, captures the first failure, and rethrows it once the pass is done so `SessionLeaseMonitorHostedService` still logs the sweep failure as before. Because the closes run concurrently, *which* failure surfaces when several fail in one pass is nondeterministic; the log line is the diagnostic, not the identity of the exception. -`ShutdownAsync` drains sessions with the same bounded fan-out and the same per-session catch → `KillWorkerAsync` fallback, with two rules that keep a stop deadline from turning into leaked workers. First, its body is **exception-total** — nothing escapes it — because `Parallel.ForEachAsync` cancels the token handed to the sibling bodies as soon as one body throws, which would abort in-flight graceful shutdowns *and* make their kill fallback fail instantly on the freshly cancelled token. Second, the drain loop is deliberately **not** bound to the caller's `CancellationToken` and the kill fallback runs on `CancellationToken.None`: a cancelled `ParallelOptions` token stops dispatching the remaining sessions entirely, whereas the sequential drain this replaced let every remaining session fail its graceful close fast and still kill its worker. The token is passed to the graceful close instead, so a host stop deadline turns the drain into a kill sweep rather than into a leak. This matters because nothing reattaches to a leaked worker — a restarted gateway terminates orphans (see [Design Decisions](DesignDecisions.md)). +`ShutdownAsync` drains sessions with the same bounded fan-out and the same per-session catch → `KillWorkerAsync` fallback, with two rules that keep a stop deadline from turning into leaked workers. First, its body is **exception-total** — nothing escapes it — because `Parallel.ForEachAsync` cancels the token handed to the sibling bodies as soon as one body throws, which would abort in-flight graceful shutdowns *and* make their kill fallback fail instantly on the freshly cancelled token. Second, the drain loop is deliberately **not** bound to the caller's `CancellationToken` and the kill fallback runs on `CancellationToken.None`: a cancelled `ParallelOptions` token stops dispatching the remaining sessions entirely, so the untried tail would be neither closed nor killed. This **fixes a leak the sequential drain also had**, rather than restoring the sequential drain's behavior — there the kill fallback ran on the caller's already-cancelled token, and `KillWorkerAsync`'s entry `ThrowIfCancellationRequested` threw out of the loop on the very first session, producing zero kills. The token is passed to the graceful close only, so a host stop deadline turns the drain into a kill sweep rather than into a leak. This matters because nothing reattaches to a leaked worker — a restarted gateway terminates orphans (see [Design Decisions](DesignDecisions.md)). + +The asymmetry with `CloseExpiredLeasesAsync` — whose `ParallelOptions` *is* token-bound — is intentional: the sweep is periodic maintenance, so a pass abandoned on cancellation loses nothing permanently (the next pass re-selects, and `ShutdownAsync` backstops it), whereas the shutdown drain is terminal and must not be abandoned partway. **Stranded-`Closing` bound.** A sweep pass that is cancelled after selection leaves its unclosed selections in `Closing` with close already started. `IsFaultedReapableCore` requires `state == Faulted`, so a session selected under `FaultedReason` and stranded this way is not re-selected as faulted; it is swept only when its normal lease expires (up to `MxGateway:Sessions:DefaultLeaseSeconds`, default 1800 s), since `IsLeaseExpiredCore` and `IsDetachGraceExpiredCore` are state-agnostic. This bound is documented rather than closed with a re-selection clause: the sweep's only caller cancels on the host's `stoppingToken`, so the very next thing that runs is `ShutdownAsync`, which drains (or kills) the whole registry — and any worker that still survives that is terminated as an orphan on the next gateway start. Adding a "`Closing` and close-started" re-selection clause would also have to distinguish an abandoned close from one that is merely still in flight, which would weaken the single invariant that makes the parallel close phase safe. diff --git a/docs/plans/2026-08-15-perf-review-remediation.md b/docs/plans/2026-08-15-perf-review-remediation.md index 269625e..5b20b1c 100644 --- a/docs/plans/2026-08-15-perf-review-remediation.md +++ b/docs/plans/2026-08-15-perf-review-remediation.md @@ -617,6 +617,7 @@ Commit anything found: `docs: remediation plan doc sweep` | Control-frame completion coupled to event batch drain | Documented, bounded (≤128 frames) behavior of the two-class writer design; revisit only if heartbeat latency shows up in metrics. | | Blazor pages' loopback SignalR hop | Works correctly; in-process `WatchSnapshotsAsync` consumption is a dashboard refactor with payoff only at viewer counts the product doesn't target. | | Event-path triple async-iterator flattening | LOW-rated; touches the most invariant-dense code in the gateway for two `MoveNextAsync` hops per event. Reconsider after Tasks 3/4 land and if profiling still shows it. | +| `SessionEventDistributor._subscribers` `ConcurrentDictionary` → plain `Dictionary` (Task 25 / Task 3 review) | Every mutation is already inside `_lifecycleLock`, so the concurrent type buys nothing. Behavior-neutral refactor with no measurable win, proposed after the windev gate was already green — not worth re-running the verification matrix for. Comment cleanups from the same review landed; this swap did not. | ## Execution notes for the orchestrator diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHubViewerRegistry.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHubViewerRegistry.cs index 0607dc3..ac66b00 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHubViewerRegistry.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Hubs/EventsHubViewerRegistry.cs @@ -91,6 +91,11 @@ public sealed class EventsHubViewerRegistry return; } + // Detaching the set is safe against a SubscribeSession that arrives after the disconnect + // only because SignalR dispatches a connection's hub invocations sequentially by default + // (MaximumParallelInvocationsPerClient = 1): OnDisconnectedAsync cannot overlap an + // AddViewer for the same connection, so no late add can re-create the entry and leak a + // count that nothing will ever release. Raising that option would break this. if (!_sessionsByConnection.TryRemove(connectionId, out ConcurrentDictionary? sessions)) { return; diff --git a/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs b/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs index c4380b2..fa77d2c 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs @@ -101,6 +101,12 @@ public sealed class MxAccessGatewayService( try { requestValidator.ValidateInvoke(request); + + // PERF(followup): this resolve and the sessionManager.InvokeAsync below look the same + // session up twice (a dictionary hit each, so measured cost is negligible). Collapsing + // them needs a SessionManager overload taking an already-resolved GatewaySession, which + // would duplicate InvokeAsync's fault mapping (SessionNotFound / state checks / metrics) + // at a second entry point — deliberately not worth it until a profile says otherwise. GatewaySession session = ResolveSession(request.SessionId); MxCommand command = request.Command; BulkConstraintPlan? bulkConstraintPlan = await ApplyConstraintsAsync( @@ -697,9 +703,13 @@ public sealed class MxAccessGatewayService( default: // Only the four bulk-write kinds above reach FilterWriteBulkAsync, so this is - // unreachable; keep the previous behaviour (the unmodified command) rather than - // emitting a payload-less one if that ever stops holding. - return command.Clone(); + // unreachable. It throws rather than falling back to the unmodified command, + // because that fallback failed OPEN: a fifth bulk-write kind added upstream + // without a case here would silently ship the DENIED entries to the worker while + // still reporting them denied to the caller. Failing loud on a kind nobody can + // reach today is strictly safer than a constraint bypass nobody would notice. + throw new UnreachableException( + $"Command kind {command.Kind} reached bulk-write constraint filtering without a filter case."); } return filtered; diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/AuditDrainService.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/AuditDrainService.cs index d4519fb..0d9570f 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/AuditDrainService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/AuditDrainService.cs @@ -246,6 +246,13 @@ public sealed class AuditDrainService( finally { writer.DetachDrain(); + + // Detaching alone leaves a racer that already passed the attached check enqueueing into + // a channel this loop will never read again — those events would sit in the buffer until + // StopAsync's final drain. Completing the writer as well makes that racer's TryWrite + // return false, which is the write-through branch, so the event reaches the store now. + // TryComplete is idempotent, so StopAsync's own CompleteWriting stays safe either way. + writer.CompleteWriting(); } } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/ChannelAuditWriter.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/ChannelAuditWriter.cs index d85c5a2..68be6ae 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/ChannelAuditWriter.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/ChannelAuditWriter.cs @@ -87,8 +87,11 @@ public sealed class ChannelAuditWriter : IAuditWriter public void DetachDrain() => Volatile.Write(ref _drainAttached, 0); /// - /// Enqueues a canonical audit event for the drain to persist. Never blocks, never throws, - /// and never touches the store on the caller's thread while a drain is attached. + /// Enqueues a canonical audit event for the drain to persist. Never blocks and never throws. + /// It also never touches the store on the caller's thread while a drain is attached — with one + /// exception: once the channel has been completed (shutdown, or a drain loop that died), the + /// enqueue fails and this falls through to the synchronous write, which is what keeps the event + /// rather than stranding it in a buffer nobody reads. /// /// The canonical audit event to persist. /// Token honoured only by the direct write-through path. diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/SqliteCanonicalAuditStore.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/SqliteCanonicalAuditStore.cs index 3ad7f91..67f4bb8 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/SqliteCanonicalAuditStore.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Audit/SqliteCanonicalAuditStore.cs @@ -31,7 +31,14 @@ namespace ZB.MOM.WW.MxGateway.Server.Security.Audit; /// retries. /// /// -public sealed class SqliteCanonicalAuditStore(AuthSqliteConnectionFactory connectionFactory) : IAuditEventSink +/// Factory for connections to the shared auth database file. +/// +/// Optional logger for row-level read diagnostics. Optional because the store is also constructed +/// directly by the apikey CLI path and by DI-free tests, which have no logger to hand. +/// +public sealed class SqliteCanonicalAuditStore( + AuthSqliteConnectionFactory connectionFactory, + ILogger? logger = null) : IAuditEventSink { private const string CreateTableSql = """ @@ -208,7 +215,7 @@ public sealed class SqliteCanonicalAuditStore(AuthSqliteConnectionFactory connec events.Add(new AuditEvent { EventId = Guid.Parse(reader.GetString(0)), - OccurredAtUtc = ParseUtc(reader.GetString(1)), + OccurredAtUtc = ParseUtcOrMinValue(reader.GetString(1), reader.GetString(0)), Actor = reader.GetString(2), Action = reader.GetString(3), Outcome = Enum.Parse(reader.GetString(4)), @@ -239,6 +246,26 @@ public sealed class SqliteCanonicalAuditStore(AuthSqliteConnectionFactory connec Volatile.Write(ref _tableEnsured, 1); } - private static DateTimeOffset ParseUtc(string value) => - DateTimeOffset.Parse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + // Reading is defensive where writing is not: an insert always round-trips "O", but the table is + // append-only shared state that an operator (or a future migration) can put an unparseable + // timestamp into, and the retention sweep deliberately keeps such a row — SQLite's datetime() + // yields NULL for it, so the DELETE's comparison is never true. A throwing Parse here would let + // that single row take out the dashboard's whole recent-audit view. MinValue instead sorts the + // row to the far past and keeps every other column readable, which is what an operator looking + // at the view actually needs. + private DateTimeOffset ParseUtcOrMinValue(string value, string eventId) + { + if (DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out DateTimeOffset parsed)) + { + return parsed; + } + + // Debug, not warning: the row is still returned and the timestamp text itself is not logged + // (audit rows are not secrets, but the value is attacker-influenceable in the worst case). + logger?.LogDebug( + "Audit event {EventId} has an unparseable occurred_at_utc; reporting it as DateTimeOffset.MinValue.", + eventId); + + return DateTimeOffset.MinValue; + } } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/AuthStoreServiceCollectionExtensions.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/AuthStoreServiceCollectionExtensions.cs index 6722fa6..e799758 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/AuthStoreServiceCollectionExtensions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/AuthStoreServiceCollectionExtensions.cs @@ -97,8 +97,13 @@ public static class AuthStoreServiceCollectionExtensions sp.GetService() ?? TimeProvider.System)); DecorateVerifierWithCache(services, security); + // GetService, not GetRequiredService, for the same reason the writer registration below + // gives: the DI-only unit tests build a bare ServiceCollection with no AddLogging(). The + // store's logger is optional and only carries row-level read diagnostics. services.AddSingleton(sp => - new SqliteCanonicalAuditStore(sp.GetRequiredService())); + new SqliteCanonicalAuditStore( + sp.GetRequiredService(), + sp.GetService>())); services.AddSingleton(sp => sp.GetRequiredService()); // Resolve the logger defensively: the production host always registers ILogger, but the // DI-only auth/CLI/dashboard unit tests build a bare ServiceCollection without AddLogging(). diff --git a/src/ZB.MOM.WW.MxGateway.Server/Sessions/ISessionManager.cs b/src/ZB.MOM.WW.MxGateway.Server/Sessions/ISessionManager.cs index 232a17c..7a9b749 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Sessions/ISessionManager.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Sessions/ISessionManager.cs @@ -79,7 +79,14 @@ public interface ISessionManager CancellationToken cancellationToken); /// Shuts down all sessions and the session manager. - /// Token to cancel the asynchronous operation. + /// + /// Token that degrades the drain rather than cancelling it. It is passed only to each + /// session's graceful close; the drain loop and the kill fallback are not bound to it, so + /// cancelling turns the drain into a kill sweep instead of abandoning the untried sessions as + /// leaked workers. The call therefore overruns a cancelled token by a bounded amount — + /// roughly ceil(sessionCount / 4) batches of the worker shutdown timeout in the worst + /// case, where 4 is MaxParallelSessionCloses. + /// /// A task that represents the asynchronous operation. Task ShutdownAsync(CancellationToken cancellationToken); } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs index ba97a03..a7b653a 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionEventDistributor.cs @@ -112,8 +112,11 @@ public sealed class SessionEventDistributor : IAsyncDisposable // inside the _lifecycleLock section of every register/unregister; never mutated in // place, so the pump can walk the array it captured with no lock and no allocation. // Volatile.Write / Volatile.Read ORDER the access — they keep the publishing store from - // sinking past the lock release and keep the pump's read from being hoisted out of the - // fan-out loop. They do NOT promise freshness, and nothing here needs them to: a reader + // sinking past the lock release, and they keep a lock-free reader's load from being hoisted + // or cached. The pump is NOT that reader: its single capture point sits inside the + // _replayLock section of AppendToReplayBufferAndCaptureSubscribers, so the lock edge already + // orders it. The genuinely lock-free reader is SubscriberCount, which loads the field with no + // lock at all. They do NOT promise freshness, and nothing here needs them to: a reader // may legitimately observe the previous array, which IS the documented "late subscribers // see events after they register" window. Where visibility must be guaranteed — the // RegisterWithReplay handoff — it comes from the _replayLock edge, not from Volatile. @@ -700,8 +703,11 @@ public sealed class SessionEventDistributor : IAsyncDisposable // every path that completes a channel during fan-out (lease disposal via Unregister, // and this method) removes the subscriber from the set BEFORE completing it, so a // completed channel implies the subscriber is already gone and RemoveSubscriber - // returns false. (CompleteAllSubscribers completes without removing, but only after - // the pump has left its loop, so it cannot be observed here.) + // returns false. (CompleteAllSubscribers completes without removing, but only after the + // pump has left its loop, so it cannot be observed here — except on the DisposeAsync + // abandon path: a source factory that ignores cancellation past the 5 s shutdown timeout + // leaves the pump fanning while DisposeAsync completes subscribers, so a spurious overflow + // report is possible there. It is harmless, because the session is already being disposed.) // // Bailing out on false is what keeps a normal stream ending mid-traffic from emitting // a bogus EventQueueOverflow metric and — under the default single-subscriber FailFast diff --git a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs index 9042aad..e369141 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionManager.cs @@ -365,11 +365,19 @@ public sealed class SessionManager : ISessionManager // rather than adopting them). // // For the same reason the loop itself is NOT bound to cancellationToken: a cancelled - // ParallelOptions token stops dispatching the remaining sessions entirely, whereas the - // sequential drain this replaced let every remaining session fail its graceful close fast - // and still kill its worker. The token is passed to the graceful close instead, which - // preserves that behavior — a host stop deadline turns the drain into a kill sweep rather - // than into a leak. + // ParallelOptions token stops dispatching the remaining sessions entirely, so a stop + // deadline would leave the untried tail neither closed nor killed. Note this FIXES a leak + // the sequential drain also had rather than restoring its behavior: there the kill fallback + // ran on the caller's cancelled token, and KillWorkerAsync's entry + // ThrowIfCancellationRequested threw out of the loop on the first session — zero kills, not + // "fail fast and still kill". The token is passed to the graceful close only, and the kill + // runs on CancellationToken.None, so a host stop deadline turns the drain into a kill sweep + // rather than into a leak. + // + // The asymmetry with CloseExpiredLeasesAsync (whose ParallelOptions IS token-bound) is + // intentional: that sweep is periodic maintenance whose missed sessions are picked up by + // the next pass and, ultimately, by this drain. This drain is terminal — nothing runs after + // it — so it must not be abandoned partway. await Parallel.ForEachAsync( _registry.Snapshot(), new ParallelOptions { MaxDegreeOfParallelism = MaxParallelSessionCloses }, diff --git a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionShutdownHostedService.cs b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionShutdownHostedService.cs index d126dfc..d02fe36 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionShutdownHostedService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Sessions/SessionShutdownHostedService.cs @@ -16,7 +16,14 @@ public sealed class SessionShutdownHostedService( return Task.CompletedTask; } - /// Shuts down all gateway sessions as the host stops, logging (without throwing) if the host's shutdown timeout cancels the operation first. + /// Shuts down all gateway sessions as the host stops. + /// + /// The catch below is now effectively unreachable: + /// no longer aborts on the host's shutdown timeout, it degrades to a kill sweep and logs a + /// per-session warning for each session that failed its graceful close. The clause is kept as + /// a cheap guard against that contract regressing, not as an expected path — the operator + /// signal for a timed-out shutdown is now those per-session warnings. + /// /// Token that signals the host's shutdown timeout has elapsed. /// A task that represents the asynchronous operation. public async Task StopAsync(CancellationToken cancellationToken) diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/GatewayLogRedactorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/GatewayLogRedactorTests.cs index 70df716..41df638 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/GatewayLogRedactorTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/GatewayLogRedactorTests.cs @@ -50,6 +50,33 @@ public sealed class GatewayLogRedactorTests Assert.DoesNotContain("super-secret", redacted, StringComparison.Ordinal); } + /// + /// Pins the 64-character key-id boundary in both directions. Nothing validates key-id length at + /// creation time (neither ApiKeyAdminCommandLineParser.IsValidKeyId nor + /// DashboardApiKeyManagementService.ValidateKeyId caps it), so the cap here is a redaction + /// heuristic that a real key id can cross. This test fixes which way it fails when it does: at + /// exactly 64 the id is still an identifier and survives; at 65 the whole run is treated as secret + /// material and goes. Losing an identifier is the cheap failure; logging a secret is not. + /// + /// Length of the key id presented before the secret separator. + /// Whether the key id must survive redaction at that length. + [Theory] + [InlineData(64, true)] + [InlineData(65, false)] + public void RedactClientIdentity_KeyIdLengthBoundary_FailsTowardRedaction( + int keyIdLength, + bool expectsKeyIdPreserved) + { + string keyId = new('a', keyIdLength); + + string? redacted = GatewayLogRedactor.RedactClientIdentity($"Bearer mxgw_{keyId}_super-secret"); + + Assert.Equal( + expectsKeyIdPreserved ? $"Bearer mxgw_{keyId}_[redacted]" : "Bearer mxgw_[redacted]", + redacted); + Assert.DoesNotContain("super-secret", redacted, StringComparison.Ordinal); + } + /// /// Verifies that anything not recognized as a gateway API key fails closed: the scheme word /// survives only when it looks like an auth scheme, and the credential never does. diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/MxAccessGrpcMapperTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/MxAccessGrpcMapperTests.cs index 4bda8ae..ebf4957 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/MxAccessGrpcMapperTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/MxAccessGrpcMapperTests.cs @@ -120,6 +120,30 @@ public sealed class MxAccessGrpcMapperTests Assert.Equal(ProtocolStatusCode.ProtocolViolation, publicReply.ProtocolStatus.Code); } + /// + /// Verifies MapCommandReply transfers ownership of the inner MxCommandReply the same way + /// MapEvent does: the returned reference is the instance carried by the WorkerCommandReply, + /// not a clone. The WorkerCommandReply is discarded after mapping and the awaiting Invoke + /// call is its single consumer, so moving the inner reply out is safe and avoids a deep copy + /// of a potentially large bulk-read payload. + /// + [Fact] + public void MapCommandReply_TransfersOwnershipOfInnerReplyWithoutCloning() + { + MxCommandReply innerReply = new() + { + SessionId = "session-1", + Kind = MxCommandKind.Register, + ProtocolStatus = MxAccessGrpcMapper.Ok(), + Register = new RegisterReply { ServerHandle = 50 }, + }; + WorkerCommandReply workerReply = new() { Reply = innerReply }; + + MxCommandReply mapped = new MxAccessGrpcMapper().MapCommandReply(workerReply); + + Assert.Same(innerReply, mapped); + } + /// /// Verifies MapEvent transfers ownership of the inner MxEvent (GWC-07 / IPC-05): the /// returned reference is the same instance carried by the WorkerEvent, not a clone. The diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs index 1338a71..5220ac3 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Sessions/SessionEventDistributorTests.cs @@ -1066,6 +1066,14 @@ public sealed class SessionEventDistributorTests /// racing the fan-out must never drop, duplicate, or reorder an event for a subscriber /// registered throughout — nor leave the array and the dictionary disagreeing on the /// subscriber count once the churn stops. + /// + /// One assumption worth naming: the snapshot is rebuilt from + /// ConcurrentDictionary.Values, whose bucket order happens to keep the long-lived + /// stable subscriber ahead of the churned ones here, so it is written to before a churned + /// subscriber's disposal can interleave. If that ever stops holding the test does not + /// silently pass — it fails as a expiry on the read below, because + /// an event dropped for the stable subscriber never arrives. + /// /// /// A task that represents the asynchronous operation. [Fact] diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Security/Audit/ChannelAuditWriterTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Security/Audit/ChannelAuditWriterTests.cs index 3b5a273..9f40f2a 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Security/Audit/ChannelAuditWriterTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Security/Audit/ChannelAuditWriterTests.cs @@ -219,6 +219,33 @@ public sealed class ChannelAuditWriterTests : IDisposable (await ListActionsAsync(factory)).OrderBy(action => action, StringComparer.Ordinal)); } + /// + /// The sweep above deliberately preserves rows it cannot date, so the dashboard's recent-audit + /// view is guaranteed to meet one eventually. Reading must therefore be defensive: the + /// undateable row is reported at with every other column + /// intact, rather than one bad row throwing the whole page away. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task SqliteStore_ListRecent_WithUnparseableRow_ReportsMinValueInsteadOfThrowing() + { + (SqliteCanonicalAuditStore store, AuthSqliteConnectionFactory factory) = CreateStore(); + + DateTimeOffset occurred = new(2026, 5, 18, 0, 0, 0, TimeSpan.Zero); + await store.InsertAsync(MakeEvent("dateable", occurred), CancellationToken.None); + await InsertRawRowAsync(factory, "undateable", "0000-not-a-timestamp"); + + IReadOnlyList recent = await store.ListRecentAsync(10, CancellationToken.None); + + Assert.Equal(2, recent.Count); + Assert.Equal(occurred, recent.Single(auditEvent => auditEvent.Action == "dateable").OccurredAtUtc); + + AuditEvent undateable = recent.Single(auditEvent => auditEvent.Action == "undateable"); + Assert.Equal(DateTimeOffset.MinValue, undateable.OccurredAtUtc); + Assert.Equal("operator01", undateable.Actor); + Assert.Equal(AuditOutcome.Denied, undateable.Outcome); + } + private static (ChannelAuditWriter Writer, AuditDrainService Drain) CreateWriterAndDrain( IAuditEventSink sink, SecurityOptions? security = null, @@ -265,7 +292,8 @@ public sealed class ChannelAuditWriterTests : IDisposable await command.ExecuteNonQueryAsync(CancellationToken.None); } - // Reads actions straight from SQL: ListRecentAsync would throw on the unparseable timestamp. + // Reads actions straight from SQL so the sweep assertion depends on the DELETE alone, with no + // opinion from the store's read path about how an undateable row is surfaced. private static async Task> ListActionsAsync(AuthSqliteConnectionFactory factory) { await using SqliteConnection connection = await factory.OpenConnectionAsync(CancellationToken.None); diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs index a75046c..49dce31 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs @@ -281,6 +281,55 @@ public sealed class CachingApiKeyVerifierTests Assert.Same(reparsed, MapConstraints(firstJson)); } + /// + /// Stress: the constraint cache's bound is enforced by the inserting thread itself + /// (GetOrAdd, then enqueue, then evict), so concurrent inserters can each land an + /// entry before any of them reaches the eviction step. That overshoot is real but must be + /// transient and proportional to the in-flight inserters — not unbounded growth — and once + /// the churn stops the cache must be back at or under the cap. Hammered on both the + /// converging path (every iteration maps the same blob, so all but one GetOrAdd + /// loses) and the growth path (a distinct blob per iteration, which is what forces + /// eviction). + /// + [Fact] + public void ToGatewayIdentity_ConcurrentBlobs_OvershootIsTransientAndCacheSettlesUnderCap() + { + const int cap = GatewayApiKeyIdentityMapper.MaxCachedConstraintBlobs; + + // Overshoot is bounded by how many inserters can sit between their GetOrAdd and their own + // EvictIfOverCapacity, so scale the allowance with the available parallelism rather than + // pinning a magic number. Generous on purpose: the assertion under test is "bounded", not + // "bounded by exactly this". + int transientAllowance = (Environment.ProcessorCount * 8) + 64; + string sharedJson = ConstraintsJson("Area_StressShared"); + int peak = 0; + + Parallel.For(0, (cap * 2) + 64, index => + { + MapConstraints(sharedJson); + MapConstraints(ConstraintsJson($"Area_Stress_{index}")); + + int size = GatewayApiKeyIdentityMapper.CurrentCacheSize; + int seen = Volatile.Read(ref peak); + while (size > seen && Interlocked.CompareExchange(ref peak, size, seen) != seen) + { + seen = Volatile.Read(ref peak); + } + }); + + Assert.True( + peak <= cap + transientAllowance, + $"cache peaked at {peak} entries, past the {cap} cap plus the {transientAllowance} transient allowance"); + + // The cache is process-wide static and other test classes in this assembly map identities + // too, so poll rather than asserting on the instant the loop returns. + Assert.True( + SpinWait.SpinUntil( + () => GatewayApiKeyIdentityMapper.CurrentCacheSize <= cap, + TimeSpan.FromSeconds(5)), + $"cache settled at {GatewayApiKeyIdentityMapper.CurrentCacheSize} entries, past the {cap} cap"); + } + private static ApiKeyConstraints MapConstraints(string constraintsJson) => GatewayApiKeyIdentityMapper.ToGatewayIdentity(new LibApiKeyIdentity( KeyId: "operator01", diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authorization/ApiKeyFailureLimiterTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authorization/ApiKeyFailureLimiterTests.cs index d843b83..cdcdef1 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authorization/ApiKeyFailureLimiterTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authorization/ApiKeyFailureLimiterTests.cs @@ -414,6 +414,46 @@ public sealed class ApiKeyFailureLimiterTests Assert.True(limiter.IsTracked(arriving)); } + /// + /// The PartitionResolution handed back by Check is carried across an await (the + /// inner verification) before Reset consumes it, so the partition it names can be evicted + /// in between. Applying a stale resolution must be inert — never throw, and never clear a + /// partition or aggregate that belongs to somebody else. The failure direction that matters is + /// "clears too little", which costs the caller a probe wait; "clears somebody else's block" + /// would be a throttle bypass. + /// + [Fact] + public void Reset_WithStaleResolutionForEvictedPartition_IsInertAndClearsNoOtherBlock() + { + ManualTimeProvider clock = new(DateTimeOffset.UnixEpoch); + ApiKeyFailureLimiter limiter = CreateLimiter(clock, limit: 3, aggregateLimit: 3, maxPartitions: 2); + ApiKeyThrottlePartition victim = new("ipv4:10.0.0.1:1", "victim"); + ApiKeyThrottlePartition other = new("ipv4:10.0.0.2:1", "other"); + ApiKeyThrottlePartition arriving = new("ipv4:10.0.0.3:1", "arriving"); + + RecordFailures(limiter, victim, 3); + Assert.Equal( + ApiKeyThrottleDecision.ThrottledByPeer, + limiter.Check(victim, out ApiKeyFailureLimiter.PartitionResolution stale)); + Assert.True(stale.IsResolved); + + // Age the victim's window out, then push the map past its cap so eviction takes it — + // expired windows are the first eviction preference, so this is deterministic. The + // resolution captured above now names a partition that no longer exists. + clock.Advance(Window + TimeSpan.FromSeconds(1)); + RecordFailures(limiter, other, 3); + RecordFailures(limiter, arriving, 3); + Assert.False(limiter.IsTracked(victim)); + + limiter.Reset(victim, stale); + + Assert.True(limiter.IsTracked(other)); + Assert.True(limiter.IsTracked(arriving)); + Assert.Equal(2, limiter.TrackedAggregateCount); + Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(other)); + Assert.Equal(ApiKeyThrottleDecision.ThrottledByPeer, limiter.Check(arriving)); + } + private static void RecordFailures(ApiKeyFailureLimiter limiter, ApiKeyThrottlePartition partition, int count) { for (int i = 0; i < count; i++) From f4b065b9f6b798f871d32f99b9f51cfd6127d279 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 17:56:16 -0400 Subject: [PATCH 40/41] docs(worker): reviewer follow-up comments and tests from the remediation reviews MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The worker-side half of the review tail. Tests and comments only — nothing here changes worker behavior, and none of it compiles on the macOS tree (net48/x86), so it was reviewed line by line against the already-windev-validated files. - MxAccessHandleRegistryTests gains the multi-candidate case behind MxAccessSession.TryGetCachedReadFor's fall-through: one tag under two item handles, the lower registered-but-unadvised and the higher advised. Asserted at the registry rather than the session because the session's read path needs a live MXAccess COM instance; what the registry owes the scan is the stable ascending candidate order and a per-item-handle (not per-tag) advice index, and both are pinned here along with the fall-through contract in prose. - A single adversarial lifecycle test — register, advise, re-register the same item handle under a new tag, unadvise, unregister the server — asserting every index agrees after each step. The individual transitions were already covered; what was not was that they compose, and a stale entry in any one index resurrects a handle MXAccess has already retired. - StaWaitHelperTests.WaitForSignalOrMessages_PreSignalledHandle_ReturnsImmediately drains pending messages first, like the other two wait tests. Without it a stale message can end the wait instead of the handle, failing the signal-consumed post-condition for an unrelated reason. - GatewayTesting.md records the two findings from the Task 24 windev gate: SecretsStorePathGuardTests.CreateBuilder_AcceptsSecretsStoreOutsideContentRoot_AndCreatesIt fails deterministically on Windows on main too (SQLite pooling holds secrets.db open across the cleanup's recursive delete; pre-existing, tracked separately), and the StaWaitHelper timing tests' flake signature on a loaded box is a message wake — the helper working as designed — not a broken wait. --- docs/GatewayTesting.md | 17 ++++ .../MxAccess/MxAccessHandleRegistryTests.cs | 87 +++++++++++++++++++ .../Sta/StaWaitHelperTests.cs | 5 ++ 3 files changed, 109 insertions(+) diff --git a/docs/GatewayTesting.md b/docs/GatewayTesting.md index 3c49583..a543d02 100644 --- a/docs/GatewayTesting.md +++ b/docs/GatewayTesting.md @@ -554,6 +554,23 @@ windev has 36 logical CPUs and `xunit.runner.json` sets `maxParallelThreads: -1` suite runs far wider there than on the macOS dev box — that width is what turns these real-clock deadlines into failures. +### Two more findings from the 2026-08-15 windev gate + +- `SecretsStorePathGuardTests.CreateBuilder_AcceptsSecretsStoreOutsideContentRoot_AndCreatesIt` + fails **deterministically on Windows, on `main` as well as on any branch**, so it is not a + signal about the change under test. Creating the builder opens `secrets.db`, and + `Microsoft.Data.Sqlite`'s connection pool keeps the file handle alive past the test body, + so the recursive directory delete in the cleanup hits a still-open file — a sharing + violation Windows enforces and Unix does not. Pre-existing and tracked separately; do not + chase it as a regression. Subtract it from the expected pass count on Windows. +- The `StaWaitHelper` timing tests (`WaitForSignalOrMessages_*`) flake on a loaded box with a + signature that reads like a broken wait but is not: the helper wakes on *input being + present*, so a message posted to the test thread ends the wait early. That is the helper + doing exactly what the STA pump needs. The tests drain the queue with + `PumpPendingMessages()` first for that reason; a failure here means the box was busy enough + to queue a message mid-test, not that the wait stopped honouring its handle or its timeout. + Re-run the class on its own before treating it as real, per the load caveat above. + ### The full-suite testhost hang was a zero-buffer named pipe (fixed) For months a full-suite run on windev reported `855 passed, 0 failed` and then never diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessHandleRegistryTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessHandleRegistryTests.cs index 14814d7..9fcbc2b 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessHandleRegistryTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/MxAccess/MxAccessHandleRegistryTests.cs @@ -256,6 +256,93 @@ public sealed class MxAccessHandleRegistryTests Assert.True(registry.ContainsAdviceHandle(1, 10, MxAccessAdviceKind.Plain)); } + /// + /// Pins the registry half of MxAccessSession.TryGetCachedReadFor's fall-through + /// contract. That scan walks + /// in order and skips any candidate carrying neither a plain nor a supervisory advice, + /// because an added-but-unadvised item will never receive a fresh OnDataChange and so + /// can only serve a stale cache entry. The registry has to make that skip possible: the + /// duplicate registrations of one tag must come back in a stable ascending order, and the + /// advice index must answer per item handle rather than per tag. Asserted here rather than + /// on the session because the session's read path needs a live MXAccess COM instance. + /// + [Fact] + public void GetItemHandlesForDefinition_MultipleCandidates_AdviceIndexDiscriminatesTheAdvisedOne() + { + MxAccessHandleRegistry registry = new(); + registry.RegisterServerHandle(serverHandle: 1, clientName: "client"); + + // Same tag under two item handles: 10 is added but never advised, 20 is advised. Registered + // out of order so the ascending-order guarantee is doing real work. + registry.RegisterItemHandle(1, itemHandle: 20, "Tank1.PV", string.Empty, hasItemContext: false); + registry.RegisterItemHandle(1, itemHandle: 10, "Tank1.PV", string.Empty, hasItemContext: false); + registry.RegisterAdviceHandle(1, itemHandle: 20, MxAccessAdviceKind.Plain); + + IReadOnlyList candidates = registry.GetItemHandlesForDefinition(1, "Tank1.PV"); + + Assert.Equal(new[] { 10, 20 }, candidates); + + // The unadvised candidate is visited first and skipped; the advised one is the survivor. + Assert.False(registry.ContainsAdviceHandle(1, 10, MxAccessAdviceKind.Plain)); + Assert.False(registry.ContainsAdviceHandle(1, 10, MxAccessAdviceKind.Supervisory)); + Assert.True(registry.ContainsAdviceHandle(1, 20, MxAccessAdviceKind.Plain)); + + // Supervisory alone qualifies too, so a later advise on 10 makes it the first survivor. + registry.RegisterAdviceHandle(1, itemHandle: 10, MxAccessAdviceKind.Supervisory); + Assert.True(registry.ContainsAdviceHandle(1, 10, MxAccessAdviceKind.Supervisory)); + } + + /// + /// One adversarial lifecycle run over every index at once: register, advise, re-register the + /// same item handle under a new tag, unadvise, then tear the server down. Each individual + /// transition is covered above; this pins that they compose — the reverse definition index, + /// the per-item advice index and the per-server removal index must agree after every step, + /// since a stale entry in any one of them resurrects a handle MXAccess has already retired. + /// + [Fact] + public void RegisterAdviseReregisterUnadviseUnregister_LeavesEveryIndexConsistent() + { + MxAccessHandleRegistry registry = new(); + registry.RegisterServerHandle(serverHandle: 1, clientName: "client"); + + // Register. + registry.RegisterItemHandle(1, itemHandle: 10, "Tank1.PV", string.Empty, hasItemContext: false); + Assert.True(registry.ContainsItemHandle(1, 10)); + Assert.Equal(new[] { 10 }, registry.GetItemHandlesForDefinition(1, "Tank1.PV")); + Assert.Empty(registry.AdviceHandles); + + // Advise. + registry.RegisterAdviceHandle(1, itemHandle: 10, MxAccessAdviceKind.Plain); + Assert.True(registry.ContainsAdviceHandle(1, 10, MxAccessAdviceKind.Plain)); + Assert.Single(registry.AdviceHandles); + + // Re-register the SAME item handle under a new tag. The advice is keyed on the item handle, + // not the tag, so it survives — but the old definition entry must not. + registry.RegisterItemHandle(1, itemHandle: 10, "Tank2.PV", string.Empty, hasItemContext: false); + Assert.Empty(registry.GetItemHandlesForDefinition(1, "Tank1.PV")); + Assert.Equal(new[] { 10 }, registry.GetItemHandlesForDefinition(1, "Tank2.PV")); + Assert.Single(registry.ItemHandles); + Assert.True(registry.ContainsAdviceHandle(1, 10, MxAccessAdviceKind.Plain)); + + // Unadvise: the item stays registered and still resolves by its current tag. + registry.RemoveAdviceHandles(1, itemHandle: 10); + Assert.False(registry.ContainsAdviceHandle(1, 10, MxAccessAdviceKind.Plain)); + Assert.Empty(registry.AdviceHandles); + Assert.True(registry.ContainsItemHandle(1, 10)); + Assert.Equal(new[] { 10 }, registry.GetItemHandlesForDefinition(1, "Tank2.PV")); + + // Unregister the server: every index drains, including the definition index the re-register + // rewrote. + registry.UnregisterServerHandle(1); + Assert.False(registry.ContainsServerHandle(1)); + Assert.False(registry.ContainsItemHandle(1, 10)); + Assert.Empty(registry.GetItemHandlesForDefinition(1, "Tank1.PV")); + Assert.Empty(registry.GetItemHandlesForDefinition(1, "Tank2.PV")); + Assert.Empty(registry.ServerHandles); + Assert.Empty(registry.ItemHandles); + Assert.Empty(registry.AdviceHandles); + } + private static List Map( IReadOnlyList source, Func selector) diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Sta/StaWaitHelperTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Sta/StaWaitHelperTests.cs index 925f86c..f3572a9 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Sta/StaWaitHelperTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Sta/StaWaitHelperTests.cs @@ -90,6 +90,11 @@ public sealed class StaWaitHelperTests { using AutoResetEvent signal = new(initialState: true); + // Drained for the same reason as the other two waits: a stale message could end this wait + // instead of the handle, which would leave the signal un-consumed and fail the + // post-condition below for a reason that has nothing to do with pre-signalling. + new StaMessagePump().PumpPendingMessages(); + Stopwatch elapsed = Stopwatch.StartNew(); StaWaitHelper.WaitForSignalOrMessages(signal, 30_000); elapsed.Stop(); From 2faf24318919150582039b0b85ba6337b6c5ace6 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 15 Aug 2026 18:07:33 -0400 Subject: [PATCH 41/41] =?UTF-8?q?chore(plans):=20all=2025=20tasks=20comple?= =?UTF-8?q?te=20=E2=80=94=20final=20integration=20review:=20ready=20to=20m?= =?UTF-8?q?erge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/plans/2026-08-15-perf-review-remediation.md.tasks.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json b/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json index 33b93bf..6cd7f30 100644 --- a/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json +++ b/docs/plans/2026-08-15-perf-review-remediation.md.tasks.json @@ -178,11 +178,11 @@ { "id": 25, "subject": "Task 25: Wrap-up docs sweep + follow-ups", - "status": "in_progress", + "status": "completed", "blockedBy": [ 24 ] } ], - "lastUpdated": "2026-08-15T18:40:00Z" + "lastUpdated": "2026-08-15T18:55:00Z" } \ No newline at end of file