From 219fa6ddb4981256fa3fa6396c20687b25de145a Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 06:17:56 -0400 Subject: [PATCH 01/19] fix(CLI-02): make Rust crate buildable outside the repository MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build.rs resolved the .proto files via a monorepo-relative path and packaging relied on `cargo publish --no-verify` to hide the resulting failure. Vendor the three canonical protos into clients/rust/protos/, resolve them via CARGO_MANIFEST_DIR (in-repo path preferred, vendored fallback for the packaged crate), ship them via Cargo.toml `include`, and drop `--no-verify` from pack-clients.ps1. archreview: CLI-02 (P1). Verified: cargo fmt/check/test (28) + clippy -D warnings, and `cargo package` (no --no-verify) compiles standalone in the isolated staging dir — the out-of-repo proof. Vendored-proto drift is guarded by check-codegen.ps1. --- clients/rust/Cargo.toml | 5 + clients/rust/build.rs | 20 +- clients/rust/protos/galaxy_repository.proto | 190 +++ clients/rust/protos/mxaccess_gateway.proto | 1165 +++++++++++++++++++ clients/rust/protos/mxaccess_worker.proto | 132 +++ scripts/pack-clients.ps1 | 6 +- 6 files changed, 1513 insertions(+), 5 deletions(-) create mode 100644 clients/rust/protos/galaxy_repository.proto create mode 100644 clients/rust/protos/mxaccess_gateway.proto create mode 100644 clients/rust/protos/mxaccess_worker.proto diff --git a/clients/rust/Cargo.toml b/clients/rust/Cargo.toml index b4656b9..dd544fb 100644 --- a/clients/rust/Cargo.toml +++ b/clients/rust/Cargo.toml @@ -13,6 +13,11 @@ keywords = ["mxaccess", "mxgateway", "grpc", "client", "archestra"] categories = ["api-bindings", "asynchronous"] publish = ["dohertj2-gitea"] build = "build.rs" +# Ship the vendored `.proto` files so `build.rs` can regenerate the tonic/prost +# bindings from the packaged tarball — a consumer building the published crate +# has no access to the in-repo Contracts protos. See CLI-02. Cargo.toml, the +# README, and the license are included automatically. +include = ["src/**/*.rs", "protos/*.proto", "build.rs", "README.md"] [workspace] members = ["crates/mxgw-cli"] diff --git a/clients/rust/build.rs b/clients/rust/build.rs index 27843f1..bea209e 100644 --- a/clients/rust/build.rs +++ b/clients/rust/build.rs @@ -6,11 +6,25 @@ fn main() -> Result<(), Box> { configure_protoc(); let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); - let repo_root = manifest_dir + + // Resolve the proto source directory. In-repo builds prefer the canonical + // Contracts protos (two levels above clients/rust) so local `.proto` edits + // are picked up live. When that path is absent — e.g. a consumer building + // the crate unpacked from a published tarball — fall back to the vendored + // copies shipped in `clients/rust/protos/` (see build.rs staleness note and + // CLI-02 in archreview/remediation/50-clients.md). Vendored copies are + // build inputs only; the canonical source remains in Contracts and must be + // refreshed here whenever the `.proto` files change. + let repo_proto_root = manifest_dir .parent() .and_then(Path::parent) - .ok_or("clients/rust must live two levels below the repository root")?; - let proto_root = repo_root.join("src/ZB.MOM.WW.MxGateway.Contracts/Protos"); + .map(|repo_root| repo_root.join("src/ZB.MOM.WW.MxGateway.Contracts/Protos")); + let vendored_proto_root = manifest_dir.join("protos"); + let proto_root = match repo_proto_root { + Some(path) if path.is_dir() => path, + _ => vendored_proto_root, + }; + let gateway_proto = proto_root.join("mxaccess_gateway.proto"); let worker_proto = proto_root.join("mxaccess_worker.proto"); let galaxy_proto = proto_root.join("galaxy_repository.proto"); diff --git a/clients/rust/protos/galaxy_repository.proto b/clients/rust/protos/galaxy_repository.proto new file mode 100644 index 0000000..7b4a171 --- /dev/null +++ b/clients/rust/protos/galaxy_repository.proto @@ -0,0 +1,190 @@ +syntax = "proto3"; + +package galaxy_repository.v1; + +option csharp_namespace = "ZB.MOM.WW.MxGateway.Contracts.Proto.Galaxy"; + +import "google/protobuf/timestamp.proto"; +import "google/protobuf/wrappers.proto"; + +// Wire-compatibility policy (ProtobufStyleGuide): this contract evolves +// additively only. Never renumber or repurpose an existing field number or +// enum value. When a field or enum value is removed, add a `reserved` range +// (and `reserved` name) covering it in the same change so a future editor +// cannot accidentally reuse the retired tag. There are no `reserved` +// declarations today because no field or enum value has ever been removed. + +// Read-only browse over the AVEVA System Platform Galaxy Repository (ZB SQL +// database). Lets clients enumerate the deployed object hierarchy and each +// object's dynamic attributes so they know what tag references to subscribe +// to via the MxAccessGateway service. +service GalaxyRepository { + rpc TestConnection(TestConnectionRequest) returns (TestConnectionReply); + rpc GetLastDeployTime(GetLastDeployTimeRequest) returns (GetLastDeployTimeReply); + rpc DiscoverHierarchy(DiscoverHierarchyRequest) returns (DiscoverHierarchyReply); + + // Server-stream of deploy events. The server emits the current state immediately + // on subscribe (so clients can bootstrap their cache without waiting for the next + // deploy), then emits one event each time the gateway's hierarchy cache observes + // a new galaxy.time_of_last_deploy. The sequence field is monotonically + // increasing per server start; gaps indicate the per-subscriber buffer dropped + // older events because the client was too slow. + rpc WatchDeployEvents(WatchDeployEventsRequest) returns (stream DeployEvent); + + // Returns the direct children of a parent object (or the root objects when + // `parent` is unset). Designed for OPC UA-style lazy expand: clients walk + // one level at a time instead of paging the full hierarchy. Filters mirror + // DiscoverHierarchy exactly. Backed by the same shared hierarchy cache. + rpc BrowseChildren(BrowseChildrenRequest) returns (BrowseChildrenReply); +} + +message TestConnectionRequest {} + +message TestConnectionReply { + bool ok = 1; +} + +message GetLastDeployTimeRequest {} + +message GetLastDeployTimeReply { + bool present = 1; + google.protobuf.Timestamp time_of_last_deploy = 2; +} + +message DiscoverHierarchyRequest { + // Maximum number of objects to return. The server applies its default when + // unset and rejects non-positive values. + int32 page_size = 1; + // Opaque token returned by a previous DiscoverHierarchy response. + string page_token = 2; + // Optional. When set, return only this object and its descendants. + // Empty = full hierarchy. + oneof root { + int32 root_gobject_id = 3; + string root_tag_name = 4; + string root_contained_path = 5; + } + // Optional. Cap on descendant depth from root. Zero returns only the root. + // Unset means unlimited depth. + google.protobuf.Int32Value max_depth = 6; + // Optional object category id filters. + repeated int32 category_ids = 7; + // Optional case-insensitive substring filters against template names. + repeated string template_chain_contains = 8; + // Optional anchored, case-insensitive glob over object tag_name. + string tag_name_glob = 9; + // Optional. Unset or true includes attributes. False returns object skeletons. + optional bool include_attributes = 10; + // Optional. Return only objects with at least one alarm-bearing attribute. + bool alarm_bearing_only = 11; + // Optional. Return only objects with at least one historized attribute. + bool historized_only = 12; +} + +message DiscoverHierarchyReply { + repeated GalaxyObject objects = 1; + // Non-empty when another page is available. + string next_page_token = 2; + // Total number of objects in the cached hierarchy at the time of the call. + int32 total_object_count = 3; +} + +message WatchDeployEventsRequest { + // Optional. When set, the bootstrap event is suppressed if the cached deploy + // time matches this value. Future events are still emitted normally. + google.protobuf.Timestamp last_seen_deploy_time = 1; +} + +message DeployEvent { + // Monotonically increasing per server start. Gaps indicate dropped events. + uint64 sequence = 1; + // Server wall-clock when the cache observed the deploy. + google.protobuf.Timestamp observed_at = 2; + // Galaxy.time_of_last_deploy. Absent only when the Galaxy table reports null. + google.protobuf.Timestamp time_of_last_deploy = 3; + bool time_of_last_deploy_present = 4; + int32 object_count = 5; + int32 attribute_count = 6; +} + +message GalaxyObject { + int32 gobject_id = 1; + string tag_name = 2; + string contained_name = 3; + string browse_name = 4; + int32 parent_gobject_id = 5; + bool is_area = 6; + int32 category_id = 7; + int32 hosted_by_gobject_id = 8; + repeated string template_chain = 9; + repeated GalaxyAttribute attributes = 10; +} + +message GalaxyAttribute { + string attribute_name = 1; + string full_tag_reference = 2; + // Raw Galaxy SQL `dbo.data_type` identifier, passed through unchanged. + // This is NOT a member of `mxaccess_gateway.v1.MxDataType` — Galaxy's + // type enumeration is distinct from MXAccess's wire data-type enum and + // the two must not be cast or compared. The GalaxyRepository service is + // metadata-only and deliberately does not share types with + // mxaccess_gateway.proto. See docs/GalaxyRepository.md. + int32 mx_data_type = 3; + // Human-readable name from Galaxy's `dbo.data_type` table (e.g. "Float", + // "Integer", "Boolean"). Free-form Galaxy text; not a stable enum. + string data_type_name = 4; + bool is_array = 5; + int32 array_dimension = 6; + bool array_dimension_present = 7; + // Raw Galaxy SQL attribute-category identifier, passed through unchanged. + // Galaxy-specific; not mapped to any gateway enum. See + // docs/GalaxyRepository.md. + int32 mx_attribute_category = 8; + // Raw Galaxy SQL security-classification identifier, passed through + // unchanged. Galaxy-specific; not mapped to any gateway enum. See + // docs/GalaxyRepository.md. + int32 security_classification = 9; + bool is_historized = 10; + bool is_alarm = 11; +} + +message BrowseChildrenRequest { + // Parent selector. Empty oneof returns root objects (parent_gobject_id == 0). + oneof parent { + int32 parent_gobject_id = 1; + string parent_tag_name = 2; + string parent_contained_path = 3; + } + + // Maximum number of direct children to return. Server default 500; cap 5000. + int32 page_size = 4; + // Opaque token returned by a previous BrowseChildren response. Bound to the + // cache sequence, parent selector, and the filter set; a mismatch returns + // InvalidArgument. + string page_token = 5; + + // --- Filter parity with DiscoverHierarchy. AND-combined. --- + repeated int32 category_ids = 6; + repeated string template_chain_contains = 7; + string tag_name_glob = 8; + optional bool include_attributes = 9; + bool alarm_bearing_only = 10; + bool historized_only = 11; +} + +message BrowseChildrenReply { + // Direct children matching the filter, sorted areas-first then by + // case-insensitive display name (same order as the dashboard tree). + repeated GalaxyObject children = 1; + // Non-empty when another page of siblings is available. + string next_page_token = 2; + // Total matching direct children of the parent (post-filter). + int32 total_child_count = 3; + // Parallel array, indexed with `children`. True when the child has at least + // one matching descendant under the same filter set. Lets a UI choose + // whether to draw an expand triangle without an extra round trip. + repeated bool child_has_children = 4; + // Cache sequence this reply was projected from. Clients may pass it back as + // part of the page_token contract. Mismatch on the next page -> InvalidArgument. + uint64 cache_sequence = 5; +} diff --git a/clients/rust/protos/mxaccess_gateway.proto b/clients/rust/protos/mxaccess_gateway.proto new file mode 100644 index 0000000..d3e3a53 --- /dev/null +++ b/clients/rust/protos/mxaccess_gateway.proto @@ -0,0 +1,1165 @@ +syntax = "proto3"; + +package mxaccess_gateway.v1; + +option csharp_namespace = "ZB.MOM.WW.MxGateway.Contracts.Proto"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; + +// Wire-compatibility policy (ProtobufStyleGuide): this contract evolves +// additively only. Never renumber or repurpose an existing field number or +// enum value. When a field or enum value is removed, add a `reserved` range +// (and `reserved` name) covering it in the same change so a future editor +// cannot accidentally reuse the retired tag. + +// Public client API for MXAccess sessions hosted by the gateway. +service MxAccessGateway { + rpc OpenSession(OpenSessionRequest) returns (OpenSessionReply); + rpc CloseSession(CloseSessionRequest) returns (CloseSessionReply); + rpc Invoke(MxCommandRequest) returns (MxCommandReply); + rpc StreamEvents(StreamEventsRequest) returns (stream MxEvent); + rpc AcknowledgeAlarm(AcknowledgeAlarmRequest) returns (AcknowledgeAlarmReply); + // Session-less central alarm feed. The stream opens with the current + // active-alarm snapshot (one `active_alarm` per alarm), then a single + // `snapshot_complete`, then a `transition` for every subsequent change. + // Served by the gateway's always-on alarm monitor; any number of clients + // fan out from the single monitor without opening a worker session. + rpc StreamAlarms(StreamAlarmsRequest) returns (stream AlarmFeedMessage); + // Point-in-time snapshot of the currently-active alarm set served from the + // gateway's always-on alarm monitor cache (session-less). Used after a + // reconnect to seed Part 9 client state, or to reconcile alarms that may + // have been missed during a transport blip. Streamed so callers can + // begin processing without buffering the full set. + // `QueryActiveAlarmsRequest.alarm_filter_prefix` optionally narrows the + // snapshot to alarms whose `alarm_full_reference` starts with the given + // prefix; an empty prefix returns the full set. + rpc QueryActiveAlarms(QueryActiveAlarmsRequest) returns (stream ActiveAlarmSnapshot); +} + +// Public request shape for QueryActiveAlarms. +// Clients may leave `session_id` empty; the gateway currently ignores it and +// serves the session-less central-monitor cache. A future version may use it +// to scope the snapshot to one session. +message QueryActiveAlarmsRequest { + string session_id = 1; + string client_correlation_id = 2; + // Optional filter: when non-empty, only snapshots whose alarm_full_reference + // starts with this string are returned. + string alarm_filter_prefix = 3; +} + +message OpenSessionRequest { + string requested_backend = 1; + string client_session_name = 2; + string client_correlation_id = 3; + google.protobuf.Duration command_timeout = 4; +} + +message OpenSessionReply { + string session_id = 1; + string backend_name = 2; + int32 worker_process_id = 3; + uint32 worker_protocol_version = 4; + repeated string capabilities = 5; + google.protobuf.Duration default_command_timeout = 6; + ProtocolStatus protocol_status = 7; + // Public gateway contract version implemented by this endpoint. Clients use + // this value to reject incompatible generated-code inputs before issuing + // command-specific MXAccess calls. + uint32 gateway_protocol_version = 8; +} + +message CloseSessionRequest { + string session_id = 1; + string client_correlation_id = 2; +} + +message CloseSessionReply { + string session_id = 1; + SessionState final_state = 2; + ProtocolStatus protocol_status = 3; +} + +message StreamEventsRequest { + string session_id = 1; + uint64 after_worker_sequence = 2; +} + +message MxCommandRequest { + string session_id = 1; + string client_correlation_id = 2; + MxCommand command = 3; +} + +message MxCommand { + MxCommandKind kind = 1; + + oneof payload { + RegisterCommand register = 10; + UnregisterCommand unregister = 11; + AddItemCommand add_item = 12; + AddItem2Command add_item2 = 13; + RemoveItemCommand remove_item = 14; + AdviseCommand advise = 15; + UnAdviseCommand un_advise = 16; + AdviseSupervisoryCommand advise_supervisory = 17; + AddBufferedItemCommand add_buffered_item = 18; + SetBufferedUpdateIntervalCommand set_buffered_update_interval = 19; + SuspendCommand suspend = 20; + ActivateCommand activate = 21; + WriteCommand write = 22; + Write2Command write2 = 23; + WriteSecuredCommand write_secured = 24; + WriteSecured2Command write_secured2 = 25; + AuthenticateUserCommand authenticate_user = 26; + ArchestrAUserToIdCommand archestra_user_to_id = 27; + AddItemBulkCommand add_item_bulk = 28; + AdviseItemBulkCommand advise_item_bulk = 29; + RemoveItemBulkCommand remove_item_bulk = 30; + UnAdviseItemBulkCommand un_advise_item_bulk = 31; + SubscribeBulkCommand subscribe_bulk = 32; + UnsubscribeBulkCommand unsubscribe_bulk = 33; + SubscribeAlarmsCommand subscribe_alarms = 34; + UnsubscribeAlarmsCommand unsubscribe_alarms = 35; + AcknowledgeAlarmCommand acknowledge_alarm_command = 36; + QueryActiveAlarmsCommand query_active_alarms_command = 37; + AcknowledgeAlarmByNameCommand acknowledge_alarm_by_name_command = 38; + WriteBulkCommand write_bulk = 39; + Write2BulkCommand write2_bulk = 40; + WriteSecuredBulkCommand write_secured_bulk = 41; + WriteSecured2BulkCommand write_secured2_bulk = 42; + ReadBulkCommand read_bulk = 43; + PingCommand ping = 100; + GetSessionStateCommand get_session_state = 101; + GetWorkerInfoCommand get_worker_info = 102; + DrainEventsCommand drain_events = 103; + ShutdownWorkerCommand shutdown_worker = 104; + } +} + +enum MxCommandKind { + MX_COMMAND_KIND_UNSPECIFIED = 0; + MX_COMMAND_KIND_REGISTER = 1; + MX_COMMAND_KIND_UNREGISTER = 2; + MX_COMMAND_KIND_ADD_ITEM = 3; + MX_COMMAND_KIND_ADD_ITEM2 = 4; + MX_COMMAND_KIND_REMOVE_ITEM = 5; + MX_COMMAND_KIND_ADVISE = 6; + MX_COMMAND_KIND_UN_ADVISE = 7; + MX_COMMAND_KIND_ADVISE_SUPERVISORY = 8; + MX_COMMAND_KIND_ADD_BUFFERED_ITEM = 9; + MX_COMMAND_KIND_SET_BUFFERED_UPDATE_INTERVAL = 10; + MX_COMMAND_KIND_SUSPEND = 11; + MX_COMMAND_KIND_ACTIVATE = 12; + MX_COMMAND_KIND_WRITE = 13; + MX_COMMAND_KIND_WRITE2 = 14; + MX_COMMAND_KIND_WRITE_SECURED = 15; + MX_COMMAND_KIND_WRITE_SECURED2 = 16; + MX_COMMAND_KIND_AUTHENTICATE_USER = 17; + MX_COMMAND_KIND_ARCHESTRA_USER_TO_ID = 18; + MX_COMMAND_KIND_ADD_ITEM_BULK = 19; + MX_COMMAND_KIND_ADVISE_ITEM_BULK = 20; + MX_COMMAND_KIND_REMOVE_ITEM_BULK = 21; + MX_COMMAND_KIND_UN_ADVISE_ITEM_BULK = 22; + MX_COMMAND_KIND_SUBSCRIBE_BULK = 23; + MX_COMMAND_KIND_UNSUBSCRIBE_BULK = 24; + MX_COMMAND_KIND_SUBSCRIBE_ALARMS = 25; + MX_COMMAND_KIND_UNSUBSCRIBE_ALARMS = 26; + MX_COMMAND_KIND_ACKNOWLEDGE_ALARM = 27; + MX_COMMAND_KIND_QUERY_ACTIVE_ALARMS = 28; + MX_COMMAND_KIND_ACKNOWLEDGE_ALARM_BY_NAME = 29; + MX_COMMAND_KIND_WRITE_BULK = 30; + MX_COMMAND_KIND_WRITE2_BULK = 31; + MX_COMMAND_KIND_WRITE_SECURED_BULK = 32; + MX_COMMAND_KIND_WRITE_SECURED2_BULK = 33; + MX_COMMAND_KIND_READ_BULK = 34; + MX_COMMAND_KIND_PING = 100; + MX_COMMAND_KIND_GET_SESSION_STATE = 101; + MX_COMMAND_KIND_GET_WORKER_INFO = 102; + MX_COMMAND_KIND_DRAIN_EVENTS = 103; + MX_COMMAND_KIND_SHUTDOWN_WORKER = 104; +} + +message RegisterCommand { + string client_name = 1; +} + +message UnregisterCommand { + int32 server_handle = 1; +} + +message AddItemCommand { + int32 server_handle = 1; + string item_definition = 2; +} + +message AddItem2Command { + int32 server_handle = 1; + string item_definition = 2; + string item_context = 3; +} + +message RemoveItemCommand { + int32 server_handle = 1; + int32 item_handle = 2; +} + +message AdviseCommand { + int32 server_handle = 1; + int32 item_handle = 2; +} + +message UnAdviseCommand { + int32 server_handle = 1; + int32 item_handle = 2; +} + +message AdviseSupervisoryCommand { + int32 server_handle = 1; + int32 item_handle = 2; +} + +message AddBufferedItemCommand { + int32 server_handle = 1; + string item_definition = 2; + string item_context = 3; +} + +message SetBufferedUpdateIntervalCommand { + int32 server_handle = 1; + int32 update_interval_milliseconds = 2; +} + +message SuspendCommand { + int32 server_handle = 1; + int32 item_handle = 2; +} + +message ActivateCommand { + int32 server_handle = 1; + int32 item_handle = 2; +} + +message WriteCommand { + int32 server_handle = 1; + int32 item_handle = 2; + MxValue value = 3; + int32 user_id = 4; +} + +message Write2Command { + int32 server_handle = 1; + int32 item_handle = 2; + MxValue value = 3; + MxValue timestamp_value = 4; + int32 user_id = 5; +} + +message WriteSecuredCommand { + int32 server_handle = 1; + int32 item_handle = 2; + int32 current_user_id = 3; + int32 verifier_user_id = 4; + // Credential-sensitive write value. Implementations must not log this field + // unless an explicit redacted value-logging path is enabled. + MxValue value = 5; +} + +message WriteSecured2Command { + int32 server_handle = 1; + int32 item_handle = 2; + int32 current_user_id = 3; + int32 verifier_user_id = 4; + // Credential-sensitive write value. Implementations must not log this field + // unless an explicit redacted value-logging path is enabled. + MxValue value = 5; + MxValue timestamp_value = 6; +} + +message AuthenticateUserCommand { + int32 server_handle = 1; + string verify_user = 2; + // Raw MXAccess credential. Implementations must keep this field out of logs, + // metrics labels, command lines, and diagnostics. + string verify_user_password = 3; +} + +message ArchestrAUserToIdCommand { + int32 server_handle = 1; + string user_id_guid = 2; +} + +message AddItemBulkCommand { + int32 server_handle = 1; + repeated string tag_addresses = 2; +} + +message AdviseItemBulkCommand { + int32 server_handle = 1; + repeated int32 item_handles = 2; +} + +message RemoveItemBulkCommand { + int32 server_handle = 1; + repeated int32 item_handles = 2; +} + +message UnAdviseItemBulkCommand { + int32 server_handle = 1; + repeated int32 item_handles = 2; +} + +message SubscribeBulkCommand { + int32 server_handle = 1; + repeated string tag_addresses = 2; +} + +// Provider selection / current provider for the alarm feed. The zero value +// has two distinct meanings depending on the use site: +// - As SubscribeAlarmsCommand.forced_mode, UNSPECIFIED means auto: alarmmgr +// primary with subtag fallback. +// - As a provenance value (OnAlarmTransitionEvent.source_provider, +// ActiveAlarmSnapshot.source_provider, OnAlarmProviderModeChangedEvent.mode, +// AlarmProviderStatus.mode), the worker always emits ALARMMGR or SUBTAG and +// never UNSPECIFIED; clients should treat a UNSPECIFIED provenance value as +// "unknown / not yet determined". +enum AlarmProviderMode { + ALARM_PROVIDER_MODE_UNSPECIFIED = 0; + ALARM_PROVIDER_MODE_ALARMMGR = 1; + ALARM_PROVIDER_MODE_SUBTAG = 2; +} + +// Subscribe the worker's alarm consumer to an AVEVA alarm provider. +// Subscription expression follows the canonical +// `\\\Galaxy!` format (literal "Galaxy" provider). The +// worker spins up a wnwrapConsumer-backed subscription on its STA on +// first call; subsequent calls are an error (use UnsubscribeAlarms then +// SubscribeAlarms to reconfigure). +message SubscribeAlarmsCommand { + string subscription_expression = 1; + // UNSPECIFIED = auto-failover/failback. ALARMMGR/SUBTAG force one provider. + AlarmProviderMode forced_mode = 2; + // Subtag watch-list resolved by the gateway (GR SQL + config). Empty in pure + // alarmmgr mode; in subtag mode it bounds what the consumer can observe. + repeated AlarmSubtagTarget watch_list = 3; + AlarmFailoverConfig failover = 4; +} + +// Tear down the worker's alarm consumer. No-op if no subscription is +// currently active. +message UnsubscribeAlarmsCommand { +} + +// One alarm attribute the subtag fallback consumer advises. Addresses are full +// MXAccess item references the worker passes straight to AddItem. +message AlarmSubtagTarget { + string alarm_full_reference = 1; // e.g. "Galaxy!Area.Tank01.Level.HiHi" + string source_object_reference = 2; // e.g. "Tank01" + string active_subtag = 3; // item address of the in-alarm boolean + string acked_subtag = 4; // item address of the acknowledged boolean + string ack_comment_subtag = 5; // writable ack-comment attribute (ack write target) + string priority_subtag = 6; // optional severity source; empty if absent +} + +message AlarmFailoverConfig { + int32 consecutive_failure_threshold = 1; // wnwrap COM failures before switching (>=1) + int32 failback_probe_interval_seconds = 2; // probe cadence while degraded (>=1) + int32 failback_stable_probes = 3; // clean probes before switching back (>=1) +} + +// Acknowledge a single alarm by its GUID. Operator identity fields are +// recorded atomically with the ack transition in the alarm-history log. +// The reply's hresult / native_status surfaces AVEVA's +// AlarmAckByGUID return code. +message AcknowledgeAlarmCommand { + // Canonical 8-4-4-4-12 GUID string (e.g. "BCC47053-9542-4D65-BDAA-BCDEA6A32A73"). + string alarm_guid = 1; + string comment = 2; + string operator_user = 3; + string operator_node = 4; + string operator_domain = 5; + string operator_full_name = 6; +} + +// Snapshot the currently-active alarm set. Optional filter prefix scopes +// the snapshot to alarms whose alarm_full_reference starts with the +// supplied string (matches QueryActiveAlarmsRequest.alarm_filter_prefix). +message QueryActiveAlarmsCommand { + string alarm_filter_prefix = 1; +} + +// Acknowledge a single alarm by its (name, provider, group) tuple. Used +// when the public RPC's AlarmFullReference (Provider!Group.Tag) cannot +// be resolved to a GUID directly. The worker invokes +// wwAlarmConsumerClass.AlarmAckByName which reaches the same alarm +// history path as AlarmAckByGUID. +message AcknowledgeAlarmByNameCommand { + // Tag/alarm name (e.g. "TestMachine_001.TestAlarm001"). Tag itself + // may contain dots; the gateway-side parser splits on the first dot + // after the '!' separator. + string alarm_name = 1; + // AVEVA alarm-provider name (literal "Galaxy" for ArchestrA Galaxies). + string provider_name = 2; + // Area/group name (e.g. "TestArea"). + string group_name = 3; + string comment = 4; + string operator_user = 5; + string operator_node = 6; + string operator_domain = 7; + string operator_full_name = 8; +} + +message UnsubscribeBulkCommand { + int32 server_handle = 1; + repeated int32 item_handles = 2; +} + +// Bulk Write — sequential MXAccess Write per entry, on the worker's STA. +// MXAccess has no native bulk write; each entry round-trips through the same +// single-item Write path the gateway uses today. Per-item failures appear as +// BulkWriteResult entries with `was_successful = false` and never throw. +message WriteBulkCommand { + int32 server_handle = 1; + repeated WriteBulkEntry entries = 2; +} + +message WriteBulkEntry { + int32 item_handle = 1; + MxValue value = 2; + int32 user_id = 3; +} + +// Bulk Write2 — sequential MXAccess Write2 (timestamped) per entry. +message Write2BulkCommand { + int32 server_handle = 1; + repeated Write2BulkEntry entries = 2; +} + +message Write2BulkEntry { + int32 item_handle = 1; + MxValue value = 2; + MxValue timestamp_value = 3; + int32 user_id = 4; +} + +// Bulk WriteSecured — sequential MXAccess WriteSecured per entry. +// Credential-sensitive values (`value`) MUST be kept out of logs, metrics +// labels, command lines, and diagnostics — same redaction rules as the +// single-item WriteSecured contract. +message WriteSecuredBulkCommand { + int32 server_handle = 1; + repeated WriteSecuredBulkEntry entries = 2; +} + +message WriteSecuredBulkEntry { + int32 item_handle = 1; + int32 current_user_id = 2; + int32 verifier_user_id = 3; + // Credential-sensitive write value. Implementations must not log this field + // unless an explicit redacted value-logging path is enabled. + MxValue value = 4; +} + +// Bulk WriteSecured2 — sequential MXAccess WriteSecured2 (timestamped) per +// entry. Same redaction rules apply. +message WriteSecured2BulkCommand { + int32 server_handle = 1; + repeated WriteSecured2BulkEntry entries = 2; +} + +message WriteSecured2BulkEntry { + int32 item_handle = 1; + int32 current_user_id = 2; + int32 verifier_user_id = 3; + // Credential-sensitive write value. Implementations must not log this field + // unless an explicit redacted value-logging path is enabled. + MxValue value = 4; + MxValue timestamp_value = 5; +} + +// Bulk Read — snapshot the current value for each requested tag. MXAccess COM +// has no synchronous Read; the worker implements ReadBulk as: +// +// - If the tag is already in the session's item registry AND that item is +// currently advised AND the worker has a cached OnDataChange for it, the +// reply returns the cached value WITHOUT modifying the existing +// subscription (was_cached = true). +// - Otherwise the worker takes the snapshot lifecycle itself: AddItem + +// Advise, wait up to `timeout_ms` for the first OnDataChange, then +// UnAdvise + RemoveItem before returning. The session is left exactly +// as it was before the call (was_cached = false). +// +// `timeout_ms == 0` uses the gateway-configured default (1000 ms). +message ReadBulkCommand { + int32 server_handle = 1; + repeated string tag_addresses = 2; + uint32 timeout_ms = 3; +} + +message PingCommand { + string message = 1; +} + +message GetSessionStateCommand { +} + +message GetWorkerInfoCommand { +} + +message DrainEventsCommand { + uint32 max_events = 1; +} + +message ShutdownWorkerCommand { + google.protobuf.Duration grace_period = 1; +} + +message MxCommandReply { + string session_id = 1; + string correlation_id = 2; + MxCommandKind kind = 3; + ProtocolStatus protocol_status = 4; + // HRESULT captured from MXAccess or a COM exception. This remains separate + // from gateway protocol status so MXAccess parity details are not hidden by + // transport failures. + optional int32 hresult = 5; + MxValue return_value = 6; + repeated MxStatusProxy statuses = 7; + string diagnostic_message = 8; + + oneof payload { + RegisterReply register = 20; + AddItemReply add_item = 21; + AddItem2Reply add_item2 = 22; + AddBufferedItemReply add_buffered_item = 23; + SuspendReply suspend = 24; + ActivateReply activate = 25; + AuthenticateUserReply authenticate_user = 26; + ArchestrAUserToIdReply archestra_user_to_id = 27; + BulkSubscribeReply add_item_bulk = 28; + BulkSubscribeReply advise_item_bulk = 29; + BulkSubscribeReply remove_item_bulk = 30; + BulkSubscribeReply un_advise_item_bulk = 31; + BulkSubscribeReply subscribe_bulk = 32; + BulkSubscribeReply unsubscribe_bulk = 33; + // Reply payload for BOTH MX_COMMAND_KIND_ACKNOWLEDGE_ALARM (by GUID) + // and MX_COMMAND_KIND_ACKNOWLEDGE_ALARM_BY_NAME. There is intentionally + // no by-name-specific reply case: the by-name ack carries no outcome + // detail beyond the native ack return code, so the worker reuses this + // `acknowledge_alarm` payload for both command kinds (the worker's + // MxAccessCommandExecutor sets `acknowledge_alarm` for the by-name arm + // too). Consumers must dispatch on MxCommandReply.kind, not on the + // payload case, to tell the two acks apart. The top-level `hresult` + // mirrors AcknowledgeAlarmReplyPayload.native_status and is preferred. + AcknowledgeAlarmReplyPayload acknowledge_alarm = 34; + QueryActiveAlarmsReplyPayload query_active_alarms = 35; + BulkWriteReply write_bulk = 36; + BulkWriteReply write2_bulk = 37; + BulkWriteReply write_secured_bulk = 38; + BulkWriteReply write_secured2_bulk = 39; + BulkReadReply read_bulk = 40; + SessionStateReply session_state = 100; + WorkerInfoReply worker_info = 101; + DrainEventsReply drain_events = 102; + } +} + +message RegisterReply { + int32 server_handle = 1; +} + +message AddItemReply { + int32 item_handle = 1; +} + +message AddItem2Reply { + int32 item_handle = 1; +} + +message AddBufferedItemReply { + int32 item_handle = 1; +} + +message SuspendReply { + MxStatusProxy status = 1; +} + +message ActivateReply { + MxStatusProxy status = 1; +} + +message AuthenticateUserReply { + int32 user_id = 1; +} + +message ArchestrAUserToIdReply { + int32 user_id = 1; +} + +message SubscribeResult { + int32 server_handle = 1; + string tag_address = 2; + int32 item_handle = 3; + bool was_successful = 4; + string error_message = 5; +} + +message BulkSubscribeReply { + repeated SubscribeResult results = 1; +} + +// Per-item result for the four bulk write families. `item_handle` mirrors the +// request entry's item_handle so callers can correlate inputs to outputs even +// when the gateway's per-entry `IConstraintEnforcer.CheckWriteHandleAsync` +// filter (see `MxAccessGatewayService.ReplaceWriteBulkEntries` and +// `docs/Authorization.md`) dropped some entries before reaching the worker. +// Per-item failures populate `error_message` + `hresult` and never raise — +// callers iterate and inspect each entry. +message BulkWriteResult { + int32 server_handle = 1; + int32 item_handle = 2; + bool was_successful = 3; + optional int32 hresult = 4; + repeated MxStatusProxy statuses = 5; + string error_message = 6; +} + +message BulkWriteReply { + repeated BulkWriteResult results = 1; +} + +// Per-tag result for ReadBulk. `was_cached` is true when the value came from +// an existing live subscription's last OnDataChange (the worker did not touch +// the subscription); false when the worker took the AddItem + Advise + wait + +// UnAdvise + RemoveItem snapshot lifecycle itself. +// +// On `was_successful = true`, `value`, `quality`, `source_timestamp`, and +// `statuses` carry the read data (from the cached subscription or the snapshot +// lifecycle, depending on `was_cached`) and `error_message` is empty. On +// `was_successful = false`, only `server_handle`, `tag_address`, `item_handle` +// (when allocated), `was_cached`, and `error_message` are populated; `value`, +// `quality`, `source_timestamp`, and `statuses` are left at their proto3 +// defaults (null / 0 / null / empty) and must not be read as data — they are +// wire-indistinguishable from "value is null with quality bad" data and serve +// only as absent markers. ReadBulk has no `hresult` field by design (its +// outcomes are timeout / cache / lifecycle states, not MXAccess COM return +// codes — see `docs/DesignDecisions.md` "Bulk Command Family"). Per-tag +// failures populate `error_message` and never raise — callers iterate and +// inspect each entry. +message BulkReadResult { + int32 server_handle = 1; + string tag_address = 2; + int32 item_handle = 3; + bool was_successful = 4; + bool was_cached = 5; + MxValue value = 6; + int32 quality = 7; + google.protobuf.Timestamp source_timestamp = 8; + repeated MxStatusProxy statuses = 9; + string error_message = 10; +} + +message BulkReadReply { + repeated BulkReadResult results = 1; +} + +message SessionStateReply { + SessionState state = 1; +} + +message WorkerInfoReply { + int32 worker_process_id = 1; + string worker_version = 2; + string mxaccess_progid = 3; + string mxaccess_clsid = 4; +} + +message DrainEventsReply { + repeated MxEvent events = 1; +} + +// Reply payload for AcknowledgeAlarmCommand AND +// AcknowledgeAlarmByNameCommand — both ack command kinds reuse this +// payload case (`MxCommandReply.acknowledge_alarm`); there is no +// dedicated by-name reply case. Surfaces AVEVA's native ack return +// code (AlarmAckByGUID for the GUID arm, AlarmAckByName for the +// by-name arm); 0 means success. The MxCommandReply's hresult field +// carries the same value and is preferred for protocol consumers — +// this payload exists so the gateway-side WorkerAlarmRpcDispatcher +// can echo native_status into AcknowledgeAlarmReply.hresult without +// unpacking the outer envelope. +message AcknowledgeAlarmReplyPayload { + int32 native_status = 1; +} + +// Reply payload for QueryActiveAlarmsCommand. The worker walks +// IMxAccessAlarmConsumer.SnapshotActiveAlarms and packs each record as +// an ActiveAlarmSnapshot proto for the gateway-side ConditionRefresh +// stream. +message QueryActiveAlarmsReplyPayload { + repeated ActiveAlarmSnapshot snapshots = 1; +} + +message MxEvent { + MxEventFamily family = 1; + string session_id = 2; + int32 server_handle = 3; + int32 item_handle = 4; + MxValue value = 5; + int32 quality = 6; + google.protobuf.Timestamp source_timestamp = 7; + repeated MxStatusProxy statuses = 8; + uint64 worker_sequence = 9; + google.protobuf.Timestamp worker_timestamp = 10; + google.protobuf.Timestamp gateway_receive_timestamp = 11; + optional int32 hresult = 12; + string raw_status = 13; + // Gateway-synthesized reconnect-replay gap signal. Set ONLY on the single + // sentinel MxEvent the gateway emits at the head of a StreamEvents stream + // that was resumed via StreamEventsRequest.after_worker_sequence when the + // requested sequence is older than the oldest event still retained in the + // session replay ring (i.e. events were evicted and cannot be replayed). + // On that sentinel, `family` is UNSPECIFIED, the `body` oneof is unset, and + // no per-item fields (server_handle/item_handle/value/...) are populated; + // clients MUST treat a present `replay_gap` as "you missed events — discard + // local state and re-snapshot" and read `requested_after_sequence` / + // `oldest_available_sequence` from it. Unset on every normal MXAccess event. + // This field is ONLY ever set on events returned from the StreamEvents server + // stream; it is ALWAYS unset on events in DrainEventsReply (the diagnostic + // drain path never emits the sentinel). + // Additive (proto3): existing clients that ignore this field continue to + // deserialize the stream unchanged. + optional ReplayGap replay_gap = 14; + + oneof body { + OnDataChangeEvent on_data_change = 20; + OnWriteCompleteEvent on_write_complete = 21; + OperationCompleteEvent operation_complete = 22; + OnBufferedDataChangeEvent on_buffered_data_change = 23; + OnAlarmTransitionEvent on_alarm_transition = 24; + OnAlarmProviderModeChangedEvent on_alarm_provider_mode_changed = 25; + } +} + +// Reconnect-replay gap signal carried by a sentinel MxEvent (MxEvent.replay_gap) +// when a client resumes StreamEvents via after_worker_sequence but the requested +// sequence predates the oldest event still held in the session replay ring. +// The events in the open interval (requested_after_sequence, oldest_available_sequence) +// were evicted from the ring and cannot be replayed, so the client must +// re-snapshot rather than assume a contiguous event history. +message ReplayGap { + // The worker_sequence the client asked to resume after + // (StreamEventsRequest.after_worker_sequence). + uint64 requested_after_sequence = 1; + // The oldest worker_sequence still retained in the replay ring and available + // for replay. Events with worker_sequence in the open interval + // (requested_after_sequence, oldest_available_sequence) were evicted and are + // unrecoverable. oldest_available_sequence itself IS still retained: a client + // that wishes to resume without incurring another gap MUST set + // after_worker_sequence = oldest_available_sequence - 1 in the next + // StreamEventsRequest, which will cause the server to replay starting at + // oldest_available_sequence (the first retained event). + uint64 oldest_available_sequence = 2; +} + +enum MxEventFamily { + MX_EVENT_FAMILY_UNSPECIFIED = 0; + MX_EVENT_FAMILY_ON_DATA_CHANGE = 1; + MX_EVENT_FAMILY_ON_WRITE_COMPLETE = 2; + MX_EVENT_FAMILY_OPERATION_COMPLETE = 3; + MX_EVENT_FAMILY_ON_BUFFERED_DATA_CHANGE = 4; + MX_EVENT_FAMILY_ON_ALARM_TRANSITION = 5; + MX_EVENT_FAMILY_ON_ALARM_PROVIDER_MODE_CHANGED = 6; +} + +message OnDataChangeEvent { +} + +message OnWriteCompleteEvent { +} + +message OperationCompleteEvent { +} + +message OnBufferedDataChangeEvent { + MxDataType data_type = 1; + MxArray quality_values = 2; + MxArray timestamp_values = 3; + int32 raw_data_type = 4; +} + +// Carries a single MXAccess alarm transition (raise / acknowledge / clear / +// re-trigger) in native MXAccess terms. The Part 9 state machine + ACL + +// multi-source aggregation lives in lmxopcua's AlarmConditionService; the +// gateway is UA-agnostic and forwards the raw payload. +message OnAlarmTransitionEvent { + // Fully-qualified alarm reference (e.g. "Tank01.Level.HiHi"). Stable across + // transitions of the same condition; used by the lmxopcua side to correlate + // raise/ack/clear into a single Part 9 condition. + string alarm_full_reference = 1; + + // Galaxy-side source object reference (e.g. "Tank01"). Empty for alarms + // that do not bind to a Galaxy object. + string source_object_reference = 2; + + // MxAccess alarm-type qualifier (e.g. "AnalogLimitAlarm.HiHi", "DiscAlarm"). + string alarm_type_name = 3; + + // What kind of state change this event represents. + AlarmTransitionKind transition_kind = 4; + + // Raw MXAccess severity value. Mapping to OPC UA 0-1000 happens server-side + // in lmxopcua via MxAccessSeverityMapper; the gateway preserves the native + // MXAccess scale. + int32 severity = 5; + + // When the alarm originally entered the active state. Preserved across + // acknowledge transitions so the Part 9 condition keeps the original raise + // time. Unset on retrigger from a previously-cleared condition. + google.protobuf.Timestamp original_raise_timestamp = 6; + + // When this specific transition occurred (raise time on Raise, ack time on + // Acknowledge, clear time on Clear). + google.protobuf.Timestamp transition_timestamp = 7; + + // Operator principal recorded by MXAccess on Acknowledge transitions. + // Empty on raise / clear. + string operator_user = 8; + + // Operator-supplied comment recorded by MXAccess on Acknowledge transitions. + // Empty on raise / clear or when no comment was supplied. + string operator_comment = 9; + + // MxAccess alarm category (taxonomy bucket configured in the Galaxy + // template, e.g. "Process", "Safety", "Diagnostics"). + string category = 10; + + // Human-readable alarm description from the MxAccess alarm definition. + string description = 11; + + // Current alarm value (the value of the source attribute at the moment of + // transition). Optional; populated when MxAccess surfaces it. + MxValue current_value = 12; + + // Limit/threshold value that triggered the transition for limit alarms. + // Optional; populated for AnalogLimitAlarm-family transitions. + MxValue limit_value = 13; + + // True when this transition came from the subtag-monitoring fallback rather + // than the native alarmmgr provider — synthesized from data changes, reduced + // fidelity (synthetic GUID, no native raise time). + bool degraded = 14; + // Which provider produced this transition. + AlarmProviderMode source_provider = 15; +} + +message OnAlarmProviderModeChangedEvent { + AlarmProviderMode mode = 1; + string reason = 2; + int32 hresult = 3; // COM HRESULT that triggered failover; 0 on failback + google.protobuf.Timestamp at = 4; +} + +enum AlarmTransitionKind { + ALARM_TRANSITION_KIND_UNSPECIFIED = 0; + ALARM_TRANSITION_KIND_RAISE = 1; + ALARM_TRANSITION_KIND_ACKNOWLEDGE = 2; + ALARM_TRANSITION_KIND_CLEAR = 3; + ALARM_TRANSITION_KIND_RETRIGGER = 4; +} + +// Snapshot of a currently-active MXAccess alarm condition, returned from a +// QueryActiveAlarms ConditionRefresh stream. +message ActiveAlarmSnapshot { + string alarm_full_reference = 1; + string source_object_reference = 2; + string alarm_type_name = 3; + int32 severity = 4; + google.protobuf.Timestamp original_raise_timestamp = 5; + AlarmConditionState current_state = 6; + string category = 7; + string description = 8; + // When the most recent state transition occurred (last raise, last ack, + // last clear). + google.protobuf.Timestamp last_transition_timestamp = 9; + // Operator who acknowledged the alarm if the current state is ActiveAcked. + // Empty otherwise. + string operator_user = 10; + // Operator comment recorded with the most recent acknowledge if the current + // state is ActiveAcked. Empty otherwise. + string operator_comment = 11; + MxValue current_value = 12; + MxValue limit_value = 13; + // True when this snapshot came from the subtag-monitoring fallback rather + // than the native alarmmgr provider — synthesized from data changes, reduced + // fidelity (synthetic GUID, no native raise time). Mirrors + // OnAlarmTransitionEvent.degraded. + bool degraded = 14; + // Which provider produced this snapshot. Mirrors + // OnAlarmTransitionEvent.source_provider; always ALARMMGR or SUBTAG on the + // wire (never UNSPECIFIED). + AlarmProviderMode source_provider = 15; +} + +enum AlarmConditionState { + ALARM_CONDITION_STATE_UNSPECIFIED = 0; + ALARM_CONDITION_STATE_ACTIVE = 1; + ALARM_CONDITION_STATE_ACTIVE_ACKED = 2; + ALARM_CONDITION_STATE_INACTIVE = 3; +} + +message AcknowledgeAlarmRequest { + // Retired: acknowledgement is session-less — it routes to the gateway's + // central alarm monitor, not a client worker session. + reserved 1; + reserved "session_id"; + string client_correlation_id = 2; + // Fully-qualified alarm reference matching OnAlarmTransitionEvent.alarm_full_reference. + string alarm_full_reference = 3; + // Operator-supplied comment forwarded to MXAccess. + string comment = 4; + // Operator principal performing the acknowledgement. The lmxopcua side + // resolves this from the OPC UA session prior to invoking the RPC. + string operator_user = 5; +} + +message AcknowledgeAlarmReply { + // Retired: see AcknowledgeAlarmRequest — acknowledgement is session-less. + reserved 1; + reserved "session_id"; + string correlation_id = 2; + ProtocolStatus protocol_status = 3; + // Native ack return code echoed from the worker. The worker carries the + // ack outcome as a single int32 (AcknowledgeAlarmReplyPayload.native_status, + // = AlarmAckByName / AlarmAckByGUID return code; 0 = success); the gateway's + // WorkerAlarmRpcDispatcher copies that value here. This is the authoritative + // ack-outcome field for the public RPC. Absent only when the worker reply + // omitted the value entirely (a protocol violation). + optional int32 hresult = 4; + // Reserved for a structured MxStatusProxy view of the ack outcome. The + // worker by-name/by-GUID ack path produces only the int32 return code + // (see `hresult`), so the current gateway leaves this field UNSET on every + // reply. Clients must read `hresult` (and `protocol_status`) for the ack + // result and must not depend on `status` being populated. + MxStatusProxy status = 5; + string diagnostic_message = 6; +} + +// Request to attach to the gateway's central alarm feed (StreamAlarms). +message StreamAlarmsRequest { + string client_correlation_id = 1; + // Optional alarm-reference prefix scoping the feed to an equipment + // sub-tree. Empty streams every active alarm. + string alarm_filter_prefix = 2; +} + +// One message on the StreamAlarms feed. The stream opens with one +// `active_alarm` per currently-active alarm, then a single +// `snapshot_complete`, then a `transition` for every subsequent change. +message AlarmFeedMessage { + oneof payload { + // Part of the initial active-alarm snapshot (ConditionRefresh). + ActiveAlarmSnapshot active_alarm = 1; + // Sentinel: the initial snapshot is fully delivered and `transition` + // messages follow. Always true when present. + bool snapshot_complete = 2; + // A live alarm state change (raise / acknowledge / clear). + OnAlarmTransitionEvent transition = 3; + // Provider-mode status. Emitted once on stream open and again on every + // failover/failback so late joiners learn the current mode immediately. + AlarmProviderStatus provider_status = 4; + } +} + +message AlarmProviderStatus { + AlarmProviderMode mode = 1; + bool degraded = 2; // true whenever mode == SUBTAG + string reason = 3; // human-readable switch reason + google.protobuf.Timestamp since = 4; +} + +message MxStatusProxy { + // Mirrors the `success` member of the MXAccess MXSTATUS_PROXY struct + // (a 16-bit signed value in the COM struct, widened to int32 on the + // wire). Despite the name it is NOT a boolean — it is the raw numeric + // indicator the worker reads off the COM struct without reinterpretation. + // It is carried verbatim for diagnostics; the authoritative success/ + // failure of the operation is `category` (MX_STATUS_CATEGORY_OK marks + // success), with `detail`, `diagnostic_text`, `raw_category`, and + // `raw_detected_by` describing any non-OK outcome. Clients should branch + // on `category`, not on a specific `success` value. + int32 success = 1; + MxStatusCategory category = 2; + MxStatusSource detected_by = 3; + int32 detail = 4; + int32 raw_category = 5; + int32 raw_detected_by = 6; + string diagnostic_text = 7; +} + +enum MxStatusCategory { + MX_STATUS_CATEGORY_UNSPECIFIED = 0; + MX_STATUS_CATEGORY_UNKNOWN = 1; + MX_STATUS_CATEGORY_OK = 2; + MX_STATUS_CATEGORY_PENDING = 3; + MX_STATUS_CATEGORY_WARNING = 4; + MX_STATUS_CATEGORY_COMMUNICATION_ERROR = 5; + MX_STATUS_CATEGORY_CONFIGURATION_ERROR = 6; + MX_STATUS_CATEGORY_OPERATIONAL_ERROR = 7; + MX_STATUS_CATEGORY_SECURITY_ERROR = 8; + MX_STATUS_CATEGORY_SOFTWARE_ERROR = 9; + MX_STATUS_CATEGORY_OTHER_ERROR = 10; +} + +enum MxStatusSource { + MX_STATUS_SOURCE_UNSPECIFIED = 0; + MX_STATUS_SOURCE_UNKNOWN = 1; + MX_STATUS_SOURCE_REQUESTING_LMX = 2; + MX_STATUS_SOURCE_RESPONDING_LMX = 3; + MX_STATUS_SOURCE_REQUESTING_NMX = 4; + MX_STATUS_SOURCE_RESPONDING_NMX = 5; + MX_STATUS_SOURCE_REQUESTING_AUTOMATION_OBJECT = 6; + MX_STATUS_SOURCE_RESPONDING_AUTOMATION_OBJECT = 7; +} + +message MxValue { + MxDataType data_type = 1; + string variant_type = 2; + bool is_null = 3; + string raw_diagnostic = 4; + int32 raw_data_type = 5; + + oneof kind { + bool bool_value = 10; + int32 int32_value = 11; + int64 int64_value = 12; + float float_value = 13; + double double_value = 14; + string string_value = 15; + google.protobuf.Timestamp timestamp_value = 16; + MxArray array_value = 17; + bytes raw_value = 18; + MxSparseArray sparse_array_value = 19; + } +} + +message MxArray { + MxDataType element_data_type = 1; + string variant_type = 2; + repeated uint32 dimensions = 3; + string raw_diagnostic = 4; + int32 raw_element_data_type = 5; + + oneof values { + BoolArray bool_values = 10; + Int32Array int32_values = 11; + Int64Array int64_values = 12; + FloatArray float_values = 13; + DoubleArray double_values = 14; + StringArray string_values = 15; + TimestampArray timestamp_values = 16; + RawArray raw_values = 17; + } +} + +// Write-only sparse array value. The gateway expands this into a full, +// default-filled MxArray before forwarding to the worker; the worker never +// receives or produces it. Unmentioned indices take the element type's +// default (reset, NOT preserved). +message MxSparseArray { + MxDataType element_data_type = 1; + uint32 total_length = 2; + repeated MxSparseElement elements = 3; +} + +message MxSparseElement { + uint32 index = 1; + MxValue value = 2; // scalar +} + +message BoolArray { + repeated bool values = 1; +} + +message Int32Array { + repeated int32 values = 1; +} + +message Int64Array { + repeated int64 values = 1; +} + +message FloatArray { + repeated float values = 1; +} + +message DoubleArray { + repeated double values = 1; +} + +message StringArray { + repeated string values = 1; +} + +message TimestampArray { + repeated google.protobuf.Timestamp values = 1; +} + +message RawArray { + repeated bytes values = 1; +} + +enum MxDataType { + MX_DATA_TYPE_UNSPECIFIED = 0; + MX_DATA_TYPE_UNKNOWN = 1; + MX_DATA_TYPE_NO_DATA = 2; + MX_DATA_TYPE_BOOLEAN = 3; + MX_DATA_TYPE_INTEGER = 4; + MX_DATA_TYPE_FLOAT = 5; + MX_DATA_TYPE_DOUBLE = 6; + MX_DATA_TYPE_STRING = 7; + MX_DATA_TYPE_TIME = 8; + MX_DATA_TYPE_ELAPSED_TIME = 9; + MX_DATA_TYPE_REFERENCE_TYPE = 10; + MX_DATA_TYPE_STATUS_TYPE = 11; + MX_DATA_TYPE_ENUM = 12; + MX_DATA_TYPE_SECURITY_CLASSIFICATION_ENUM = 13; + MX_DATA_TYPE_DATA_QUALITY_TYPE = 14; + MX_DATA_TYPE_QUALIFIED_ENUM = 15; + MX_DATA_TYPE_QUALIFIED_STRUCT = 16; + MX_DATA_TYPE_INTERNATIONALIZED_STRING = 17; + MX_DATA_TYPE_BIG_STRING = 18; + MX_DATA_TYPE_END = 19; +} + +message ProtocolStatus { + ProtocolStatusCode code = 1; + string message = 2; +} + +enum ProtocolStatusCode { + PROTOCOL_STATUS_CODE_UNSPECIFIED = 0; + PROTOCOL_STATUS_CODE_OK = 1; + PROTOCOL_STATUS_CODE_INVALID_REQUEST = 2; + PROTOCOL_STATUS_CODE_SESSION_NOT_FOUND = 3; + PROTOCOL_STATUS_CODE_SESSION_NOT_READY = 4; + PROTOCOL_STATUS_CODE_WORKER_UNAVAILABLE = 5; + PROTOCOL_STATUS_CODE_TIMEOUT = 6; + PROTOCOL_STATUS_CODE_CANCELED = 7; + PROTOCOL_STATUS_CODE_PROTOCOL_VIOLATION = 8; + PROTOCOL_STATUS_CODE_MXACCESS_FAILURE = 9; +} + +enum SessionState { + SESSION_STATE_UNSPECIFIED = 0; + SESSION_STATE_CREATING = 1; + SESSION_STATE_STARTING_WORKER = 2; + SESSION_STATE_WAITING_FOR_PIPE = 3; + SESSION_STATE_HANDSHAKING = 4; + SESSION_STATE_INITIALIZING_WORKER = 5; + SESSION_STATE_READY = 6; + SESSION_STATE_CLOSING = 7; + SESSION_STATE_CLOSED = 8; + SESSION_STATE_FAULTED = 9; +} diff --git a/clients/rust/protos/mxaccess_worker.proto b/clients/rust/protos/mxaccess_worker.proto new file mode 100644 index 0000000..06a22de --- /dev/null +++ b/clients/rust/protos/mxaccess_worker.proto @@ -0,0 +1,132 @@ +syntax = "proto3"; + +package mxaccess_worker.v1; + +option csharp_namespace = "ZB.MOM.WW.MxGateway.Contracts.Proto"; + +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; +import "mxaccess_gateway.proto"; + +// Wire-compatibility policy (ProtobufStyleGuide): this contract evolves +// additively only. Never renumber or repurpose an existing field number or +// enum value. When a field or enum value is removed, add a `reserved` range +// (and `reserved` name) covering it in the same change so a future editor +// cannot accidentally reuse the retired tag. There are no `reserved` +// declarations today because no field or enum value has ever been removed. + +// Gateway-to-worker IPC envelope. Named-pipe framing prepends a little-endian +// uint32 payload length to this protobuf payload. +message WorkerEnvelope { + uint32 protocol_version = 1; + string session_id = 2; + uint64 sequence = 3; + string correlation_id = 4; + + oneof body { + GatewayHello gateway_hello = 10; + WorkerHello worker_hello = 11; + WorkerReady worker_ready = 12; + WorkerCommand worker_command = 13; + WorkerCommandReply worker_command_reply = 14; + WorkerCancel worker_cancel = 15; + WorkerShutdown worker_shutdown = 16; + WorkerShutdownAck worker_shutdown_ack = 17; + WorkerEvent worker_event = 18; + WorkerHeartbeat worker_heartbeat = 19; + WorkerFault worker_fault = 20; + } +} + +message GatewayHello { + uint32 supported_protocol_version = 1; + string nonce = 2; + string gateway_version = 3; +} + +message WorkerHello { + uint32 protocol_version = 1; + string nonce = 2; + int32 worker_process_id = 3; + string worker_version = 4; +} + +message WorkerReady { + int32 worker_process_id = 1; + string mxaccess_progid = 2; + string mxaccess_clsid = 3; + google.protobuf.Timestamp ready_timestamp = 4; +} + +message WorkerCommand { + mxaccess_gateway.v1.MxCommand command = 1; + google.protobuf.Timestamp enqueue_timestamp = 2; +} + +message WorkerCommandReply { + mxaccess_gateway.v1.MxCommandReply reply = 1; + google.protobuf.Timestamp completed_timestamp = 2; +} + +message WorkerCancel { + string reason = 1; +} + +message WorkerShutdown { + google.protobuf.Duration grace_period = 1; + string reason = 2; +} + +message WorkerShutdownAck { + mxaccess_gateway.v1.ProtocolStatus status = 1; +} + +message WorkerEvent { + mxaccess_gateway.v1.MxEvent event = 1; +} + +message WorkerHeartbeat { + int32 worker_process_id = 1; + WorkerState state = 2; + google.protobuf.Timestamp last_sta_activity_timestamp = 3; + uint32 pending_command_count = 4; + uint32 outbound_event_queue_depth = 5; + uint64 last_event_sequence = 6; + string current_command_correlation_id = 7; +} + +message WorkerFault { + WorkerFaultCategory category = 1; + string command_method = 2; + optional int32 hresult = 3; + string exception_type = 4; + string diagnostic_message = 5; + mxaccess_gateway.v1.ProtocolStatus protocol_status = 6; +} + +enum WorkerState { + WORKER_STATE_UNSPECIFIED = 0; + WORKER_STATE_STARTING = 1; + WORKER_STATE_HANDSHAKING = 2; + WORKER_STATE_INITIALIZING_STA = 3; + WORKER_STATE_READY = 4; + WORKER_STATE_EXECUTING_COMMAND = 5; + WORKER_STATE_SHUTTING_DOWN = 6; + WORKER_STATE_STOPPED = 7; + WORKER_STATE_FAULTED = 8; +} + +enum WorkerFaultCategory { + WORKER_FAULT_CATEGORY_UNSPECIFIED = 0; + WORKER_FAULT_CATEGORY_INVALID_ARGUMENTS = 1; + WORKER_FAULT_CATEGORY_GATEWAY_AUTHENTICATION_FAILED = 2; + WORKER_FAULT_CATEGORY_PROTOCOL_MISMATCH = 3; + WORKER_FAULT_CATEGORY_PROTOCOL_VIOLATION = 4; + WORKER_FAULT_CATEGORY_PIPE_DISCONNECTED = 5; + WORKER_FAULT_CATEGORY_MXACCESS_CREATION_FAILED = 6; + WORKER_FAULT_CATEGORY_MXACCESS_COMMAND_FAILED = 7; + WORKER_FAULT_CATEGORY_MXACCESS_EVENT_CONVERSION_FAILED = 8; + WORKER_FAULT_CATEGORY_STA_HUNG = 9; + WORKER_FAULT_CATEGORY_QUEUE_OVERFLOW = 10; + WORKER_FAULT_CATEGORY_SHUTDOWN_TIMEOUT = 11; +} diff --git a/scripts/pack-clients.ps1 b/scripts/pack-clients.ps1 index 12aa220..a3b7b21 100644 --- a/scripts/pack-clients.ps1 +++ b/scripts/pack-clients.ps1 @@ -187,7 +187,9 @@ function Invoke-PackRust { } Write-Host 'Running cargo package...' - & cargo package --no-verify + # No --no-verify: the crate vendors its protos (clients/rust/protos) so + # the verify build proves it compiles standalone from the tarball (CLI-02). + & cargo package if ($LASTEXITCODE -ne 0) { throw 'cargo package failed.' } $packageDir = Join-Path $rustDir 'target/package' @@ -207,7 +209,7 @@ function Invoke-PackRust { Write-Host 'Publishing Rust crate to Gitea...' -ForegroundColor Yellow Push-Location (Join-Path $RepoRoot 'clients/rust') try { - & cargo publish --no-verify --registry dohertj2-gitea + & cargo publish --registry dohertj2-gitea if ($LASTEXITCODE -ne 0) { throw 'cargo publish failed.' } } finally { Pop-Location } } From d5248f61a29eed80da39a8e3451a646337297543 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 06:17:56 -0400 Subject: [PATCH 02/19] fix(archreview): CI pipeline + codegen freshness guards (IPC-01/09/19/20, TST-03) - IPC-01: regenerate the stale client descriptor set and add ClientProtoInputTests (semantic, protoc-free) so a missing contract symbol fails the build. - IPC-20: make publish-client-proto-inputs.ps1 -Check source_code_info-normalized so it is protoc-version tolerant. - IPC-19: document + guard the "Generated/ must be committed for net48" rule (docs/Contracts.md, csproj comment, check-codegen.ps1). - IPC-09: harden the per-client generate-proto scripts (PATH resolution + pinned grpcio/protobuf/java version asserts) so regeneration is reproducible. - TST-03: add .gitea/workflows/ci.yml (portable/java/windows/live jobs) running the build, tests, client checks, and the codegen guards. - Also: check-codegen.ps1 Check 3 guards the CLI-02 vendored Rust protos against drift. archreview: IPC-01/09/19/20 Done, TST-03 In review (pipeline authored + validated, not yet run on a Gitea runner). Verified on macOS: NonWindows build clean, ClientProtoInputTests 5/5, -Check exit 0. --- .gitea/workflows/ci.yml | 124 +++++++++++++++++ clients/go/generate-proto.ps1 | 50 +++++-- .../zb-mom-ww-mxgateway-client/build.gradle | 32 +++++ .../descriptors/mxaccessgw-client-v1.protoset | Bin 80222 -> 116804 bytes clients/python/generate-proto.ps1 | 47 ++++++- clients/python/pyproject.toml | 3 + docs/ClientProtoGeneration.md | 42 +++++- docs/Contracts.md | 35 ++++- docs/GatewayTesting.md | 28 ++++ scripts/check-codegen.ps1 | 96 +++++++++++++ scripts/publish-client-proto-inputs.ps1 | 131 ++++++++++++++---- .../ZB.MOM.WW.MxGateway.Contracts.csproj | 7 + .../Contracts/ClientProtoInputTests.cs | 97 +++++++++++++ 13 files changed, 645 insertions(+), 47 deletions(-) create mode 100644 .gitea/workflows/ci.yml create mode 100644 scripts/check-codegen.ps1 diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..9c6cbdb --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,124 @@ +# Continuous integration for mxaccessgw (Gitea Actions; origin is Gitea at gitea.dohertylan.com). +# +# Scope note: the x86 Worker (src/ZB.MOM.WW.MxGateway.Worker) and Worker.Tests target +# .NET Framework 4.8 / x86 and need MXAccess COM installed, so they build ONLY on a Windows host. +# They are out of scope for this Linux `portable` job — the `windows` job below (self-hosted windev +# runner) covers them, and live-MXAccess integration runs are scheduled-only so they never gate a push. +name: ci + +on: + push: + pull_request: + schedule: + # Nightly live-MXAccess smoke (windev). Push/PR runs skip the live-mxaccess job. + - cron: '0 6 * * *' + +jobs: + portable: + # Any Linux runner: no MXAccess, no x86. Covers the NonWindows solution, gateway fake-worker + # tests, the codegen/descriptor freshness guards, and the clients that build on Linux. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-dotnet@v4 + with: + dotnet-version: '10.0.x' + + - uses: actions/setup-go@v5 + with: + go-version: '1.26' + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy, rustfmt + + # protoc is pinned to 34.1 (see docs/ToolchainLinks.md) so the committed client descriptor + # regenerates reproducibly. The freshness check normalizes source_code_info, so it tolerates + # patch drift, but keep the CI toolchain on the pin. + - name: Install protoc 34.1 + run: | + curl -sSL -o /tmp/protoc.zip https://github.com/protocolbuffers/protobuf/releases/download/v34.1/protoc-34.1-linux-x86_64.zip + sudo unzip -o /tmp/protoc.zip -d /usr/local bin/protoc 'include/*' + protoc --version + + - name: Build NonWindows solution + run: dotnet build src/ZB.MOM.WW.MxGateway.NonWindows.slnx -c Release + + # IPC-01 / IPC-19 / IPC-20: descriptor set + Contracts/Generated must match the current protos. + - name: Codegen / descriptor freshness + shell: pwsh + run: ./scripts/check-codegen.ps1 + + - name: Gateway fake-worker tests + run: dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj -c Release --no-build + + - name: .NET client + run: dotnet build clients/dotnet/ZB.MOM.WW.MxGateway.Client.slnx -c Release + + - name: Go client + working-directory: clients/go + run: | + test -z "$(gofmt -l .)" || (gofmt -l . && echo 'gofmt needed' && exit 1) + go build ./... + go test ./... + + - name: Rust client + working-directory: clients/rust + run: | + cargo fmt --all -- --check + cargo test --workspace + cargo clippy --workspace --all-targets -- -D warnings + + - name: Python client + working-directory: clients/python + run: | + python -m pip install --upgrade pip + python -m pip install -e ".[dev]" + python -m pytest + + java: + # Java client runs on a JDK-17 Linux runner (the macOS dev box has no JRE). The protobuf gradle + # plugin rewrites MxaccessGateway.java with spurious protobuf-runtime-version churn on every + # build; when no .proto changed, revert that one file so checkGeneratedClean / a dirty tree does + # not fail the build (repo memory project_java_generated_churn). + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + - name: Gradle test + working-directory: clients/java + run: gradle test + - name: Revert spurious protobuf-version churn (no .proto changed) + run: git checkout -- clients/java/src/main/generated/main/java/mxaccess_gateway/v1/MxaccessGateway.java || true + - name: Verify generated tree is clean + run: git diff --exit-code -- clients/java/src/main/generated + + windows: + # Self-hosted windev runner (10.100.0.48): the only host that can build the x86 / net48 Worker + # and MXAccess-adjacent code. Not required for a green portable build; runs where a runner exists. + runs-on: [self-hosted, windows] + steps: + - uses: actions/checkout@v4 + - name: Build x86 Worker + run: dotnet build src/ZB.MOM.WW.MxGateway.Worker/ZB.MOM.WW.MxGateway.Worker.csproj -p:Platform=x86 + - name: Worker tests + run: dotnet test src/ZB.MOM.WW.MxGateway.Worker.Tests/ZB.MOM.WW.MxGateway.Worker.Tests.csproj -p:Platform=x86 + + live-mxaccess: + # Opt-in, scheduled only — needs installed MXAccess COM + live provider state. Never gates a push. + if: ${{ github.event_name == 'schedule' }} + runs-on: [self-hosted, windows, mxaccess] + env: + MXGATEWAY_RUN_LIVE_MXACCESS_TESTS: '1' + steps: + - uses: actions/checkout@v4 + - name: Live MXAccess smoke + run: dotnet test src/ZB.MOM.WW.MxGateway.IntegrationTests/ZB.MOM.WW.MxGateway.IntegrationTests.csproj --filter FullyQualifiedName~WorkerLiveMxAccessSmokeTests diff --git a/clients/go/generate-proto.ps1 b/clients/go/generate-proto.ps1 index 1e88740..9c807e6 100644 --- a/clients/go/generate-proto.ps1 +++ b/clients/go/generate-proto.ps1 @@ -5,25 +5,46 @@ $repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') $protoRoot = Join-Path $repoRoot 'src\ZB.MOM.WW.MxGateway.Contracts\Protos' $outputRoot = Join-Path $PSScriptRoot 'internal\generated' $modulePath = 'gitea.dohertylan.com/dohertj2/mxaccessgw/clients/go/internal/generated' -$protoc = 'C:\Users\dohertj2\AppData\Local\Microsoft\WinGet\Packages\Google.Protobuf_Microsoft.Winget.Source_8wekyb3d8bbwe\bin\protoc.exe' -$goPluginPath = 'C:\Users\dohertj2\go\bin' -if (-not (Test-Path $protoc)) { - throw "protoc was not found at $protoc. See docs/ToolchainLinks.md." -} +function Resolve-Tool { + # Resolve a codegen tool from PATH first (portable), then the documented Windows fallbacks, + # instead of the previous hard-coded per-machine paths. See docs/ToolchainLinks.md. + param( + [string[]]$Names, + [string[]]$FallbackPaths + ) -foreach ($pluginName in @('protoc-gen-go.exe', 'protoc-gen-go-grpc.exe')) { - $pluginPath = Join-Path $goPluginPath $pluginName - if (-not (Test-Path $pluginPath)) { - throw "$pluginName was not found at $pluginPath. See docs/ToolchainLinks.md." + foreach ($name in $Names) { + $cmd = Get-Command $name -ErrorAction SilentlyContinue + if ($null -ne $cmd) { + return $cmd.Source + } } + + foreach ($fallback in $FallbackPaths) { + if ($fallback -and (Test-Path $fallback)) { + return $fallback + } + } + + throw "Could not find $($Names -join '/') on PATH. See docs/ToolchainLinks.md." } +$wingetProtoc = if ($env:LOCALAPPDATA) { + Join-Path $env:LOCALAPPDATA 'Microsoft\WinGet\Packages\Google.Protobuf_Microsoft.Winget.Source_8wekyb3d8bbwe\bin\protoc.exe' +} else { $null } +$goBin = if ($env:USERPROFILE) { Join-Path $env:USERPROFILE 'go\bin' } elseif ($env:HOME) { Join-Path $env:HOME 'go/bin' } else { $null } + +$protoc = Resolve-Tool -Names @('protoc', 'protoc.exe') -FallbackPaths @($wingetProtoc) +$protocGenGo = Resolve-Tool -Names @('protoc-gen-go', 'protoc-gen-go.exe') -FallbackPaths @((if ($goBin) { Join-Path $goBin 'protoc-gen-go.exe' }), (if ($goBin) { Join-Path $goBin 'protoc-gen-go' })) +$protocGenGoGrpc = Resolve-Tool -Names @('protoc-gen-go-grpc', 'protoc-gen-go-grpc.exe') -FallbackPaths @((if ($goBin) { Join-Path $goBin 'protoc-gen-go-grpc.exe' }), (if ($goBin) { Join-Path $goBin 'protoc-gen-go-grpc' })) + +# protoc discovers the plugins on PATH; prepend the directories the resolved plugins live in. +$env:Path = (Split-Path $protocGenGo -Parent) + [System.IO.Path]::PathSeparator + (Split-Path $protocGenGoGrpc -Parent) + [System.IO.Path]::PathSeparator + $env:Path + New-Item -ItemType Directory -Path $outputRoot -Force | Out-Null Get-ChildItem -Path $outputRoot -Filter '*.pb.go' -File | Remove-Item -$env:Path = "$goPluginPath;$env:Path" - & $protoc ` --proto_path=$protoRoot ` --go_out=$outputRoot ` @@ -35,6 +56,10 @@ $env:Path = "$goPluginPath;$env:Path" mxaccess_worker.proto ` galaxy_repository.proto +if ($LASTEXITCODE -ne 0) { + throw "protoc (go) failed with exit code $LASTEXITCODE." +} + & $protoc ` --proto_path=$protoRoot ` --go-grpc_out=$outputRoot ` @@ -44,3 +69,6 @@ $env:Path = "$goPluginPath;$env:Path" mxaccess_gateway.proto ` galaxy_repository.proto +if ($LASTEXITCODE -ne 0) { + throw "protoc (go-grpc) failed with exit code $LASTEXITCODE." +} diff --git a/clients/java/zb-mom-ww-mxgateway-client/build.gradle b/clients/java/zb-mom-ww-mxgateway-client/build.gradle index aea6eef..cdb2b4a 100644 --- a/clients/java/zb-mom-ww-mxgateway-client/build.gradle +++ b/clients/java/zb-mom-ww-mxgateway-client/build.gradle @@ -57,3 +57,35 @@ protobuf { } } } + +// IPC-09 guard: the generated Java tree is tracked (generatedFilesBaseDir above points at the +// committed src/main/generated). `gradle build` regenerates it, so an un-regenerated .proto change, +// or a plugin/protobuf version bump, silently drifts the committed output. checkGeneratedClean +// fails when the regenerated tree differs from what is committed. +// +// Caveat (repo memory project_java_generated_churn): the protobuf gradle plugin also rewrites +// MxaccessGateway.java with a spurious protobuf-runtime-version delta on every build even when no +// .proto changed. CI reverts that one file (git checkout) before invoking this task; locally, do the +// same when you did not touch a .proto. See docs/GatewayTesting.md "Continuous Integration". +tasks.register('checkGeneratedClean') { + group = 'verification' + description = 'Fails if the committed generated Java tree differs from a fresh regeneration.' + dependsOn 'generateProto' + doLast { + def generatedDir = 'clients/java/src/main/generated' + def stdout = new ByteArrayOutputStream() + def result = exec { + workingDir = rootProject.projectDir.parentFile.parentFile + commandLine 'git', 'status', '--porcelain', '--', generatedDir + standardOutput = stdout + ignoreExitValue = true + } + def dirty = stdout.toString().trim() + if (!dirty.isEmpty()) { + throw new GradleException( + "Generated Java is stale or churned:\n${dirty}\n" + + "Regenerate and commit after a .proto change, or 'git checkout' the spurious " + + "MxaccessGateway.java protobuf-version churn when no .proto changed.") + } + } +} diff --git a/clients/proto/descriptors/mxaccessgw-client-v1.protoset b/clients/proto/descriptors/mxaccessgw-client-v1.protoset index 6dd07cb9f386813485116e85529b8e0212a09f58..bdb64fb1824570a7ec8501f9dd82fef6ba09d79f 100644 GIT binary patch literal 116804 zcmeFa34B~hb?DuF`&yQ+yvWy^EH882-XxDC+ww9qo(0;HY-`4rOiLOwLlU_ywd7`` zZlP}3)({BGu;sA@hBa(ifMnRW0D-(fAPh;!7xq1YyaW=Gu)L2Yge7_ZQ_H=#TP-hv zykEZGhZ8`j>zq@ks!p9cb?Q`I?{mAnZBwPv)O3E|>|Ci*IzK{kdW1m6sTfODw{CKH2QEp8J$@Wy%g8n*hSus~Em0KNtUzXLtv8N`zwd19k z?i!xQR$7i{M2a)Xdk+|lsnT?=IMrR6n~DUiEX?N1`|c~2E)_#zXV3pInfz@hb>hsi zhn#IERFk?DGeqt_o1dP(OYld8^v*Br@EUw~TVgKheQbr7T;Zp-CH$Tbt_T#tcxgH~ zHa|I;pDPDlfo8R*98BaYxu8(2^5y*8#r#CK7hI@RX3IzS?VHG7%uknQsmOK+eWTln5M4+vLhSRr zAe)~klq++E^Ybz?L9RFv%$M_s6O>EybK`mS%lSevH@BeVDt88#3Y7~%X-@rbX&yOe zN)v_2!nhjS&LB6J52$&jP^si60*3lxVS+y@7jhN;OIkNQUAj~#P6aZ&1*x_yMDWR1 zjv|u$+gl@8IVeq9iN;G4`Cw+gT%qNa9Fhvx-1*YQJRD7rc|oyMDU9bkDJ+zOY1$|p z;x*FJYH{d@@#$P)CO=0{))#`xVtrwSpp6srbX6#a7YXt+$>GU32-k_y`20-1SW#Up zL$Z(lFHtfVpegyeLT~N47?BmzLk?g>+(UGC-uot8=!;q#`NoMe#p#RZl zvi-xu!B93BIDO{S0Nh}o%?yqV^bdCi1B1P%M*9W^Pjm)w2nL5nyx`Qp>46dOMus|7 zwTtP3q2s~n{%r3_*kq0koEjLpM^$)yU}R7#J3f^4f=qBGlN}l89X*xF24_aIXNHFR z18G^`z;N%W%)sgXzHaKL&Y=J9{=t!8_+;kPsp^J$!O+>k{;WtJZ)R|;A1O1(PW4Mw zRDBOjv;Dmz(xB+CUOEo}Pj!01@R|PJ0sf*t`ss#D_MT3|bh!UfqhL`O^kq(GPV^54 z?N{p+2Bvp3+kaZ5rYnX=j}4CujEs)-2PcMx`c!8R_h;`O=?8$z9XP^glZ)K!IhEa~*kvJL^qALt^)XreA|HP?*6a9m|{UXnhusJ(0 z+~2`~4ItYI;VIpGHbbQ-3Kfe+A{Rp>f5%jpPG&rVf#X4@?{1j%S!^_5c)+L&M8l)K zCrzKD{*7MJ@l!#fRsL%9Q|ZJFUX$lE-k|@~Kki7JR6jQRDfv_XxHHkIeoXo)`BVS6 zE77ffgp>TKf83o&s~;+c^1C5 z=eMBScGbm1&&v8##1iC!a$%~NpXdxG3zzc~UDNsE6e~%2HdhoARKfB_==ljR$O+AO zX}+jzw{Zv5f;2go8`lemEDPFC2v}c~W1v7!mc`PvTGg0Gl>oa_C{E;Ou~kST73PWy z!FUcSCUSF%D<*C{KZ~T(h5PbBdLp-w_Qa;8XRv)2(w5C!ej1yr4BrScmn*rMS#7Az z3dd^5@|uSkMzV4#pD$wgE|r1^`(kc-K94}sz_6=i?JZOgiA9D5Z9-N}YA?^9uSj#H zN5qIi5;E!zvdUu+MzfgQ%LQ2urWb-6yAJH{+`oT+u#nH;1YkaQ1pSv0bUI%KFnAEazY72>!0>Ds6^xu|1LAM&o3zI>6NMkFyE!ZFI-W`O$ zYx+Ba{T;#MwQNCDXo+sTJvd;5h?SbEDPs)uKZEpyAU`esRXqWys*V7NbtM71t{{L2 zXY1c@1&9rQPc;0|SgP@lz#AjYxPC3@vqNk<)sA(I{={@d_jG*7Z?7Km;KoIxV44;U zx+T%$5F=Qm1rbekKgR{A>f(r?Ztq6J8zJ5iABNai#_Ookqli?hzS0ODj7Bi*Yc+su zD)JSqQ@8xrM1ERrF@iG-I9$q`k^!j5#9WNnj^Qicey}r`DHO2`<=YSJ@2H-eklEVW zYl^y$W1>WRRGsaWi4OE_M{wsbOY>=L6n;NH3>5V*dZ!~!ddvXHE=)`#1{7m6p_c*K zqp)RQ1F2nCjARb8S~~yO{CI^u3BPIw+#u40%4`YeGTWWPRn~uo(CjVh^Ed@8${K%3uK0RM% z>(LzyvLQJtTrs@oaVO9W*?q7jmX%L$O^$lOk^LgXKD3_gCjfu!yD-P!J%@q|bNiST z{C)7q;qIQp%$kMrK3QA&Nlj3_(0bkqZ<(}inKW;?g=l)AakYg~K>k|qsVUVFls(Sz z+rIDEj<7nUwqE#5O3|vFo0xB-JSWp%Ob6{ob%)avUk2@@?c!q<=~lyy=7*jcJ)m}s zxqz-br!mUsaMJQ9qo_bGm@SprsI$RFmy7u+wxRrePR#>@sP){Md8PuQ#43btrZmTP zJjaH(*mZw?t`sOINP-G#z^l0FowZB^s}8bH757o5faph!oV6-)%PqHbn*V6^i2V># z4Le>*#_Mt6cqL^_8dsX}N-|!LYgu!?0spE}I_Q0($GbttkYa%&>jy3s+QMB9m(=Ty zZ9Q#?*w(6$d}G`^tp0yOBMy?w_aCt>#z2E%l&2B63%;jM-T zlCt!*J-n3wSGe>>ev|}rW=kC8s9+E)EdRSG;3O+=aKCs)_&w7l;!lLuO%SRxbo;M6 z^2K?FoQY}gE*|+B!B^5DG&VRYawV=C4l@p6ezr7zK|IURkzTyriK04dDzUA)Ggq9K z6Q2W}!GT+D+TW>K1SedeX(D7q>QBt&r%IeV6hrA{PXfFAWkUCO*f?okJ*@M&@%sq= z3r0yf&Eh6XL`hDx%#KsJ=h_{Y6F~Zjle*wkJ~wL|Sna^4%QJZ3`H3_q3gY4t|106| z=4L(X=T>s(*(TBQVqR_R#Mx9QZEV`~VM-^57_za<1&`@D)WrsjQ7IH*3lHJ`%k5V_ zQbgBOz*}wm&`u}^S={TgfbRFnnGc(e2@aW-N`2TMC zP5ryKJ2C&a{LcTJojz8xn$M^9c!RW4Bu5_r705KQYa9>mCFq#Xh#l*1(9vz&M zo@wtmw>VfR1nR7Xc$Jf-jL`zegVnM=pBj7(GcitsH> zvVV)oMqB5{={1VjlmwNFrp7At-Ia@-ffC3@5l3u86y-HE>0)=~qBQVIEvhIaPS(cF z*0QRFv3O5ZyXy6`h2p`U;M|FPWq6@n$*UMlrhJ?*fgr{jdwk$j{|LuR!DPiWb19nL zlNBrJC`T_x4$;o>`^rRr+S}XpFCCMW?ukn$F&upaoTR3X;I`X>gFPL=uTlYJr%IP> zk?psA`!Jx$J5joX6@aD8PWP5ww*$aAff`(m^S>3HgB|hYlUO>EIFA7!@L5 zzch_%}M$`Wx2>8o(A*iak!j~#a`*9|?UDdtN-Vtkfb>j`P^aNP-8hT{OhMM73c1PCcnz*i2;P^1vKy5gN-Aw!5NKGie z#z1X2^1+oNwMeC009k&nYxU8g&Tf`c*0lopCW?*zRc3nn@e+4?Lf7NfsT}`{Eq}tu z<7Mf7!qLa$LGa&9(kDEo`|%P=6MOrFU%r=hnwS{*X_c z92L{Wf0AWS@{gM@;9qrPu2qCdg`!C|BHg4wGi@-~bfn}Bi6_~P z_iF2sSUOo3_oh3f`#D9#snG5WBGf374mCy<&2n-XRU*Ma<;{t2GcM<}#Bw=4mKudp zXTykgORLI@q!5Rxs1>3$4*PWg5DV%O;KK2nFu z`_AUi?~6p)m(82=>3t`rOXqXbV?*i~rMyoB+83(?<3&=|g%T=2W3-i#7_Jp5I5%O> znd~3uY=g`>wsA@2^jf*FHBF>Z=WxF!W#G=mT7=Zyv--43QuKQEO&8A3u_f6jh}{<| zGt)cNUzQgABAJLfjB49{l+>bN&#rsAcFlC{ni$!2^60M9M|TZ(@0z@K4_gWj`Y#pA z#HBG;(m~NEq0O4xouvtV;!djGW6sh4(n%vofeLZj-YpFmkhSKQ+jR;a8F-I{s!L0V zJsB5IWGc_XQQ4Q9EvPY(W(k&R(LO!w`xdiTO^xdCf)44&(s`BnmNU&H1q{`iRr7<0 z`Bc7`msCsHHXxbuOwDOn*!?J|*R0QF+y|14YrN0cgBkakNls=yk_@s@?uIc^5{{P% zCtL}~ONPVMyeoBfqt2mBE=16Er02lF!-sDQt2M_Jlgi6}!SC4{Nut?QBLYu=KY&NGWh8BFd@uEqb3mTbffzm*$Ma9-eec&Y+m9+%)4_ zWE}O5-HZYL>YzP>(Ld>v6jm!q%#1`-DU#(1_8_>BgM}Qtge_5Gc$QM_EOnRwMU?Hi zObDgE;n|xVK0Rh^Q=us31EOxzI z9^-r|&(dtpQr0f|{o;YPtK=59UG0P>{mP`*`l$K*+(L%!^~HQl+ay)~CgbN8vv zE-A%gFr_KWva59gzJ~nH@Eq|rM!1y{Nz!8kQbYq(b5>^8hZL&xZe8f7y%oyt7N$8L z&6iu31t~P3>No#U-o}aiWNv<1^rgD18m`t=>kVbECNw?fpYm3#ZWTGP?v?p+>vF|v zM|YjsqSMvE9agB>RhCx6{>@&inR2KlgB5RmBWzb@H<)rdEZ+R|1#fNdG-1aYE&ch= zpY(^5-kR#Fu0ctTdCilguIoP1iodF7d)+QFEs!3`XFWxw{;t>6qC?F+|Cz3q4#R!R z=VtmZN=~Gm)>rSa+@x3yqoZXp0W7|1D!bXWs_?AR^(XWolIf9?iEK#cpY*)>V#uXgaoJs$%V>xe@rlv;kMtVZ`Hd9_z_qTpf|_I| zCI-NXm@J1(e(`&~Cd;9xb>$TrWLWe>Jfv#j(c=$z&2#w~ZbWEFR$ZY*S%p_EjHii} zJhyobxrvMT$gQgtm--`~;771#HtmkLNl^;`Q5QLgr*Q#0$*R5qZU$MMK71@B?5rQge%qE!E$MGM*1*&;= zKTN50s*?KlYFOh$EG?BxED2IvMBdi*ibMU0r5q@bA{HT?e~Y(FE_38tHz)@6P48L3 zh^RG%kKQTLp4N?uw!Ype+Sn8@H2#pcQc+9vheMKy)>g%&y>3SP17;(BBss8V9Brk-Cm2C)W}}2)UZ?>KGf#IpV8!Gv(Iw6*!0T>Wx>_5vxNs zx*K%JIV`cb1*4xB(iD3M6eqt)G`CC7|b*HMP{$OSzD#a>l z!^RlHc#&W9HsiR>EsQC{YAP#-we7k>qaIbeSBa8*Ms-;y3{&COKM)SOwhdrvIU3Du`Cb(M1niQ$hF5DRf z^k|iBSB&aAXo^u!H76~E`qBR-&sVJM!k{^JD30~VS#dNryP9u{;uI_L$Go*w+>O9{ z75nCi}7_DZdJ#iYLdkb`B7toBk7zt!-Gm-xe;{ABkxh2mtXby6{|Z|-2M zw>tx1Mr>EYBl?}K+lq;~T%kC|ZD=-Ity79){q?C&5cQ@k;@7Om^{@V@x4L{`zQUn( z(a70)K{2eqJPjLMEl`J31Ezo_dPM4SDUJ3*%kW7lj^thuXw96RlAUDBJ=lns|&K? z6PPFz)$MH)P+BEF_l3WvpOW-en+mK8^z*I3CkCWM;fSF^+K~ySHOp_k`lmE18?EPN z5Ux!MdR{R%Jt(2QWm!*tGW|E|t<>9$pD)|9-fHV6{CpXo^IEmMv8~U~w}xjWy$y`G z-H;5ZaZmxTYwPlBRjq3ocw`B#wt>1fsUGQVsoAxS>C6UwZfl;QXwx`$i>8w)I;5ss^GeHJh)W6XV}XdMnhr^mEk@;>*j_oevWz+ERDnIeqzVxg49-sQyN8 z`8;Mzb}kZEUY2bTa9~0LrdfNTiw^ zZ|kzEw*D;_K;nMF&v#gM6T;VdAbOt|M#)#2pGQ@X`|Vx}OA7BaFdWrmvdaE$FvEeC zW;FL)xfk?6r+?q^*2UZSa~{_J7LD}lD2hG;xN;ZOC`$S_lHSH8Vgc7|0o0H!=$L~( zxAOE)GKT(sZ(R&wjD&~srP7=}K55GO5#||FlFqTbED=7qHkkr%h0&_9sd;qJ2qnJ_ zOq}2sJQm+L6fw{C+L+@JRdQ1VA}7>sO?f;>9n|n6EcI{cF>k$IA!X&d?kcP|q8$cZ z!?LV$BDBX)mI@+onNc9xJjZLG^c#szwgFT?u{R+ohqp0H|$h^+!W9a$m=`oI)^*a@&_k2}$oT4y8NKQue zOHgC_u62w^&(QU{Sf>z0bPTnLaxf(Hf$`2)=pVIE;>Zy}U{nl_teQ>PX7!sv=_L3+ zexHg~$cei_l!c`!+d_$`1_cHi7gD_;RH!UiR57qka^}g?Lj0@|)1PHYt<$+gy|=QR zC=(2pw16=gB;vF>(gP!qPX(}!b5 zJ^ISP>KW6dei8dB_fp4oZyMukL9wy&npmz<(o4S`O#7o+GvY`^nPqx3S6HM$nLiAW zg~ujJGi z$nmz;_F#2fE9}SErd;=<2qV44uq*o!Pfl22e^f~Dn3^li&uR>vVbP?1QN^*b(A~K7 zSm^!>ySe^Y=#C4KmKqDmB4D!E{=MQ@NNNCc(Bg(;an9Qivr5;!MqI->deLi9uHtp` zzTI1nt(cQc4RHXLI>Zl|_k+BUwKVA`daJ67Bt9b{f$BI*EJ_eWxY+|wd|W~H--00 z^tdYMe{H4#t-tElNV4%HSV4N<*o^i+R%V;bFQ?1tJzjI{z(5=cq9Ntnl#V<%Wi6O) z^)?W@kB9!vaa|xr)sS{fxp+f57)vg&9j^M_)ZnahFH3W0(gjQ7r8=uZ;bw(hqg=lg!WS zNq(x62dL+8{bUbZM3ufLr!n*8MR|Z*z2)M=lBlSBvSlZeS^L8?$>4p2w}M-<^K(TL z;<}c|1!+h^q2pV3c#Zh6)Xeg`TrI!#lR8dc)&e5#<%MiW<9B)fL?JiD%`p-t#;k^E zH)Ksza$2jNE0d=V#=O{dP} zuuQ8mUxt^=e5rp{S(jsN+6zo|!b&VRMW?-757U~Qmq%YI=jA&1E>3qjvqZYfbug_- zb{SbR+2uNTE=qJcht@{BYn@VIGF=YTJHWI?H$tNlv;ZJyL>m-S^ z?ohs2{WOh5$sskBtXY&Hf*Tu8&Dg1!*IzV5+mwT4bvKrnalAS)W7qYtiDzXD*Uie< zeLd_JCu5A(PsY$`DA(B5Iw=?!^(X~Hr=rw1ebRB_k=cG@H8KepWX`b$s{19fAm&Xb zM*TQq+*z`$*peu6Sts)n+$1BIzIX8nldfxG^En6v2BNOi#>i~6y!Xnw&V5IH6B{Qx4x7s6XsR7 z#)uKBfMa8JCBF`bKZ-Q@xj6!t)?qa9GWDbJ71JZ0zj#&kiJEACSUy4_;iCGJs;2!} zOO$@D7PYyk9qVe48npF_vqsex!hd3Hf4<*@c!Hpd!bNU&_x#t#W?9TnRueQM`$+w`q#V7{XvdH!V7yG4@{Zd3qtVr zXEwq7>ch|eGB!S422Un*Q+B2OwO4*jpY&R47iP|p8cYJDHeyRfQdR6EYjD%O-j*x5 zu3}d8)nt1CQFp`&{i9yo(o^2nEBdBn2sf3Bxmk|Z$j{W`EpZSk%c8!fJ8THJl50uR z@Akaqr*-s4*38KqDOw9^0bV+sqS||0VP*|Q&8<^CWU-sSG`c!MtXHM#2yxAsfHJ)C z%)w~Ri#52mc&mie?3@=%ag{kQ$2dR3oNxS3%y~4~D13N3u?Z>MteFS)Vxrh6gBaN; zeR1!KHcHb(HtKF~lfASm&vwgUZNyJs;ky#QjfOa@X(PU|B~vc&)ds;Z=X1GBR_En@ z9#_NpMJqhV$K0Kpoz5+AiG8+J-@B`SAyWLR+`@_6>`4a(7LYjGptp+ajp+|U znG0O?w+BPTJ}`U9I+@BVjb@EsQHuTO6GDRDCAnoK$LM?lBD~R6KZG{{NSN$N%kUB&Ly_WZykdZC%r@W1@J#R1N#rs3Yzq;%0w}*;0 z{I$4W)|KiM`ds9pB7HE^qu2yI*WmKvm3(lj9f_$r}a^96X_A;K#uW@N?}^1D}|1PspZwYObWg{!d)!0PYrG`=3TPap3YL8xI zn)%Vno_=2g9aXE1N@@NtW&v?nX(kq(7*BOrseYEORp}0oB7-)ZOk?UIxHM^1&l28% z&fvBA=2 zg)~|^PJ8Pj8Dgu^)mP)S5naR%XG2>>n{=a3I>H?*wnvcIL@m-KEhv&a4>`G0ZSi=*j^9XH0L{_?!YdS?}lzrkxD99X8WwUcQe z@$*cm%w&04kllxqvyZD>Njt$x|4{?kZHqnT|ArGkK1;O`yS7=XUFR$H!_u=fRAjw1 zlnH-DL(@w-p7IIA$;*dIt)Qw|rL(7&oK<@MR?RBB*l8Y<8;jkfpFR~r~Kb!p@uaiV+-%$C_y#BOQ+N~`9q|>XlQ1a)Z^jYix z9<|tQv(dMpY`pC#>N8WB3N=G}r`wopak8;jtkUmv2nW}kHHqK1Afbg=Ho}rQ3UzV~ zwC_?sOoP?D##{q6?Q0^FQ4n2E^!g_()rkToh`vOcfXs4Uy;_te5ciGiblZY`utyp4 z@WGxXl|Y>2>w5xB+y6RwsJunjqG~Udl^ayaLe8WLJlw z6#8n74WdLlIFYCk%obY|?#WJ*1{7KwwLmrZT;~oc#z1cU&e|Q3*kYcs5@lGjJ>vD* z)mW_Wd>JX_H7m8^>tA9APw98tB=2gXpK0Nadm*Z$VqQqJsOyPs7xn!_A$cQDXGCAw zqHDzT|8%^SRoX3%fjQ|=V@?3bfOux%ildx~RXz1j>@2@t;{?XszdmvQDtt*zUX@!= z1G{R82d*raVfCzK*0XfS7R09kV)Lg)kz*wdNtVCtRzl}fd*iZMv7-3w6V!$#CWH=i z$v&sg{7Mx>nW7myrf&V}MR!&7^9Xl=)30470YU9J{%kuxq`#PxFCDN-vt2R%TC@30 zZ4Nn5E0o8I7;x2fjoF4mc@RiUt7=x5wPd5V(yH)BUXiQW&P6?Bw{skadFM;=ES=nr zRQ|h6(I)v#w{knF$i>RR9)lqZ`7$s(^`|3BMuHe_fqbBo_#I)v| zY`0BS`eCT0_7!Sl4iit2XgRiWq>qB?=E%cOD?n)ECtaNNd_G6VW5i?eYV^}hOD;@` zz*g3x=@qDWN@PZMx#>S>5It1|{$1Xh{B&N{{MgQ37L-*zSB+GxHk6^hT!Y$f^SlWT zfe3hU;|0x4wXbA8hre8__8#!oOYbcfU2CyVVZUxxC|t&Cj+D7<-fgQNCq0IO+ip}d zK=ogl1eO&sbuX(eYx5HQ#Lb$6J(h`{B~@JPj3}Eo+v@i=T#g)yb1`#9$r771W%F!X zeJ;mkc7gN^ zH2b+vpGv91(~jvPN@%9OA4gXc|CcSN)ND7nn{oisnU<}R@W5z>%3){ zh5Ti;$gj+5AUZJ;k7Ma*q3R@ab3VPpYYOL%EX}6R<&cd9%Jkw+&x)fhaO-(Dp znx?!}c#*L!jeo1>$_Hvq_@{nSVKqrF=vhS+Pm$Lf0GlGe_tnj!*b=nkotC>Kl6ix- z`Shd5dWTM*&J6aA-8H~}jSdc<>F*slKG5ISlJK{9t+fc*{u2YkBmLQyq`%E;t1TTJ z49lH*)S14%v4N5P(=91ZL^st^+tQvEx1Ma-{?kKu_iOrP#78zPrkC&Dw=~q_Jvum+ z=~KnaaSGG*Nymmq&-7=(8p__&(#SfseKB6&*s;;$$NRJWrX@`TzdCEthWke%w9zwt znGxh3q}%S!oNDp>&ECdE6r*(fU|&mf-EQk08MqrBEi3#@-uhb7vst>dWo5k{QAIs1 zt8kUK)+HMo?(ZF?EiJ2Y7PnmyrKhE(ZjX$Poa`SQ8R(TZjt-+4Yk1LNXDzEtw)bTJ z@JKeJaK=W4#s>OY{CaX(EuaHW-PN+z-{}R5n5mIenxL`P)x(PEi^f`yo4=!$SEv{< zb{pz-#_;H|;oj`Pv3|{LW4$VwKGj&Q^{^tG%&AQF^l;0jdMX*`W$@bS4nXf+gF|Oe z_4l3VA5(ZOo9ngnQKS9Yd&V-NNd|AYWefZCj@k~%)bblUcF)*g=5&9{)_PN3shIAP zZS`iYRzZz$L%nut21XAxUQn+ARd_ubaeKX~p6$=bBt?1Bn8S6o?K?9tc%nrTUlvX7 z6U^G-{^8+)p~11?5t(mt=k|tL27-2WD0^3bmN|WVsAZBX_0EbZNOxYp|~-LAfthP{7oLqJiKT&<}zxVDCRD=SNHeKgdDB}{c(eT3Nb zq~n(4(|f8vlVzC=)hBAPmM7akk{vj40(QgnhPyhcpz zuPnw9^Err+dTFgKAG!;<>XXWXEyg6ZMETiFc2LS)&e(RNVYRHq^BNtLl~dHbKbsxO zqK!-84jvykF>0oS#%(YQZ7ts+J9RRr4AL?#QJ^9X!=RJ!2;g+3?V4wzt-hX^*oO!;X+P#?__Sepa^O5&C)T z)agevR`rpF&xi^eBsDFf0Wp%qI1R$dy&5M$TIL_{x~{}2GdePKS`YBhu{-;FM{rN; zlY|m3CRx7suU*fJ@<)u1sLb8SJ!fh=Q7?M&Qrm&GC4)oCUejx}CC7$_upoozc5R9H zvC3?)rSWNcRL}8KLzxk}WTUsX>i52((PO8?FRv}YAC&pA++XjlsiKhYNaBF&JwjAL z|EbKGVH|H&++RfHy;D||tKo^aoG%bFIx2vj) z)OKP}X%=7trY;tyGbtvjZsEWnd!d28F*EB>K@4AAOzV;j4vmZ*9~wp8)HUIXA9xSb7^k5w zU*p4#4rcDo44le{iY^lqRTo)QZfI0yFI%a)7`+)Z4>ufjUaH2B?(Tu1Q)=i?at1d1 z-6&|U^zrFOXL@__MsdCdPO(R8Y1+%l-HLcFx=aw;J3?A(2-qW67x&U9Vk#LIno(T_ zgRC;6x?dO&7S(E$vl*NKZ0PZ!?AVzBrkL8sSL5LM^$nlQ5KWMU4-cZccwi6~qOiip z)TAs%HJTo~23eD<3wlrS6<}pEuuF@J=>EpVM8`9ur$#7h>iN(z35{jc&7KpSSMfP0 z@-&-=qAw!(^#d+340B-LR{u1t3Q>;1&v`52ADHp$XP(7TYz6f*f2uL1h7qq}pm?!# zU*4}TQW*8IjO#~PV`#$l(db(2lQK)l9Y?Y3mpUR*yNmrvf12~>nv4Oz{u|k~XtstW zE^^l})oBPh+YWuwe#LfF9mX#)z_mx`G^psUS#%x2um3FfB0Srq`f5=p*=m3C*zSj% z9rqsVK0S20`|R27)0d4VcjLZuj6Pl|>n}6i`OnWvdCUD}iNv#;lJc?o6@Ds_@Z~SZ zPdSM#<^%R_;*sjB_PhL)`7@R9Qw@oB`8AdFQ_B+_-fL2x(~$7pmc*`P+arTB9Mti- zcKu>>kU29TUkoHrqo)>Mv{d&<%fSU6RmjWNI@L$&Omcj;*YgR74>8Z>yGTZv7xU8# zd5KC-=lKJ%NK2ldo8@~_^7Xs?Wgg0tPZ;X&=Y?1vALX2%(O<6P zE79^*Q9gsHzOg6Y^bE`76R-OE`UIb>oR|RX9QTXlfuD(U`XlWgUkT;=gR!5Byx-M< z@RB!5)djvltG;6!lqm>wI=*~uo*L(Q*n*F+)2I?3i|1>X>Z^U@<8(rig9Dm5m*=Sq z)t|hAK_^ZGTsf2PE_w_a!`0HbkxwwX4GD)~T;pWC72YxdC|c97#zLsT8Wwm{mIB(x zfu|w^(RVlqQ)Jv6Ncj!Eg&_0S?zRwQ{sBIRrN}5+>)dRL9LUx-tgw)jt=wuM$kw(W zF=PPiLLo)n)`hjDAX^vKmTCYxaFbz@61tnM$gZDS@0>A3E=<-pY_Jea)^F^#5M=9b z8T7uv7a8f4WqxY2b7$L^9A0%+D%&;9{|4jq^&FpUSIIEJB#AQmv$q^$@sTaQh0E!t z{w|_?d(mX0qKhg8rb|=VESI2NzR-G3m+AWe=K?M)XyzU2L&TzMq;gD6^KsB~mN3km zITzxbLn?ET;GCLqqU#HMiTAwDiLV%4;$x1(N+WA#9^V?(U)Am6+p=0rK6fmtb}MsZ zA+WPy(#&k;*Q9*lcD}*~AWJh!>9m}Ip;Nse<FB;lxx%zTM@CdY@2)ps<{=P@}mg|gJZH#>D- zs_~W&dIs3r+3rBEw_so_#<2$6_Y)f-A3`PVobMp3FAoZ>e6>@?Mc;IhU!;$?&OD8z z6%^#$QfpDSlCS!Ul6-#GeE%5(RLpVl`I0mMbqz;>CYpv_!s?60*Le~2oP6VVPCkJx z9~Bjerm)?}Kg*+`h0CIaL4Iad)&lhdy(SO+sJddp8MNRbww-OQy?gOXPd$YVoe4PjqCJKCw7sqt)DD71hdv@m4kn$;IC9 z4~3P6>~Gp)gmn;h{|y1lsKWx{9!%VvlqJ=bCLL^SQ(cynCH0VVgCYTd-66vc2*g88 zZMF<{hqi7rWE35Cwwog9vcn;n@IP#%NIHDOHgy;&WPp+mJ$A(OGyr5rLNXEli0Ns{ zAUi@&YchbF93Hn|fRc_Zn>RUDELmW0+Pv4631*k*hT}TEdo*!ew^^7UZERKS9BK2d zMoj>a;#MOC5Qw)LH35R%twv28;c=U(7XY%`LNaOdZB6Ty6qG@BTk9rM8*RSb(Hi8a zHs8+oex)fuklo(6$wH9bzGbT+1GvN4Wr|YNc1Kv7wE2!E9s*ZP#BsP|$4*0r0TaYbC~|{>S`4CCw#*tUvRJPSB7aEck>f1)Q~l2Fws(c*D>OIOJGNG$AXhCV z^NSTj`QXeCKHWDf>u7gq&#)u%5oyX)VuoI<9xKZl*`>xoXW25J|ISa~wNGGqinH?= z^7HZya-ec~o@s~2sxu6_OUTB3S_-1o2f`6=-vKsh?djd{w6&^D5JgoJ9jCy z^jAe4TzI!_HC@qC(!fCWn=9()~3>ql_U^i&k0f9JZqyU25ppn8ADTa(4 z0buNhoK8=U8YuuEyEh~gDeg5=PzKq( zX0kR&7d|GeP3GWZoMzSY6hZcw6?)MFg6uJ>HpgWf{VR!IPipB!S$~C3+lvA$7wN|w zZIb}tH|DHX{D43lYguO@_>FCB^kr_9!cC(R0Oy|Tly*Mp*FqU~&y8!^ymJfxLDUx5O_Iof9W0?>1;@~M2zL+&fP{x5T{&iewRC9ET>#;nt>pj;-ErIGZ17`oAt7= zTx{iqP{`$wT?mDgVZ0CuDK_UqC}g9bD!7LY$41BsuI=(h$O>Ea>bl%%gsjkW$Ozfw zrylESW53*Kg6y%bp2t9tJ$7@aVc7)PWA_~}WS*b8&ply^Jjm{I)y|yOc#z$AQy4rw!F)8V)^LpHtE*4+%*^j02_P-KLGWVE!{8vLZ+p3p(AjoFh_nNk~_^HRaU52a$vd6hrF9>?4 z9=FLV90dFziHbhKXrddwh6NPL$Xbf-5-)|g6#f~tj$k7K5SbXWRDNWtqrorhqJp4 zvd4#Q+w7-))xFgWDWMD?t0kKu`_(PhU~Go$S8qOQgxun%p5WeL$hJWCgs`?PkUe3m zikUH{TOfPFtz2hT({!t!dZIgE$hJcE#Lz}>h3tturK1hrR>+=s{G=h<=BIwmwUc8T zWWVOx1!x;&zqVd)UY0xCAp5nnsG6)|H~8+8l9=U{%7)&+Vhy<7g6xN$l*BNbQhc%} zC9%w!H1ORgCtY6PQ=|c;fSWA{>64RNx-AIllamKzs!6TeefKFz_lPOn4k_Rk3qtyo z0=>H`|eYdSaw~Qh7@qE1tEQE65DP{A$@8R9$EMO%~Zfb@anMzN$)>rUT&S`tg43wJ^a*k(aUpO(bNn^H)hmfR=HfspR< z-KQtreWq|1q=4cL$fC6i(x)dkwObI7|YEC}f{k{fML z?1uCi$=w}>bdT>oGs*PVqp;Rpwb6$@!SB}8*5yIeSX9jM@2x(m&h?-$QSqhHp8);me1Octutp@iMNp0;Azr}Z7nRIp=Ts#8-t$J|u4TyKF`L6NZ zS4CriJ0PI#-!)eD`$` zF3Yxn7Izc4uZwWoeE0Q9M;StTkhNMsJ1^S6eSH#lNz1$0cYiM$Hx_6C?YM0Q_xGZ4 z+v2;wAN2#PvVgW9wt)NlQ9o?;-G`FS0n=wJ#RA$c+zRePNgNF=!#3Z2L(~}7UjeN> z+Xn6%lK9semlgPpk$^XV3ux`m4dA}ftYEa9CHGBHKLp?c+I|SYeN)sA+kN-Vk$~I5 z1+;D34(^+ywxxadEs_0Ag9~VFR2tm3B-#FHd3X5kTO$Ew*#uOJ5(4f3_pQlIb{?~A zzAch>C%AxiEOvtXHnUNFQWj1hh-SE^yzT+-Y^4h4CFxV|IfJXwAiLaNm*K zt~?z0vJ$>C;=2c2K+AUzxbM__9k3ff{g95Qy z05*y?2ng7_MT`|2Wwwy9w9zgzE<^S0K0oYOa9be!y}C zgzE=1S0SbFgJyIEf)r3U6Ht{lY3BtH(hur|&T*Ulh7TtbA2kBWvhiWtdCelwpIU){ z;0mY(0tEL@tw4Zq{ZlJYvk3GND-aN*fLb6xRcZtRg!Ch3<#n6IZ+JMWRTg+aODc>X zP7;fj5d;^~hi&Js@EiU-nfOc77FpK*+z#0aY0F<&u0U`F)NKI-_b+T)0O9%u&sJQ}y`xPq?AY8v<1!@t2zG~+K5Tt-w zsR31~nGb-Fe$~#07MTx!8`UcF0nm~PJ42-mOKwycpVziyUl7)e_IEvdBS>&Zs7x=AUdU)SC* z5lFw`8_C4K7=gs!`i8ZBz6kVBRv;j_0&1>+;Qo^p2oSFSWCij?pntZb4FoBmB^81G z*^V})kp8nB?X`Zxw~~qfZ3Ggh?pt;~tQCR2Z3O~?E1>2I2=2G7K!9-lwiRft2=pB* z5D=t*mQ)1#junVfNWWtRTIV->H<|dM5lB48?u&sJQ}y`#mcV zAY8v^1zINpecuWM1Sy~;6@k8Q1)>zv?`wgiR@CnYNk^}dKi%sdU3nY&TE}=}&Aote0;1scj1o z-2kY&0T9xk+HL@Z^ryBP)(iCS#?L`~4tNE$q|y!lp7it{pHhX)F{y4VJK_I`YL%lQ zKr66pjQ_*-;ChMM1DYP(;5R(QN&LF$LD?!i#o-)DxNeXhe5#{2d;-B0P;&(Y_o4{EkUrq3wNOau!KXReofZgEKuao&pXT7U zi8fFQ>C>DxD=^U09qnCHD2fDVNu>v$?r=P%Nu>v$?y!%4u^hi{!0Ncqa-4N-&qxNn zWZ!YRTF6azmF2+;Nkk0PlemF8O6?8?>Z7fu!_Jmn0*HGgp;k=mqmd zm3gKz6lY0;BvAzBWHNDdt9W%mL5|Nbq~6mQ)7wxppuqh4i_0FgMCzKFXp4T242aen7H@w72ywd1_?9*T3G^#+2+bVkSQmY3*R062G6%gE) zT0H=S>r1U3w90V3%+bLj7>OPLY7YZYm6}ZwAfzvI*d+ZAnO?1;2(NIQblVdG9#4p6 z4}?=~iAi-|K0h0qQ7UPX%s^&Mxzh_~@}vcgmjhl9&QF&+ZJI(bT`1-OC0{ps>PfN` zWKJOrq9ctJT^jZZr%A_^DMj>GIDzco9e0!8@M8c%GZ2FJEK(5(ut< zx-S92eU0r)K)Alf_T?t2^0y*V=}SN@5QVDL>|OvN{Vh%ExNUyJ?>dRs8-XNP^1BXs zOk%OyM4;DNu0U`F)B*v5`&ugyAY5N-1!@z4Ugzj&1B{Z)AVA$=fN*-9L)MJ+AD}|! z29yEe^g1UkhsKV(*>CuLC-Fujp9E-r-)YqA)Mkt$bTVzPDKU zfZz(K7zMZR}h`GDXGsO19$_nlTgK)Als%C}YIdzY0D2>AfDe1LFzmz57tAshJs z;q)#m-!{MDJx=0%Mm`CCy~i#x+eE(iTKRzB3aI4+1oypGK0vs>*UGm|160UH{{Z3ihoXO#hqZYKfCV_2v(UT)NotF10iFg>QHO=h^HrE|4_ zq(4&4f9aQQn<1Z`PP*flPB3>aFKoXpN59{pZZXWz|Lt4=%FC6Dv@TDFbB?V05lJ++ z*9v9}xY6^oa?zl8X)ZU*ttTaMmngPm#_Q&3*_dadAzW@`%5j;YFo*jsiHNsWU7C>0 z!~lV$$?9uDLAy-N*!Sl3bd(;0P1u9>SCid8S(xJTPKQxobWE)022q1gIXdv} z0%4~B^*Rm+?x!4bRArq6#7=$6A${*27V8^iXg`C+YrCsTNpDxvzH2g9n3ju0`{WNb zxx)#u&&&p8J?MaZ*r~?Bfi=z|ymhn@P*t@~uh)Rc@fop~zazfc4Wd+^cbv|)2b0nV zVo@&TD&rTrrZIJFSITlHtM;~6(9X;pe$=VpMo})hhym?aDgJU{f&Sv&jLdlq7ne#V z^0&&}CTlbir78maG&R%sWM+1SrVCJ7dkOJ!1Fd(E~a5kvvD5VOS>6mAR1%&yW#8;SM zXM>S^ZZ4P*D=pM-x@7sT%`xNGSy8C3|WFf7`4=QE~WO}9m!t?K~3l7L2VAHtH ztg4{!YfQ5AYEy+~8}btGfO5-AwEb&#)@@hX{xz9(T3nR<>yEP-aaA91*H6rxy=}xT zB&D|KW{JxnMJulC7$CD%h`N2&~V&?`9<~!v6zaPM z>S|<{3#qL}RP!Gkvk%y=RP!IiW6G#%m=OQyI6?ZLicLt=0Ij-DE?-J+co6AJJ3pKF z1=lL3@|6zTEQu7s!`3JJ(SLNBbe7L{wIBUQhxCP8RDIxngAIB5#_-keTx-a&oWJG>ygdtHyjhl-mW&U-;hv@>>tv8!@oF*?->6w4d%Z{M2gQR zxM|U~Z#vH2bf;M*0$y9e{V7aiaKfbZGQN?lN>f4m9k(AaZG{#YAlBX}Ae_JHG^!(0 z;mn|Y(`oPUdQ?ReehZVB-YK^AJf?{cNr*3X>4IGKvR{&@u0YffB3pr|;kTSdZIaST z4ZkHONpvJFHuu|@By?n4^h{S%rwL;EtE&-Jgw$3es`zbLk7SL3FYWlYcoW|iQ?kQv z_<@u7cTCAwllXL@;<3z{3x7u2xyr3EZJNh4AkXR8C@03Qg%e&j+^7F z8t7MPMjQc*S$OSBgpDpU+Uu~YjxM5NFy)R_iEFL0_+x*#l8%pJzF{HW7x6T{%_TSN z<}pF@bCWr-PRd`|gZ<3Nl0rL23F0fH3Jl8U zqNfj}^zOk0qPGBbZvle)K#IF$qEvwN)&nW_0Pm1Jz%F9Gsl?0at>1O7VTQNMo%PPp zVeAa1=1TLk9YJNDD@FPuEIzqhlb0Tw)$ajh&-5}&)!eK4?u5G1uXfPePY4^9`zh|j ztIO?jcQx`IW!s@=+XPqJ$18H9Uo3!eIST5b3x?^^B~^=c{(JdiNx3_k^z!v5F~Q>( zU@bR!*^YucG08HPGS4sEDrE4>Hg=ZFcYs%1~g%x zVudY&tFa0re5&lU-s+6)$Dt8eaR$H|2$dMn{PZL-N$!#2$?~*)5qGgtuDUzg#dkO9 zS9u?NE-0f1eM;vR^{*6#mK7S>2$8KoY}|uU8^x+UC~XuQw@aWG zrHn@p1Sy~;6&v@Wl(DRoDr93>cgeQm#dMFFTtJWlT2irbFOHfeHtxmJEL|(OFNsLS z#sOMVv2ibnNX5pzBqC)C^3s$!soLc;Li)0lNQi)fm^l7Q2RlOo*_OZKV3cNC9&ia7h3aEVy zKyY7~GG~2&aD8RUob~ONk$F|hL}r2LP(Zyv0>bH4DHE9mRLCZ>2neTFrSS9LE4FNp z-|*U0;`gv+4>^H+?l(}MmJI&alOALeJZ@f}$wiwOT`G?*P%^h57mSrX2=`_L2yivR zO^<}|*p+D~gDdn@picF0TbbLMNq`|MPIO`7XT`VxP3ar$Vq{nZa4VCLsnWOA|S4+?o8G#tnVc-0ZCEBPH@w^Oa@$I!K4mU+J_sD3z&Wr*GdfU^>W;Hzu}=&;>||e+QED%Wm4GMMZPy+5Y(gt zLOwt(A0W7INSWhdK)AlaPW5(??~N&wX8?pifcjJnP?eg~Q9wxF$jc_GSK7r4y(uCU z0|RJl6~=E$nLGn8TmA z4k~gd6hJt=BehE|eK_u3zu^y3iT9Zy+Y9C&q)dk7UXkzJcF2H`4^Ybo2=2S> zkO9K=-FC?KihS>}{SJgcfO^OPRjC;=KuF(XhitD5*?Uvw3={}bKuao&- zS)|Y^VSwu96R6Ywa?pNG{c)}%I6XQ%B2LqN`B~mwk+)aOF{eG{^se!kQwu2Gi%vC= zLyx(6@({2(VyWs_F)OwBE%MfsQPCUer4Ode(eaI<0)Jvv6o?)L)IACa?w?o{1*Av+ z#H#3xGEyH(IbBLcflyIEt)hSo>xWX!db0qikhiy6WH@~&b)(#|blgr3cT$PJG%DH& z=0{TIbg@(9d)TTd5b^lU7H87!5!@8i3$_(&{K6?f9hC(JmRPPo?y|85am01=KnUNRNCf)vT8~K!u#{ zP~!>+r%$Cii%sWAS6)lo_z{X(dt-2#63zn_jG z7bAjTT1NqC#h0y)0xGf1(%&s!{$Ja51PD?6V>I{q4c|;9 z{x|ylH_NnwXY)C;2-FEqnA29-J}IZtyg|gW!O2H=;Bn#e>JC>Vf4>dt7Py4~r2;;mQ$02i9Fa+h?B~CmP z3)5$n+O@mjp||Pr=IeLdG?&q$=EJdAHb27wo~pvUrlP}@G}k;kXO34Hxas_4h4!=ZPV;b> z{G+dtgJ#>OSfqy}+8R{%h1wO!wmdFvaQp4S{&SLtpgZ4eO*k5mlRD+ZOynnX^V6Kr z9@xKse=wtV_7Xtd$6WZP_4@XSx%!qhr$87EKy5ex!Tpvsr-00xZ&`D?PprzfQzoGZ zh#3f|%_$(9zMV1&MSu#~By<46>DwulP~^J%#CmneMs{nE7W*_Z(kTmdy#KybfjV+Me5{a#AN3>^1>-|(-g z#19SE17QBEw(pMH<2U>$mH4qK>Y?aIDV2fgxCi})pQaN3VTuk?^wX5d+d3$+{JU)- z5V8PjSpdQPciTchWchd7!b5(;likEqUA?D21m=@n;}0DYu1|6Gy+s!YqX(!r0)XH? z#ofF|tzdv~eTu72Oda>I-|#dy@h-#lFqlts&ED;>aDBRK_HIDP3aDiT1o!E#2}J?I z_35q&(;SwedWLJfEFee$Eve|hGu$TaWl;+0GhE|k9TpvUrfYH!foKb$&VmJm^qH=? z-3vfVBHLu8A=nmMQjJ*966Y=rTaPTmnM+T-VcwpMa1)*EK=0 z!!k0@b4~mJ2q6Hi5WMgL*lcT+K-cr!P1>mf8BWi0b=C}hPoV4ht_gGjArqk07727c z-!*|QN+Er|OQ36&3>(kzG`zrd-sz@vU&uQ=?h9Nt_cHZ)q>VgVmp4{)NAVDhfI7L6 zLU3Q`^5G>@3g;KP`kphS+{=EDoC#CNV=w|*Ws>WE54vk~cm!NXA9VF~YDl@J{UQ#` z4Ji-82xx1SJAN;6dF5T#>OuM`#%7s99)%Im_M+Uzd$GG|uPuf2#V#*? z=(ci+`z0<$&h5^)fuc?(g-7k&0iN|3Cv;xaDwUJ{!rZ7acr^c60zoakD)9{$RRlm}x3w4^?yuZ&1}G6pat<+k{%qH$dd zDWD~ldq%HvH`;OK*%-iZT)8~{YL~Zkw7@(ZBcLUf8$qvjH|oRf!wyf!0O~v6w3WN$ zuZaZa@fZOusobJ@O(ZbS#{eSm4|ui}TJDxVwfQJ8ABeS6Zdm zIGgI><}$yCBYh5Jo81w<3LNX9^ChCi<}tnCKEBsS1SsMqN8I)ydy^yDf0?M09LDp! zYq>6a>G-Pl$uU@^RN`35o}-oZ1r<_PITj<*O+cvnUHEPhrBf{~I)={Etww6;Ks;QU zoX7c;iX;n`X0jf!KJnn-i2PQXKk>F>i3w*1+&$5#Y^74JmINypi*3L&}>)0{Rw&^o{OXomYO?;ngBQ^8n3ZhbIr- z_IB7MBN^wM@KV1gPJtpjO^Tcx%)zykR7uweOM{|JGP-R1J>oaK z+fBUJIJ8H=e79>Z3LX)M_C3}w2Exw()O#F2aNi@iA}%0Y-{YEF`ZxIv?{gD>WVqf0 z=KEZ83-c!7`hM#s1Hlzga|HzV{jNFU0EFxNwVUj?H~S5L>?S^FxZVurA6r>(7Oo$# zT!G*UsJQ}y`vJ=p5UwAvTyOCkKIA6;)Ns88%n!NdZrClt^~07c5L^K@S3qz-Y`Fr$ z^~09yQNQ6~H}PkN>rpTtwmooExPH_+kwA*N<7QKyU@rTmixTnB@ux*N<7QxA_f!MI~zKd<%G zb?=aU(iiM53J6j_?NS4(Qged~5YjK$UDO@20sErW5g?IdbLMwJ*o-EDJ#ZH?!$erQC7?PrUP6BXa7g+SBIoY+# zC2Opt*>T=Y`n3v!n+Tr4ta&9KcY>92nm6h3iqzx1EY^56>Q9#BVl1J@Y&zk``BazU z9h@5wH-U{LK2?7ap~v%c-Mts`q>0jad0%FpckJg1_pAG}-7^!Me@)DlI2)HY@Hx2> z(ZoRuBWEi4M#l-Y-$ndy@#*?(X?9-q#5tAkKQ=>%EjLBq-Wd5Mu@iLKOyH37JozMt zes=%=wD%_9bzbGY@1FOk?Kp9^6UQha*%1Vs#F7&SLP#1KNtSITwxw*zQv%4EY>UW} zk>ofIbO<4Vgee1MN)J=oP||ZaP)K_?z%4_WDJ^a3?LbSIdRhpCoYK-V_5A+tyVmz@ z$#N3zJx_bj^IV?rSpU7oZw>Ey*E=m{1X~--iNuI-wN||3%sL?G9Ij$bd^>K)tF)ZH z6W2G0MGEw-0VMBt;)VpWe!deo4W!)4R#}k5O3Ft*QAz-02|t2lV~{>E93HP6tx-2XUtZY2*jm=^y36oKjD{az{D$ z`f|SgeQmbzByw-r=#txOv9qZ&S@k~gvXWM3CUq_VAD323Vj$gsQo%QFt2mms3clH> zn7}P8mbSNC81P1$Fs^-J_Q?3bVCLB3DRcf6ZZ?v&A0_0ovsmiFxTjKZ{*vPWJH;g6 zNX3_zWv5n@OC{=(K@ZIM=g%K8Hv^mwFkI09@YqREbo0lHkBsVJApI7+&r(APeq-lB zC0rUNs-e4@KQtueMq>iCLt|a~ZppZ-BU%T|UgqG_Vo%YVN8$f^Zh0-vdaYA+zHRxX zbc6Y`#|tnpg^pVOMc?K|IpJx8ede(;7YRC_$08XC9tQf_R&vjCLS|0&X7tlY@u6PU_*<#FKB07b>8;$RQLd1xow zCTnUFVe<8&$GpzAMu%zb!i~P+Pch4ER0kMKM9=XA7lt^ujpHVsad5M*^w3)F%&MLE z(CUG_-BAu={2o!pJIg^E0D|5E{p10X_s(*-^#CD-ca{TF)}yoKRplUy2T>I0q#9_c zfr|uE^i|~`i|^5)_3Cm^9)KtcjIFBTSC_-Bh*XNcx*U`TJwmU^I;>L)=;s}Y6n#xO zaFIaEoOT$p|Jt%gu`Bf$7YU3-bu7KMT=jb|sT6%}`81Cr)1w}~t{g4~AWjUROTvK^ zeI1Vv+Cd7W=7H_szXPf1o^p`C1F7kra*)6G zNYs6AIcRG@^dA_fS)%TH%YpYMm7@2S2@fs>r5!0lj}f`uE0iurCBs^o`|UUkId;H+ugHrA_t9Tgtg#4~=Xh=Ud9bm~N9Axi79z z5VZnNwrK*A_rCHeE;j&D>wUgL3#Gn#<*nu1{h`)Aa=x`3E?#}A^=;)~j0R#rfIgu> z^1e+y&?ON_t#2y_W3)c4?6=3U1W^>|V+pj>p!ot)^zG%K`Rdb_c}F=gYaogOV^LN7 zj&d*+CY7S^CL7T9H#O5x2&7_9U%sm2qN;w?rmN*(IV}*eT97mq5?V z>ad)^1InLaG3hfWgJj~8nPR%M#C_>_d9t4%jdhEi6w-^M`8eU#u)f8M!CWB1%@Q9A z7kYEp9-?pP@o^H)mA?LLuFiKG*nwPR39z}`%Msg9_V`>$e#ajd4dd~{UU=O zi+lx$sRTNu16pb@N&r&yW09}u7k&MBTm>MC0(}(#Df;oa3V;;-cw~h7g+7tBD*6hH zMMYmf5xE)iTIRsb^y}jM2a!1d@vT7TD}WUJgUD9^Df$PIIp|jpKbaNPO&J(l6-)BT z$XAe;qMyu)(#)qKU(v7kRs@}|08;c*k*@$!^iz?q=$AJB)A8H}(SM+?03bDeI`S1j zYWj5KD}dDW>Bv{`oD-tnc%XskKQPXfSd!1g1C3OQekLAh{bETzTP|$0l?9?zVC=tG zlFw%S7fbTlGOw}t;zIlWxpKjELj474-+{5HwC|rQ^U{M3Z&E4xxpL46^-KHy`FP-g z_;g_W^cD2~^W|VyK`J$Uz8vf-fYkK)a?q&v7f{B3A@U_4iUQ-OOBw%#a=J{VjQ>J; z&Dz?pCAK@z#($+;IIsQo1a~ZZSn}d>8q8UI)m^*J1}|}ca;NZNI8m# zXfm*4+urCQ6JI5jk2c{urcnp>Au04Q41r99Ff1W9!$IA>u}B{ck5yWBsJzh(odD#M zwVFgChv?3YQ;Tz^K9O>58*vgi;!Zv0R_z{#soYa?K=b$^k+WrXdP2cVkd*2GHYnMs zb(yOr7gerg2rwz;wvSM=U|knR!(IjHV8Gin@a2|DL{0oMvzL7 zizqVK&>|q%VCV3LuC6X_+>37ga1G(fN^=>Ak!}>OMN_!Xdq2YbBXXT@bR3&K5?>$< zpDy-@J2^)`q85oCcs*2)n%XdMV~>)Pj_?ChPpu}9CR~wuC#G|^udxB?gC=NxVPy11 zF0#7fwiya5A zryQKV=1-AoEh?!PT3Ck2v;5zkl4!JMYQAR`{2D{6q`{c3rQtGj{J{MB+W zr|Fjw&%@ZteYhM9Tl)2i!`I5;F%}R-fo}f;r0CbmXS$sZ zkfL8J^Kz5Zc#L?yUJiQy{(_8nfG*|+QuOQPvm=d{5zp5%8jlgrH_G9;n|{6eBp6$* zQ}i3c=+KUsv0WC7^e;o}K7q)GNF6B1<+xgjjKlxVLD)VnL&`Iy$YbTKSC_KWE9 z{9@`YBJI0*DM2RAj4iu#*NdRa0~2}-p+VF~T_+&L2PcWYfBPb<+y!_5b zhLZuf!z9KrE2iEy&ru=(lUJ{(++`d{|?PD-b?Gt7wvvi;}hN1P* zc?G)S!sB!0GlQ78m9RbTF)A}~?Q`0Mv!GGuEP(Xru`&;FhEx{eW8%ktM>btoi^u&@ zx$w;P`%BhEAL3x%loE1Ih-J=M`fAKZI>_k>fKU^|Bu^$Zl+U&ad6y2sa!Y{E{(rIt z7B2xE>KvWA8j3sO5Yf@pb>cO|F(DuRM*q-~uO$j!n5D?-Aj;MbNGB^?Ls`S}qtGps zfFrhc#E02I%>fo%`;X&)u6&$6bv7=JyJ=?fCaPFwJ>a%rQ4x-6ipED3D~c8NqcXZP#DQg~ z=}*hK+bSjwENiv>luL_9cDbP4@w0Nl$_bGOG?FdI-Rx&&i^QtDB>o#5uvW2^{Q+B;b(ums8{Am4Y2kl+;xW zXa>K6BCvNhMcdcnBcQe+E%aJS;@UBByA(<&Ps4U*j}%^ zrjmPYNZL-)Ybs`^QYh`HSMI9hek~;JAnC43V(-_Kc4&U?u7rCMh;9N;GC~3*@7JBcaFk{LfglU*N0{w3l6}x4|D1U9Wo+PqSn8 zTsU@kvJvNQE=IlSMs_fJVA~YoSi3cw9>H&Y zqXWt9xc!pGYZ)+VN9D02+N2Luf?|SHhU@|D@2jjMpm#$V+c!3L>sw`gteN&psNZlH z1=gqYS8x%2xs4&b2h5F!D*QGAq`7xj*0_x<&>DcaKIH2Zs{I}^w?B|y9DY2FUuTzX z+%pN!VG`uo#=u}NyQ6VYr#Z+SRNM}8;?Z^8W>~7$Z0(7aj)OWxujRO-+9u}~#s3lW zY^SpCFsB7AAxgND!=X4EH&Bqh8~t|b1>_^&Q%T%;1IS0dr-D=3-}*|^-W9DFK@9vf6vXx^9Xx(hO8LqMy3B6C&eQi-&U%)e zMLfdnvY%8asGRiv)q4&RJ!_mI?ku9t4LNqMBernW|8-Z@;oy> zb(pJ=%lMtxrAG$+>uxz~vd>|E1bRZ2cq~~@(={bODw>P?RHE7sRe~OOz(lnl(h>f8 zUsK3xf4@>VyZtsIk+PN*g{m_Rgf<=`P5$rdqYQWK){3Ak)-9Q0Y%u)>$&Be%{07O4 zf4|b^O2z?`8UKC-CFAY>4G4`tQ7N3!ep9+Z!?uv>?tnrPu3H$r3HR$m$1u|*+Ay(% zGe_BQJ8|mEO2!CA7KZCXDAK{XSt1n96OpoGXmtPN!ZH1|X(}G7^DPuo8gx1mdo6MG zCn{}jcQ9ZY^iNbyTVqkX2IMUEDcBgIc5OX!Xmkdd!03b@l6I~lZ5t;OwAaJk`7#rE>Lakg2djRssvjN+GJgRs&d9tYQN?Khur&fIxcR{H+ntvUZ~ks zgeD^MKjSHlr7Hr0Eb?d&9v{M_6UpQ((b2NC%F!2t_Fa+LH2DHZza$L-i2yNDMY+Gx zFJ}h_Oyd5zO6`n52UhSApR4fJXjs9B+`p*Z#R|5QYy)AFPmo;(f^<7`1(BY=~}D_A!%9NT>FAepuEgKh2`14)E-J>C0TV+An~rD+*xRQ~t1iIMfU`hnyyMewHftK3l-wmYbUsW(A>akW~ z-2R%P9nUfkG6aD93o3f7Z<|45(5S?j2_OyrbtM=e0cr5B)u4YS z=qusA%tsgbOrRfUK#G2)f?un|NkGeN0qk`OSH4=weai>0teju1lr1b-xv0T=I9|#@ zBm(_X4&);rj+gSHy7x7%7HWf%8Xlkv27u)KnmBM>o{PFJem$#KMS-ziPYsv-qQ3JR zS-t8YFxIQf{x`CEVH>|$2{HbQ#(n`!ibkWl?0>U@6z%1*2)LFv>nge1tNHdDv!JDr zMdU_Hqn4(}xbfZ@qod8Fc(xEB+i0V;<3o$erEihi9-g^HYkw`D^ADBqfY-Ge?{9~1 zErD2ZKwoh{@_svBYJd#Px5KBCnZ)k|VTBMyfw8D&<2&(|K`KSRqg#g7&b1ovN8%O% zQ4|=9s^UlDy?|7TJ`%UcwL;&GOg)IAKwq&yiheh;zd*|z;y7L_!OZvIer-*HSg}A~ zu|SG`FY<~&ihfVLqIZ~P9u3Qf?u*R?IxPlL^wCOiW&=|6(XdL%Oy2KTf@L9ynt<_J z#rAzap75kn^!t@yS$M4!>W@`|6(xwGz*tml*<zws#R7M0_m|U|aH!{55Ez2PgJo#Zi8=n{n^R;FxNGov&#bo`U5>(`* z(xV?#E_-h6Mb;yDtRGeiXSF{+WfM8b%}q2)M8cKBxEJDs>*5}EM(8i6SukPIgAW-a zOHK>$5=i;A1@Tcd7KciF)DI&EODY}vVFgRXTXc3_D@N)^h?3e-B?bmIi{YGWAQp(6jtXa*@TJ30LG$XF@6$RjB5*GF@7Qz<7EXKCRmJH zs<0UUcpQrnmBIg~F&K^Q?tUy4z6mRaQ$EbJ5nVBgd%Cd1iZBSyid<`~$Su{tiU1k0 zTdJ@kzu^lDUgWkayvVQVy8SrJNE|<;3s2CY;0zI&nk!&AkB%Ob+WNSca5&#tW^Q1Q zslEgbt*?k=yI6b=-ES60K(UD8o< zM>WXPK_mkGhy_CO@2G|&wj>gNXO^hL0%+VqnMCs5Sq;zKlyq3Uicz#<9zkwjuxdIRk*7Tl>OdI92FS4s=Zu~?KOt(7@ zM8#1I7ZB=Y(P`5*+Zfn;sw-Tg4x+2AkdHE`&`eE8*}YYa=dECkg7;Qecm(98=I8Zc z%u66vGSDdwkXm0~ZF7bV$UwecQ!uF>Xw2VGMToom31zqz@mzO_6YMDa{1H7hGLi~B z!|tLP`b0aiPg)jls0NuTsr2a$RX92c2%DPtH;Ei=OsV$qVuv8B)}1YjnUgjrzJn@a zbNb3RRYMSH>ZiFkRbd|9sLk9IysyeF@s*jJgUdw}!fQ&j$Ku1F7|$)nKa*CszT4)bek#YQ1v9ZG*pD>Z1GvY7` zC=_It99eSl2(G?fWXd}plto688Mo+SWZp{gr43XMmk{al_#wx7c-deW@MTvnwfx@D z@`IU&xAau^zAK~gx0|X*RgY-Y?h_ba^Oc9?YHPPi7w5onA+9w;EXf%7Xr7{zzbzN~BmsyrW={rmHM-qV%lx)Xu9poT`-9I$2aRM!_IFD0wxOiZWW*-x z>=8SUjfHR+p&Pocc(5^XF#OD%-)?~{mm^XDyh?SddT5b1|1hh_ChK&@z3PJeXH?Fl^t4KVT5FM?N_kW+H38(I$+^anmu( z5c`IMMcW+1YNOlL=nZy)=B+0QVvBFwI$1~TqhB_{!QdV@>^PBpqfs$=vMX!q@#h|t zYHXj_;3K$bj!lYmMc=h(o5nD+6&Yt2Y5OEYqJXFp7cdp)Bm`!(^m&R;iK}PcYrQr@^hh}u~S-pRf z)fY@`j0C2U5Qk~~!&-|UtQO8~ zN2wHFDY26hp)OHwqD@tNt|XoU)-!zv29qy7GIdyt2{(@;V`!5n*`BU9^y|MV$`E7+ zjgHkGju4usue2ZgDuVdIYIshBRQmeCD$j|0Nn(th*ymSsUtlah?r)C;4|+|vhS6vORKDY&~sI`(EmH9FXKPE`6Tu=9Yoc+sGFC zfUNd>Bjng`6PZ7HP^yN7veOm{9q@$$wEncYoduB2ezJ<4#a8Pq2j{1&r3=~#r0Ra_ zHj&|G98|cEXBm-=k%TBz72DBv`E>P6SLN72U;dF2 z`wND=vxbE*aj_JOTCz}j$G6bEINzdppR2BYW=yrthaj7~>Xk26bAKAxwO!==Vx%^^ zv<&|Uw#3STs1@j!I3Rids2aq~Kx+LX9d7Fxpj{fiKdvGxJ{#pgh&*7|L+dkvVg2<; z?=ZO`4rKQ`FVISdJ1>x`|G0`&*sEsFzvNXPT&$GxC)YT3M(25uGxS{V_XI0S;4k*o+aoAAI93> zdNU)74JQ+NOd`Lkw=y%rX zG$DYMD z-t9mRV^Im%zgZ2hE09XjZ^pabZhhRhA|V7(6zC)cNYQUqgOL%CqTdoBl+|guUitfK z?gvg6%4+2At7UUS$9NR7@DEXv1R@dWk|ZGG`wuQjDv_7OZ->zqa_A2-1IhdCFuXEK zMG+D{REWGlI}8LZ)o-mpI{Qdguf`r2>ySzZk%Fy^h0_t7jb`ik#Y3l?ycl2T9rpAqV1=>pHc zK+B9&6cmqT6-jjfbZH@pftG5gmhv+FKg#lI`hhX8rvFFPG%wTt<1DYH z9~kp$`hT3|W%_@T<<;~9V_r@FPqMsB|4*w0v!J3!8YJKbn^GWoe_GvmN%%IV|7TfV zO+PT^)%5?&^A<{b>XloP-0Kq$|FDOgwLLG%FV(oZ0HUzP-WVjyGm zvLsv(_h>k8O@d`Fhz|uiDF9k(cq$i2(OZ)c0A!Db{^d#FpFk7^I?n^7=*yF;lRY3s zU!DZAw@2u*e6a~gsHEp*gK`uqAWe#$wJ))SmXB`&B1UlIxk)pRJ!4w%t(c9Hw z-BR}Gmh#G^aBt$b6fn78nFP{76Zq<~tJnNOc12(U!k6 zDf!6)qA1XnL_mt(nFKo&AVu#?P!f4jmcXmBqS9CbV^OV$S7k+|v3yll6oT;Ttf=g_ zfU&6P!>hBRQZ2qZD++yhO%lAg_L%Yy=sYHnqOVDig6rG?QuH-RP>Ai3?(el(QK{j8 zv8brWYqO$K!@V{uiW=^9N$^73Q;-@C==?a4qOVKNb}uv_MPHW$FSI>!^1Um|D>WQ& zMd+c_aCapq#STjicURUbTDZHjd6hm27>7x|e0L``zXj|uebn9AyrPb}Ckf7adkRwf z0NuC@NYQ(ey5Bp2w0cis&U%H?UZS-mxp#%_x|f`9h}(6qw(A=cr^Y1^wE}&+0?GTv zWTmSWfYkcN#5Fr)Ch<*i^@I2pps!J&rH1Var0ARCcHOJ(`sT!m78Pk90e#+p)b!>g zJRJd~rZ*>b=XZeA^yVb+J9~xR5_d0%qQKY*?cTS<-Ak%v4!d`+TD>oE0~9);-3#>H z3#90M3DR|~J0L~xOVD0u_wLp1{ZC2Zfv|hQ^tXGP_) z4~#{%d*7ZF)$Ve`&?tNzx;wkUd?!7-NDvy0&EUMjme^ykx_x`LXyZ1Me()nRXwR?fiT<^8r`y0t> zV=)*+?cU!=Ob4pTVfX%K+`S-*0^^F;?)}YVrSD!+Df*k*y|>GMhCS!kzcaZ;wXl$K?-JIq! ziCLso5~5;qN0YOL2@GAz>G!}z?+2M~Sd!fFNkm254$Zz##&YxAIc1bB!6KJ6>bFa! zB+H{W_|0^3deZcv@(+*>A$>JQw&J+8>$+-EB&_1)EpKmdU&kQ*7oBWc?bnHfyeEs z2094=(uwybA;c??n%_gc z6PgrC&#zbhTax?lp=-}4=YLD^I@PE@UtRlP`fkx#}(fE4{?Yy?OnpNx&{t5-gg=^s`CbO-X??^4Y|%wx!W}(?2jQ}b7hp`bLjr?J3WUOBK<0SVdp^-6i{&5n52ac(cFD34t zSOU=q&>I1=guj%maApHY(Jv+E`AZc*8u?PvrRPZsrSW>@&yw7qhepQ9`DY3KEn33k zYUImtPCzsQ^hSVu$(Q4t04e(AI43|F`Er<(av*i)3$)&!dKqabvb>L;xUnc z@Csi~!gU@*UZ8O^N+j>s6PJMjVGh0#P8A{Y0*#RnB=0u@Hvohi_-0nGn0H{D;}zun zX5tUmkeB2CTiKZDya&eqoCH|@nsj%n&BMIUX z5P5+vxd4*)e=qB5ba`28e=R|K&lfi6)5lK1<` z>CP4d*)!iyLU_dqZLi0|5-fpO_&`61ftDIfe}EKyEHS&`LOH2devst;Ezkxn&mSaZ zv#Ba4>d-&M0RoW-bc=K#-}O(4xwx0eOX5F=&I>v8@dA?fpF?j+1^qBC6cBlVE=dDg zYLKKQq7DBN=L|$%;5wT#AbI~KxzOh<5tH+;SzgfwV9YDp@UK~3hUiC0@MBMmHUN!b zrh}plKT3ihJAx7fOmRJes2AwsG9Y<>99TArLK}XP1Oq-0d4awjf#m&3(qXTjkQdtU z)2t5~d|>Q@Xv0slK0q6OmiSwCloV|M`q~AO_h*Ug)yi%4$}MfVSGTztqjKKTRyMY% z+@>jgSzFk5AQFL2Vt`chvbI2C+B7e>wuOBMa%cxK6=_OuZLhK zprwY-Zqtga-N0;gDfXT4}hZ`R^T% zRcj}^B~{;y$Cj+0Ut5)$(V#z5*ypwRMtHoDU9 z@jGJIjTd)qL4k&W@mA&AnrogE3L36&s-5IHLp>|3o^zI~$Hya9v%-rltwp-YV{1=( zqFS2DMrT4T2J2VXP7ZyMTPT<1wJWWjb5GoxUe8EPE4^s=p?+QMw8eE{s){Mv+LNuu zXFOqzP35NA`l4#>R}Frw8&%@u^x9Lb(q#vKw^@w!T0RYasLpVGd+n4^(fq+93lp

HtR9mtX~FuDz%cL`E_76ep2gRssBQ)T|UXPb5b!L-CGxaIo#R#p7G-=O91Jw|qe@%&b1e$ScG} zZI|P~@&)=G@&XkKQx!WIsF42}`}oDQkG_GIZ9+71nI;&BlvdH911X_AIsQy{_*H$0 zzl@Ae6a2$E-PT+gf9=(e_1@YkMteq55^t1q#m4q7KLnhzb4YXz$3vibvK2{j$36Ml zN+0w{M=q;XBQes4tk`&VwpPQ8iN~w5v2Gp@Xo~V(su)j_^FgH3rIf5a{)a-=iU$OBi$IaKKaHuy|SoT57UC-L_12(oZ zSUc7G6Wb95yP{Tc!?DoF@}tFyGc(yCGk9%=A|2gvCpsP5SSvdj^npL?1Pj74H$7{| zLp)jUy~?Q9&$C;eT`Q~qPE1bT_&B6jeV=T4?)u)^nezFQ*)K-{M%{RBw@kQp|KqR5 zwJJ`P-Jx(g?e&ed(=h{;R9d#1I<5Kl2#13x#meUDkl5>Ky-H#h+Of(R|n>&(m{Vh zD{xj-incY+47t*trg{1&IzCf~b8lw_l_t?1$yUy!;Y%msIs}WwY@k#BNlcD4-JE*5tOGd&sT=2$pC~b+N z%3Hg8pm|MS(~QOXMp|O*uFBP)QagQ7Dh6fE;r<<45Pd8yxRsXzhPHNJqtrtE+}cwY z73|;9KiuEFt^Ycb)e^Bp*FaBOTWj$M5CrjhREmYv)Cb_@?Oc+aU_vV7((nCcNWccgpg@K*Bn(^&t& zj*-pX{oCkdp?<)3==D3$F9ESP|HvyE$e8>ys`X7w(ew^w$<0wp1yqj*Y51w**7vkrJJ`6>|Rx? zUr>9-@;SBiwCYBN`?vQE>>OUTqURY87M^xpPuKQ=?OnTf!<20Ct6vwE>llrXFU)t@ z5qI^6o>r^Y%emaG`Ft&@?bGf?yBIs=)cc%VI>cvv-sv3`?CFiAo7IXDtCAUt$ zBG4#SbLZQiluB}q+DG-;2cnnNxo6~Yy9|wYF3fg1Wo+~h_7a&Yyw!js7KYWGho^{1 zwttQoD)KFXqv24QM>R^2k%j5WP8~6$GsO0E^V<0O#PsCM0YWsJ*9X?8Ms{HBvN(1d z-(PI(*0WU5R9thW#(0a7s3nJIr>DkoIB%ieu;L|xv-$856Uc@NyYbm+ zG^uz&g;>uR;!Dxn)!4xiq?o5z>6%fo#XmhVcX(FnUkuJ~@^`R;(@vVJ#>g6jJSAlDE}gUdo*LM+6EOy#*3lUSphMy!Q|1MW9u^-@nWP`)I*Y3KfVn_X zaFqY+`K`2V>5IqFZJI4GTIJaZ6yRf%9-ow)%YEWO;yu|Yj!ok15xy41at;4M#C65h z=%7}h9?BugE1rfF@+1r}0A-;hA~K#vJgAUX1R*N+VkDhzKC~)^q!<*7ads9@$8O4t zAtx@Lj&8z>v69yy$nqpCI6$!&(!Syv;y8sgimf5YgBJswf#}SWFh2psV%jHXU^*7k zlrz|#<^&`in2MnpDxSG2_7aPe;+f~18;X(fth5;RCZJdhsbTS~v>4VV#k104 z)q3%%yz%VyV`~D61B3vv_|&=#lGQerCdH?s`S4CcTh1FtP#dFdj;Z+Dw91_He#dFdj!NR0?PC6vDda=Q4(%up5OF+_Wt6M0PijCEA zxUeoMHqLuyXl+Hkcy3AhOJ&5i1SDGrLlirAjbgeQqS(3XyFxK&X-7Je*p-0dNTOyc zcBCVTRY|cU9m$pT;?pqe@*0Wm4vH%jjZ^Vyr^Tf}e1hWBu<7z4Ik{eZdfFP+V4&C< z%BSMf)7G#BD?XjpYLI9|uqjyeM=YK(UL` zV(R=wq4T6s?4r&{;7h^T1?vg~Yi!D~KA&Om`wg%W3C>mvpQoM#h!Q3#nuayE#0|I-V4#sZXTCb zb@DAJ_Fx#T>Ht*NgI*{gGWMM3o0&AK>*;jzAUA_zFQ(X@QN8F5y)d7HV((e8N{xIk zX4x7G)Hs`STfG+|$D6~3m5F}(utFNuZNZ{Q)uHLRI=9`c z6Pdm`9JVr(C|(nW7C`T=L3yMK%x0o^&6z&5Aga4246Q6CirYdj0Q6#8=!Kb26t{(5 zfGD;t^a35yK<--ag{b}jYgI!lg=}#!tQP>i7z{&e+S%eDMkeYLi0THzdO^)Jl-udm z=`;&32p0IC}fOW!nu#o@D^sDY?%7^0>z!th{sZp5q8DX|-CF!ipc>h^>! z2jFA(ge?amfjiakHjf7~$Y#TQ`5>osDg4s!N~B#ONNFXWI$u@_*q>#gxMFAS@# zCKGi~ET&cd!Zu%Rq*3gJtTs&yRvx3dgFZ3Xcoav&Jg?BijD>FoQ27|DPPGdnPP=Q5+9M4L~o(LoYx|3q%b>vGG9ER%l`-!a4#la1$6Ud21Sv2`u6QqLK+h z6a+*i6M>+v5Sg4z+txHqrfrMPPo`~)GESy#)7pMi$zG$TX@6Qw)3hH4`jAGk{b;&1 z-Ye9t1B6cVB=Lp^!oDJjVh7H1x&)%w0pg}H-eq{y7v&agyvrKK7nN73y0UTRH#p@L zAn67t7D9~94H#5;Mv|ueD@oFrPNV;&7eZv5_U{4#4y83}Vh*J>T3T8o{hRT=Q%WN> zlNLj>UYzMLc9=9qY8HDIZ%kt|>vT#-RlR3q z*k0^wog^*O7pI?wLcRE6U)`j^8_)S(6(HlBUo3>kIERAVYlIt~Pcw=V&Zij@GR|8@ z%~1J9yRdUdDr*#PBx;jtFt+WeZ(9M9j`}_jqK2csQ9&f#gf)o15@5)vH4`_dH8K+~@$-_#G!rl3$pp`+ znRtnxkfbpaFHLLIOuRI$Q8V$ZdHxeQ9HCnmxk%D7!|dk4zHgtZb(0Cd|$^9dvZmYMZBLM;DgKQ)tQ4 zKeKS@vzEzIjCp8m=ZP|vVkR0R&m|j|si++Dgo>8Uv@&Kox7OA(J3GBhJxR<$JwwC* zo;k2grnZ=g4)q)(0N65lP6~N?Zq1ttZK0LOW?LcVokGjq*#hKY#Tj=Sz&Y zA@}Qu@!q%EO=h^_OJ&U zA!c}d0+tX5a?mz;6B07yQM4SwK_CfKP-BG}3Q?&&E{23+i1s|FD)L3W7liWVa=N7d zU9hBMI8g9TfycSF@K7PPj$R+6YNTLeh}slW$w#cFR~)9si5G4{#A&`!HMF+Kcd203 z9X-uAJ&G=Rj*(ox6ExN`79N}j5vfN(&ibJThU3)zzv+j)b)Zi2#61xRkOsy{+JJ=8 z-#KEXAYvSyHleEzwM`Jk5Cpc&ZTe9=CYd+{rmEOtATTh#7=3N=LSUL^;ghbY=%Dcw8vHv0Aq64iOAG0@~i z;K6PE)JFBqZWJzkL+cy;J9@Y6+=RwyeS;e0cT;O@>)+mw>Z36{u-=+IzFcEqb7Om7 zvlm6!aCcAtw*KM0*5Ky;;T>vh^FXsk_?AIw(0X?gWwJ52vpG00)Ys6LNx!nK8x_K) zF50KfhG{k$sMNM?Tl}HK*xbFNuc_X*erBVmk6w27Z0l1~G`*Kfn<#AbLD^ru3?3ca zhQrg)U|%mvx%G@kAH&ey+`B$hI@E^(gwg2U*4Wg&y?aaFP-EQ_jS3UfyR+FR6_ehe^u{e5%TeHz1YlVrAi8!ajk% zvabk2N_35&MLxe_CzvS)*o_AKX|@@{ZCSKKV|bJq;bz0CW?M zQOAf5M7~vT&f5#TB7>-(R8jam9R2)k+854Xt5s}6_sp?<&_Z~I%R>Mf^|5vI?Hlxh ztk=n3b6LP4RU1nB^63VgN{U6G^X7{Qtpt4^oLXwc>{1|A3|8%;%{;gsiKIzk|?;Ml0=9Dc4Y zjB?@}*6}im%1=-HGSTClZ>?49c*2R2#58y4u>IuaG&G~rW@WattBi{)TrbY84GPyw za_5-B^_+T9f4Ua-S-GwDBl-ya=|4U@x88o_j-o&P$4hfv_9NFA{pmkGC)Z&=q6N{P z{^Mmim}NA(Pe)A4pZ?=>b641p=OKjU&p%e6IBx%vf3|-@uC8{ohyllJ!|t1~yTraQ z$BnCppon5nvL8Cng5IY5jG|u;*2Cpe`a8A}z3_EEkp* zueY!nX0qk7_z$}V32!2>1nz&3n9&tafU#IU1KZ?D@x*{3WTCo(Wj zJAx$Cg%CRu7jS|YryZ+K^>N}ByE}K?iN}eX$)(SF;&ICEmDxDyVxgC3|6Lc9q$9h- zIGGY^Z#qu)DT~I*6;pWWORONB6@~YPadO4fUdG9`9!j6F+~kS3o-87s%s{o=!2g3s zO5&lhu=PxJHIZ&T|CmKX<@&6Nu=QM@H4(O+>$4`})T4n zCo@o5Qvcr=Dai+K3QNj#ayKtpQcH%491PXXVM&1~eREh+uBf}2CB96T^;zlqQrHvID=LQvQZmINuS=edhao1EJ72K6) zp=msUjM!ayR+`ayd{f_?mk95KoWq!;kxx8enb7h-c)XCDm)?v4q48K!=a%k6yAXH> z`wZDtecJH} z_z6CtAK(+n(L2tloj#1?VDIeA3~?usT!skL9p}|nFHOMFqJDC%Fnfau)srTFb+p&k zZkb%zhLJkP?9;QyWJe#X_@&yKWztm3+FCWR(xIvd>;1LW3sf;OyMM%t;<2EoR&zCa ze1rZ~DmJh`>q*ClO0~6m=~bqKTc-%=iCFkp=v)2FTHE2#1Ct~3Q#VgqAIi-{KMaB8 z=4govvo}o6L^ntMGOQmkaXn>jc6MRpfWKucU&bP)(6f20gIC7y962yOJGQpX+z?jqqds#BKVjrt=fXLCWMXn8ePwap zd_0q_`R1u1t$P6yi6~=oUj6{)UIm+g>6(|BTiQ8_IlTY~4TpuBxlx5aqmH?e2~b;qTDW z%p$K}c@DLD{bIH;OIUMR_%h*7VjI_D_U?rf(E<&@gf4k%)*O$_1(1ti_)wxuJI2K^}Qy zA&ebsZ78vvr+Qk?v8*ZlfFhxBIs`!x`B@o%HS$yMtJTsW7Aet&Jn!)~WJ+C{Ed@LN zZyectUea_8a9Xtwo~DN$q`%tuu~QZAY((>vK8X+}Yn4)lM-MdPQwg#hJ_PATSLx18 zzN2f0Zra38Qh#!4oSY`N&e=)&RXMCx7KY6PQQ6GvDQPk@7GF&?(4w`COt4>hyA_wk zxEK|GZBzVJq<%@pbW3OgljiA`oEC^10v|cPAsvM@lr;y0rhWCHo6nQ4|sOq$XqTE~)lY1AX)(HW@hi25Q}bI>La%M2(ZE{&P&gVi z8iF0KF4a!&u?^6BaB6yjg&qmx`sEhB?Ldy*8ezPhg&HYMzA)p>L>k zC;E|Y`;+{uJvkhmlM{BRFJejLYyvMr2Jv_-{-8q1z$9ht$_~4%CVE-8w4ojDVPW9*E0U6fE&vC4$WuEl84z=}j z-CA38te;{Dwo3g}fwFimgul>0U%i}GShhD;8QwZNZ~0l(b{_C#3`}is6*an0;ylo= z(SLN~@0Mz-oYFPZOGf=*?c~Mme*MDbY3SlyVJEep|Ku`joL^i!^|)I>y=(dM$7PJ| zt$)(_LzBm`awlr?xQtKG@vB`6n|H>BT>!Q47Zt-$H(abH* z727z$%q`8#H}!NrSh(OxIN=4p_v-sM%ZMkJ`v^unC}NiQ;ypDeXG*y(Q8^R-bYDDo zz(bBz#VzkMWo%`+Zl5LxyxR(mCQ(*AIO&)X_J;0wra|l8)wip=F?0+@<4|LeXHaBo z)9~d^v3RL&th=tKF?8)VEIT+;$Iw0Urp2~N%q+&kn^SUI(=!krNYuGzl#=WfIvr>pk=I|2r+@c(g+fE9^nPvb==j58~ysyn@KwM5oae0V^pYU3>KA^Ss4mPw|8@lOPi4anRU7KxbZzEhp zgr%tIF11Dg?lme3qw7>6|ON!-A4s{y9C{Yrn&TgA~Nb`Mko_?Y|+?t2f?#QYY|3 z9@lea&X~B#fOB7wP_7FzBT-BEZ5{$RIm3H5p3`KyM`nn}?9U4M!$g`M(b(uRmQ8e5WAoHB zMt{tdy=;ut)KYOFE#h4+-{>BUG@$=u{`#TOocV(=`4d@vzBtptma^--bSuji{JeB4 z%gy9GwlcN~m?9L{;ba~?gx3+3UH%}*nlET|X1su>{2&;vvc9koJrAY$xeyhGG-j@z zzOdRB5v^Q!(n@QE=DTuFvsSe0yAri%9(Wf;uS2blixP8gz@Lq7T^!s8U1NE1XvH;_ z7xUnR+H{TO#R&b?IUMWc`RD!?0M^vT;MW6ETJQq{(YcMm4-7r$v!lzDhV9w(?pLVJ zbcB}`*0j$xwvXO4bqFtW*IQ~{{k&x*&T^1yQO12J^3=elgOi*+Wu}w+6(1|CJ6mgZ zZqH4=i1jP8ikVL13}gsqDoB}VWje^q7U>`_vmn*c5AQO3rFU3Y>G$P@we8)!k~j(} zpkD}!L)U!e=ta;;-U2-`zsyQQ)^vE5ZIHGsmz~RlO3z*SE)UM#AO`Ppe5t0?vfQn& zEOZy!uTLLn?BY#t8n+^W#uha_=Jm{e2#A~Gsxa4IT^2wg=ee$OcHpsUhiS~OR!h&|Enx(Yw2BQ`H&+*G)*{b0-B^vVco zhJzx0xy}s2F6`Gk$0Ai-jq4`o;<{qQ5EvT)u{7-CM6S%rYrb)0W?FV?%&^swPG6a| zzbW`YmCO%m(^K7}97G2<5hL{y>mV67^M3U+(hkNMiqf;j6lR1@s?zE!jK=2RnN>0; zv(3RX3&dz_J~a-ooXj>i&aE}AF?>e07P{K8zl?8r%JQxgdMQbfnL_H+F&$-Wv@xnT z2eJH)k~cGV&h|)^zHCkV!ow=Xt-NpH^CY*Ft;CoNh`wya(M69{mWZ}lyqZ^RugdyD zq%Gd*ReXr!dL@;D^Z8-@Mk!1>Bs|MPeeP*Tv1|RJRC#KengZkG{BB(+?Watu!HA0hOG;j;tdwg;vL$!IJ&}l|3qELq;t!(zk>D$ z)Ar@tI9QGI!s`{q!Kb)4ILgwY++O#9l>Ng{IE&2wVK}(-0T|!mFuovm&#*lsZ~n>@ z8(!;P({A`MjAzhge4KeG@8YHG3o*)@>6|=tn5ZfCs`DsZn#}kmO_Q1H=#4xFIEp7x z7(hz$0lQM;3LRs+f+IO;e84V9l;%(msNprMIUJViW|_k(+*yppY-JsEBZNUQ-i5Qx z7&+`s(=Ld(Mt@v;nGJtj_h1Img*~_s>O9Az`ugbJD>86BC+S=D@~F%q@I$?{{VqKb zFHP_0bn7Xh7BP_s%(n6J8Vs2Vb280C-lk%=;J}U8!DMq)6(^1F1V4%?q=?(_G&YBD zEOsj}%V@W86hAI%iFd>yS=5G4q<_rh3j?!Y#u&vHp5?JL%od~g!e>3(J`peP(cBHb zOhjGeD67$z0Ruj+YZ(3Qt(^DcG2UH>23i^Kz+T6@jGszo_K)W3bZb`7=)6<>f)3ZFJZHvt(R(F`F=(4%?D@zy(+{qb=$uJb|YCd^Z z^ibD=o5W}MI%|&wccAdB_U%E$vHbO9*@w)7Oa$o%+_lUN=EMoXAg8^MSN4Il7dp@n z1n!OZ0qDg6_b+7N0SAd@EgA=4sU8g59YnE%Ri}6$uf^kOzp2$0gk0s+yB*$`1j z!PD`;b+>T<#SW*%ROR8|8BH3+4hPR@#Zku}*NHZOVlPgMX@p2)kjh+$wCMPpqT5$Bf-laK(QlfG41aoNgOT(+c^@v>=k~!c%yqZGdDbs z1oHg~qLLc}3j(5&8$-k%5S85MUeA<+6FAD72i`56+(*+&6o}_2kL-jrDmlul&H+)$ z(RCNFjtix>dimzu>k)G52B!o5W)`3pxJ77qNtkQ^oq9=_HILBnk}%mGq2VQAvK6@< z-(Yux2T<&#X)!IWmj*X@(&)xZaf5#gG8&r00{PZ_;VJE}DmI!twi|XhVou(m9h>b+ zm5N-s6%q>t+~nMP&OIaV3kTE@A!Smk@+syVIjWIx`LQlRdWC$KU66WnES%r4wB4>lauSiT7jpx5!{oRrn1|TMRMDS@0Ae|JYo2ZU zW}Vd*WB>L%oYA z0@-I?F=Pmswh9w;N1jLlo~Sn>fw5I65wPm7{_$7|lwPo;uW?MEmB8?( zA1@p5VLUQ_ZtXq(VH|*W=L=77f7@{%7WMA%DXnVx#~~7gEljCIScm`HzfO;A(&6}Z z5OH8)Z&I+Z^i1E~d3bwYXT&YICr?ZbpK~IgqJ%NH zH}7vp2vHOmiz=$Yz1b4gGky2w2`-?=U~2Ws8}hmT4B?k8t#ZCWqyGYGt?3EHH{}bz zo-e|h4I(C-fuL|v9LF%NE67;5y20tYbOz`1Rsrcc7mc=<$uxNR{vy3lr!S;>T6aKKnp72lU%5&I+pcb{TMh;1cS4^KPhVb!-SBiCChRc?{* zx#|Ie?ee^5Uv$wG907+L{Bcnn8*xXf+!v081p*?ovyl^j1cV)XNZW#Gte5W3Ber`$ z*P?r!qNongNe77(eK3!pC8SdH!F+fKwI-_b+wpV*Q4|=9ikAF#9vO)j71jCeJVFvt zofY-Ud-A4J7u8WDmG|V)4BV^LOa!QU>3zg+X@7Yh3G-s*j4iBG5n|HB_2{Mb283?s zqw~DdIVTyP1jl3C?2nVc!Ls3fc+6uRtTPn|?!uC_2X)Zs{K2tVUi3tUHi<&r$COSn z#PlkKtb+ID!<)yX@(J(DuXREJun-b{HKvG XY+Q1p&>xO`GO2vohv8VG@5lcFnrCv9 delta 30843 zcmZ9Vcf4Ii)%W+FnR9YWlAG)#_a?b1H;pvXE(z%gDS$}tNRt2oLQ6=YD32$R1nKPx z7Zi{pf*=qG1Vm5-DFP1)DvyG+2T_osQU&Gl{jOPS_W69?|MJVM-^Qj(Nu9^S6o?ZXm^T@8h?f)0|uU=bz ze?VT{GVH5w`7LXX*)w-skJNrTI-f;R&I5E1T&xI{#owc8dMv#!bA^$lC*Z(8TOrk;1Z`z-G|sUkf7k9;HTB%Ssj%6hq3R_5UKs`gAs%6bh=2wtyo+go*j-ldfuTJN$t z53P4uov(tX%`_!mc(antp;q`7x0jU#w6s)P6GBT%+vJ4cwanhbDz6m2k2}zcD$x2= z+Y*A-XW-O?sJzeIW^Bq3-6hIbeZ(^}oA?psk@8)gy?iPjM#O#;y7 zrdW}9uFWM2#K&#z6H=K-PMRE>k{4lcwvDz#I(~4&z^E>lq15gMMhXBs+s)3r7DrZl zW4~00zV_BQ1Vb*PV2D)=fHx%ZoK!HRae&csXhQ}Lvg&BxP^S&cor5;i^)NwEIkaI= zLY|HvI%KHvP%_Mou`&;DSXrI4dRSveDig_3qm32{KirM7q7YG2LMR#D9H$Mu$1($b}Etiqi(cWn~W%P8+h~png1V@xMS0n0Uf>mfE`X-yJ z&_-bMqd9{scRdO}#%(jWtJh5r)rAJ_o?enmG>yZ$inFE}qb+?xGCd1iU4a#us5y zHcicto|#0C%AE0H#$+=CfV#=14us%jGXsRW$!bOo9%WP9@aQ-IZk*zJCnRMpI{Ybv zH^nBPm!LJ(ybpjj)v5PE(59OALC~h+eX%|iVY_U5wO)oqZ9DT~5c8*-`2ftHZsvoK zoNnfW&^O(@7^JEh=0pIr86{1eIK!Ms8MGPZ#7YrnW%DqjM&ywpHp}K~Ma-COW&qGP z+w_4DoNZ=+&^O!6sE8SJ%nSguIVDZZm}6#825pXw{EDmL#kpm5;>EeHIXb@*y*Rh0 zMh6hQxl|`!tQKKGc7%FSM*4z=zHtmx#rlO#r#Jw03tdZ82SRXRpZ*DNN%1`%D<&|*3Pi^E9c$@o|t)TnBVM2q`tqy?dK@$eBwqh(7%T<}6Qj*lfl zLly{?OIl@i)+$m6Z^@`J#se%3IsvOr;_uQB7f}Rdyf5w7Zl=gOwse>V_NpWH>=-l> z0if+zS}6l<$Nq5*MTx|o9XmAIS8Eb`b_!!nrNo|{N;73(?9`e}k=V1-=xxkO1a}TP zKdVk+&(1+Z9SGXag@$|(w4K{E%vL4ze5tguN42N{lQSjse5o{3LeH0ynXaMmyM!5L zWdqgi5|Y;&pzShL7p$slfVNBLbhEOt@LvuZJgcq|+Lwb)W)QS5x7W5cosICmJY|~k znhL*b*x8Dj;O!b>#)JEs;O#nQQ7VJB>(ZT!)?E1Af(G5HYlgO4*>%m(cI%faZ-%zp z(BVzZH5q-bXW@4*M;JL1C^@Akyxq$Y))U_DEA+iTneb4s7W?;R!?tq-)lLmC@>a@OFz2c^;12j1Qjwl!Yg!tWEdGhSbK`-C*| z`%>LLLz2$E(Ds?yX|$s7`vzT0tFC~yZ|R}}+P*`lnaTp%zP0JPm{wiC!tWO}wE@7} zFC_Q%gSTH>8UX#^?KgJ3RoB1p`-hn7;fDUu_75ri^oO?p;H0uYwEZVdG1`E_A0V_E znO)TlfOkO28vyTsk~aX}0VS`s@CTOjpcUGIrBhm=9XKS-gH~tw{X=wZS{6?1MIV2bFy=u<&0A$zcQGeWjc~1EGDTJq?F}(7rNxs3cu)=A!EKuLR^8yjA5wPtV0ed=)eVMsNPC*CgP|QVC3Sgw;SUX|;qA~4 zEk{^8v_l7tvo3FkcId>(X62B=A66^7d~6hR0ct+@#Zuv|9>0uWs8Vz?9}Wx5>ziT{r&Rm%EujHaHH z$sX#MQYGulu`b3oieyDO*2U>42*G2A5zU&UH7qE{m93E_iQ9}dNtS|>+;r(tiX`}-7;#b33&B-`lY=`bG zHHsvro>5xTir^W#u2Duz{aWcD38G&s9n^;4*GdNwM896vEn)NPrI7=53NUxX@5kzm;d_ znp$Rg_*Ndb^&vEf+H-QIlV}B@71$#|WY5VP1|;Z<8kn;KQQKHl&(E`qOf4ZHoS!#{ z`$K3HwHG9{Ak+d?EfCoYl3F0fUZ7fqO5ue$y($)gpaQj>jjl!%2R9B(#c(go34CHv zQ&GJn&%SLI5qrWV$$w2^(WS{E5Nd&H5fIr+lSM$ZUYaaw5{oV~i)sM~7px`$W3gET zgnOA<6cDC=yXw^v>;)#OXzt26nUS_43$-h`x4EcZk!M#~V~CI83VKcVU(M2*?xYrk zTA;QDh;Vmm4G^{6sWr_~=ao6(PU-|f1tzLA=E}UGUn+%qWzH^A+}pFLez%rq*P2NL z;_zLmMS8raSafx=2!vXoss$o@b+QPE+N+aAJ;fr9Zq*_XRA8ctMb{*Yde)+UuSq8L zDyrAz*^Ooq5kFj)dc2odbbYc2gj%4g1tNQWvIvMp*C&g5iA6Ugi$G9;LS_F6j>YCA zAlw_0MZJsaO?h^US;W#3Zps@Z4G5vPSoDKr5eT(FwFrpp50XVdwEiGj)LSgN8H;MM zV}NkMi7OV}oSZ}{+?$g{Ek*U#JiEgzVhszo<_)qF4WUIWx-D4*LMu?U0+GEfSp-CD z?Y3l6i zciu3%sh8|OD297?PI4sEqHj^XH_z@ji&)UZy(td%6^phei$JIas#+kjTa!gVv~Eon z^%aZmOJfiO6e^$p_M`D8!=$Mc8A|wB-XKRyA@mcA9!?g4 z&^2=gA@vTwvmgML$m#Q400vYLQgS7f?F#j5&e~ zEIcla6RrKl5l1VIHRs`&8N_+OVsisAk`IdOnE@wuE$qc{Qt7nr!B`8k<_ zRtojGd~mWD^n8k=1Dp&dV4{i-pU*jMR_Xw8;`0(nuaLWm0UTq7-{oEZ`O~E}{*C*8 zm)sA8`n%M1K$yQv|5I{5h+2T!79hfZO6~`u_D^X9 zw~3S9%-KfEU;#k|>beSyrM5=^LVZ*A2JLII>g@Hx&ojiNr8pD1qypuAXfzp$iAD}0z~V(Xc5QwIKW1uFv;3#bt0#WtvH0p*F)&Jz#H=XX4*?))s$P|)39wG*=cebeop%$nH z0+C(sY#jxnb-jywTA@<7p~g2U(Mdoo+~CYYAgVSvb0JWqof!y3)dtt0(_&~*-RQD! znSrwH-{>sD4iy78B?Ccd1*(C-8p4~Bg<#ZfN)`?k3(s~o2tgPKR2KqKb+)rX2#mru z2!Uui+u0x-R#eY**#&0cFeJ})x-Y1OVPfHV$wClnfodTT;q#J#K-8X>3>+o~o}WAj z!a$%J2t?EQ$%DWsY#s!n>3ml+7Y;9~7rE>bvv4?~7dh5n>Eq#I;Kk1A))|CaphgEE zvKPBX9qK^TUhMiw!s$XsQN7G%U8c1I$;-6eE{rIum%Ho=Hkt7RW&~xIYp1y|vZ!9^ zvhP^YNQ$m>7F0%xF;_X;!GbUbsKx-1y~=S9EVB}bF;_X;Q;#aDSG(+5(>e;tt6f7) zL%}FfdyV4=P>3KF7@(fj01>{%wU14usJ+HbkdrtUMi(XzbX=xl8Q z(HNj6$v~(#I_^27sr!K8-ss{1mQ)M+zT>n_N)L2p z-fwYT*S&vFjekdAgAs`Rx1_-cMA0p-ujX1n6y4(D*^=l%_J@vhbge~Jd!VlN6hi%> zYidoUP=DwK>6$ZI+Ip*VwQa1liWOj;5|qZ`=2R^Q`;-ofCxleXeK3%3N@UoN=0Z5! z<~TT2EBF>gVCn!lGQ7<-CL6gD26EWnTc{1Stf=1Z+|Qg>wp`nX+Z{=^s%|L49ZU~X z&84wGUCu^3IX1k*6*~BUsJ_E-@UGOxBHZbMu63j3;#gqnA2}Ml)Ai--N{!~~7|ij$ za+}z~Gxe;H%VUA5S~lq0jd+@J}ml<$|o z+zcwWCtG;OwXSlxL>8D#mb0z9>mz|{WFVIUO6^sIEpW<ngnJxUgjy|^$pSTDqY&;rZt%oZ3ilqzEumK1!pVZ;GE=EsC<{y-BIi2yy7nol z6zaW>`%I;3YuqjiPTT_S{dH~u_x?J!UlD#>?^`aI1tzMTApF?1rS@>e z3@p30KL;L;n|HOCOJ;$sO*O-CA8>6cm~hVw9r5-DEDB1*U3ctND|<#auN5 z%htAX;NeOibJ;9V7pT#4Gg1S$jxwvbZw7Mjpw+e&;UUM@FKQw;&H{DC7%ex>!2W4^ zapepgnIgi#BK*{aaYp6RS)c~e(N5CwpSl5YcGVbu+&hEqzF|yJeZ*zIuvKCVqL0X; zE*6cERpQaK*nkK$KurLE$Ud4903cc))dYY>6xGLF_M~au2Fb@=JnLcnZzF1-NQ(~$ zwLn!1MED6=;>8U>)IO0`%CSZDDVIHCYR4k^R5Es~Xni_q1)&wFT7k$ujaH36VAMXH z)Q;mU!DYWTwc`+dHl=CfMD4GVS`cc1suqasuadDqwEik-9bZ(RciC@E>v$xeXIa++ zukoVxH)$mS(PKbe34sXz#`V!52Sn{}bR}VI71bA9_Ip!10m&DV4+^2sC~)VF&PLS0M!RTWM7gML^=$J)|Z?e zb!}TzUvb%Mrgd8x|F2kr8MYO*uO_u1)B;s45Vfx+wLsLqn$%7zs{iY&gh zCD|m=`bHYGAhZHi>m<$&5q=|$T8dHoMoL>JNe1$#l%Ig$0(E5t#!}lm1EKyYr6-dl zwRtm*6A)Bjs#X-gDdQx@ABy3=nKG(Lg8p1LSxzp1)#y+P;r`k6&>#Yg!nRkLB$>@$ zQjde60+Yp(%>2bQ>NuiQ?(Y7QdVF$G{j1B~H78C+^sgzIoh(j#$GNf51Q4|VHCX{N zuHJFYy6gal0 zUkW)uzMTPEQrszC`A?cOAlePob^~Ln?U{g3|C8oSr?`8)w~Y`8Do}eF2z9-;{XG!s zdM|TEz8&loxWU_24FngMxZ>^&-WDKAqj21+2^A-P(_1J6(ORJH0Dw@x>Fve<2=$vj zhC-oI%~`cNw<5S+t?VAo@)kxBhI>|>OFbLC)0D9@?iPTW5d)!a^h0!y2!y)PPtlaI zQ+5lRJRe=ic<-#a9<%_gqkvL0Z}PVB0HSJ>w~Yr7RhztRJUaQJG@R{ih=XVVFm;#g z4$k&=+eazXv%T${Ivu-%Z+X|L>o1UM!KnqZJ@{6=1+qK%mgmb-8ONQD-N8BD&D9n- z*&P5ARdxsG_#v7SBMbE$KWmOr*&UqgEf#dTW`4LLI5obR7M$ztD2HNHo$GDy2SnAm z-ZK79K0ylSdE2{!paN6lWwUUeFUP8E7S8jn1LLl%lP{&K7kGD>_Zt8C55zC<4L!|D zHWL?m%NjeKY$kvj@PWu)=vy^22cr5yZ<%?g>?JPpw*3J?1?u((2=yXw+aI9B|BJlk z$en!H6fX9BI~YfXlg$KBqdJ9fFZS)4^8=~&V$T<&+9hlzF7Y9CiR>kSdW_iVKU`o5ao+SuU7QsH7{NsRj_1QT+%E`W;P?a) z{*K>PzSk95M%#BiUoWYw#0p^CzDOam-}U2UAK}7wMfF;rea~X`c1T|9uXT_whWEEoD*tzRzy5q8Su@-^Y_c7q%~|H~Z{IR9Dbq~x#a7?_v*G!3(*fw+ngRq6*{$AgPk`v%Dg*NC zkxAhLzWWy)`Kd5v8GNA56h}PZ`7M>Sl~R}w$ml#cGI7L1zWd}6`Tj6vm^@TxiXR?I z5t&k$4~YwRh)mGKzWdV=`P>;Z<8j)<-i}g%Fdz1+E7OBW4`;?k@;q* zfl41s4g^Ad%s1#M3J~gJo@3Wnl#1|EzI(S(`5&-X4$z)TqX!7}DL+Ib0}$#{e&(#E zGo!u8KI^-`GCDsErks~OTW87&_^cnIr$Cg#eAaI_z3IfrWCeWQcZV_gv6w5zWzW}r zBn#g2-cn>rVLmS-^xJAKE8Yvf`?@jt#ZYCzd!bI11@8sls2LNbP+#z5l4mQG74Jpg zee^c@2JCHw`eL0b>)ngKQO}Dgh5DlJ-`ccaw3qenW#2t$Y`!OzHo<(k&XncuWpA4h zN@2e2+XgjVu9dRhz3#iO8Jk}MRSv^mPtyVj^>yE*N3%evuX{d+wWhMZz3ID;9G72Q zo9W~%?9Dn=mbW*3lP*b=LVeQ@l%q`vKGnB;_P&M(`B77N%U5CwT5&QZ-cG?2L?KXv zCy>GWb_$*`vi>zjaQVO(h%i_~I1u5#%8ZeK7~}3cb;VLEFe#Ss@J?MZ*1nsT24(@W zz@%5A^1F4t4DI)(Sh_?tGMeA>PCwNIBK)4@LmP9#!7O$mK07;fKeKKAjoK_kKk)Vq z!Ypa|hgN9-(ThM$7l6oqn4$p?3qLd)5T@`Wi|&G;0uxo-@=*#0ltTST!a=-ypCv8- zIK@;DTwvmg=8scApcL-MDW=X6^hpWUneVS>TeBoB{ye2~AgI7Zm8kG}nq`zi{oHdCaa6RJpzwuva|Z9< zao};wPwrTI-13f-R(9_+CEsKEEGK8?l%}>zkoY2{r<9`U3qNai)A3OgF=Bn_esW5_ ze8#L8Bi4t~28j{tgGDAvVXhDSAZdfl^jVS=pB1`~oSL6W%OyyhRi{djI4f9Wq7>>` z!4BJJvE>UJgJtm`>;>xD352>aSdaih-53ZG+E>JgO`&X?1PNd*aV1J@3Kk`1#VD~U zSd{QZ^_-Ah7~&SPBBSY?U{Qh}my&1)OR+!{0(CM28PewlOR?mar{TPMq0ELFIG*8G zC`I_Zz)vM5N_d$K=LgHFKnMdh9|OixOUJwn{|kcUR3K!5niB$%y&%M#(1qDW_2Q6S zrELkb<*N4LU>DD`r7@QT+gyU^8K4dR1xr3Ch1wMerIHBD7Ib;&9=JV; z008FYb*8xC@?e*!ltx*meYXt4*)s314Bh8!pI=r3K)td~m05pf8o!i6y)wky&xJWf z^}8YazIkvClHUz>+BioXcy)3h2(3W%0ub4&lLLWhy;>X?!`~cf=rzIaCP8q4i7Otw zCfNB6rEsqab~iak9C)que*iJIfttnupHUt{ zieq9-Sa#G29mgzNapLl$JC?1D|FdMz-8)WPcEdHDYae~ zY(om9p6db$+%BDD@2cZ_IYJte!ocbMz z)|=DBnJc|>OPV-yb>@H*SNi>yG;tAC z>ZIi-ukBc~?#M}NS1(^aesaf-C#_n0M#o9Zmz}(((YPnb=b7?(H^p%8Nx@;BH1FQDW`f`XweNs%?+teH1%!I9 zOjF5@Ls7jyWWUg^tB8U3hf2&rDnYvL$7#s{Q3$N*nghngKh`xTAWZ24=5Rp@qlX0| z`+#{E2>OXNRS>eknD`5fr5g8uSo>gIub1fyPKssWd$6t;_diq@mgWGHtYpUz)n#$g zPlJmGNK_*Z29A@y5s2(h!^C(?48*@bGg*FK5j_e{!u+z4;!>E@?cu=h%~X@*YQT8F z36&o=0>{U@Tp+pH&+CdL8v|-;K_Rj~PpJhtP!S%%|EX3~1I)3E{hJg6t@lc7MFtN>H2t1MM08v;2YdQl)&l4$~iF>Oj zgPwXol_3C(#~cEYeKP1VhwPvDMmyOjyQv6uAEVo+Uj{wbMV2A(R4L2n-3Wt|und8x z>cR|xr%U0uQUH^z41uTXvJ8P|>ax;wV3L&~@JwBnA@JXUUyaHY88*qJ0#Av(3`F?9 zL#G@B21*$M&x)|h$`Al1Ss4P)s;mq1i|TVBd&6Sqd?cTfcqsYoeDVJCDW3&l!t;Tj zmx@{`Mfmw($5%ky^n6+c=Zi;wW8OzrQVpPn1Yj(+Pg8+Ve-rG|Zoc%*Z&LsSK?Ul4 z-+Vr=hx^;mLjxeiaDU4$Ow44^?@|<-AA=__RVzLByOiNj8ig&xnJ=;Pg}TKOJAoPi zDTMlhtSa$}5)AhRFAj0zZm%CyZXFZA^w1MPcc6ZgBR1D0toj--BU>aF$`X+b7d0;OjH>a zFV(5CiF>I|B{2N4PL-n_piqfF^W&#Zf2?z5+xEvgmq77yu;cFeao+~i91cjeFNbD5 zW&uKdIqjh4OW1g&PL+KdFi|Cnyi%)kW#jfroy*4U)nMoK^W(-1sE6)AxUYs`dR`BN z`f9NA`uTEs@mgJ0Hg3RX^SSKXUJE_VV#Yt**Xq@BJMntGZ+l@fFgaOnCteRtx|q+8 zw-c{x-@34%sJ&_s(#;MKi~ekJkWvc& zk_Ivei-0=XfU(r39uVqZ($rfZQ|~PcK`4?b0aX73(exJn*K;{As@@8PrZGTNy_M3K z1%lpAGZ6$An5>YQ_;#9!lty8ji3_CKzXtoj7z7olGjRc5Uc>#X?1yAZPz?94vR#vz zxIkv&yTN^IGZBpRyOJ--bXp)yc#m&mjVs3~z+MR=`(Eg+^AuS$zn2aM7RVy}eh9S~ zpV1^|tYD2pK)COR7M-X-xbFx13~zzV#J>gmf)4~0n5Z%n|5m5UO#EA&%1r!wo!U@? z3r<{Fvj1M^%1r!woy$!8N8oWB^_1Ky0(BR&AkM^pgg(iSG86w1I^x*?&0{8hP^ZeR zA`q(Nf0R<~2X(H@#1HCRX5xn-jJIm#2nDEl^MW`NKMehnQ8E)h4E&;B`-hqMQJRSh z;yEi&R~#U<);GWAz$v_M+ppI)G zgW|K$tbPGP{VeV7fK>8X+L`ZA)T;ju+5cHBJ0SXRemx#**+FXgA`N*EwE(piAhmpv zHm^XaUxdEe6d<*Hp$WAMi;C*{O7_i)eo(Lo$@LYhWKm6O*-)`f{GwP3P-_8F%Z7?= zk%4eGRIC;twQQ)wdFI06qPnq?oo%%&Msj1SWwBJUDOCcZDL|=&_yeYvO{o?j+)b$# zAhm2twJhOetCF2(wJbsMoQi#+zC>y{w_;~)AZh_>Q-D-*ZpESv5bC*=qNZ0~U}`zH zqWg%jlq=*)c9GSx6wwP(ElZ`A3sWs1Y5{63Kx(-#)dGZiVTI?4WXb@kNM`r4^@lEFfwDYArzKz@?RDy`=_1y|gk$zrO@h%cYgc^4+Zq zI~CP$SF$UtlAVzJc7^*Q8OJ+ONiAHS`UOZWK&=HxTP{!C0)%^c>J}iiTyEV$;HzF) z$!^zLDiXY|tW;v&SE))#{-xCRr^lWwG|^x~wc1K>b7&O_FtAT~~}BuBi*l3IR-tCF{PXE{l_{ zt>_*ZMUs61ZSj3I9t51&%9?lp=gXC7vJ!AaVP~itWnc z(_#YkHV268jTMdY$P%Z&Un-7)8EF0NcwvT8{eB%;0`pDcwYdKPVgxu@%p)^EO&1d!#;8K8c| zT;X{uzS+L5uBkv67?TEs3xsd0=xqX!Z?u0@?`Qca8yJ^DDMa>1l>z#8#Lh+a&PsN- z&HJ5^yfX!ton=ODuEc#W@fXOr1nWr-5aG?00h+e}nL3**_R-$XGRy9=aR^n?L7*mL zz*uT0*g&XvRpP~!tMJ&$J(cWd79(Ul-&3j7;%h6F9%9nH$wwecf$=LxDWqlhR^nHV z0Z8FiHNPTAp?VC6>{c_EQqX;Auz-*S>hT4T^Ea@aU-Xb9^ZwLtAcTSAV!r`Vb$?}| z9-{P+tn$ZoS&0$ABr7rE$8}kp^gzWvKIsu-1TZEvHB6Ks@j%7CKItJr;wNcPf=~?9 zGYTNGKdIQ2OAm<=4_54s1%xb62PF{M2P-4x7m(sLV#GsrBP2!uYtafRMfjn*5d?{! zR`iEZ6iSQ$>IeoR`_qd4@`*LR`tu6!IqO|hJoBkkVxm-OkY0TxO+OH7fEsFmsHr`Y zVoihe&7*1hffUB26HU^skLuDHfS_NbhJ%m=YQurC)EeF(z4}<1xgcbL8WI{fe?j=M zO1x|m#dPcAWog_T0F%Yit&i6Yrdyv#eclke6{sgzKxCh&*a=pH^y-r-8iFtasQZ6v zm2Q19h4TjK)?eDPOrdluFy?jwk^QAEykaoj`c&OQ@jp=4DhiQ(Dy>xw68fL6>y=&| z22P5lTc1Ypk=0eJSFT;TUDGQI?i+h}KCdN|4|q3zK3sD5EITtk-`(2Y)X{T_hM|@^U*xa){9VLh!>uEajR;G zXW)7fgOo;5Z&o`kA_RdFjbY<^bLD4c(0a4-2~D5zZoxNN#^%kplstxzZ{ZseD}&d< z?X^}1=tG9AMQlAmsX9iJ@53dRm5HS7J2QE*wQ$3{mNm$aOnt!_j_TyKx_(@7Vg$=3iH)530sJwrOFY)o-Hfa|~(kyv>o7BVi?~}}Ax9R)aL!7); zSNH+7lE+@t4=8zzML(eAvD4%+?h#fU@7lH66S0}>GeOBkj7r~H#P{~7oLwd#4(JQ9 zJY`q-HhYy30If}a4Wwo=LVa7CKFdfUyf%B5k%#RHKd^L)yoNfkbP9vj4=kO+?$Qq| zox%%vgZMp;>Sug{WDg}yVQ1+F^@|Rx34}LDUo!*oAl_hmmJtANu)Qn`f;YG|KGY|I z&<1a-&oaj6@!CsYa=PQ&OJ9=5`1aD5Y$$zu>B}0Am0%GUrB@JkdFtn;g|ioc5#HfQk7;qOsYwYQ9K~WsfKNoLE23A_f3; z<4he0!EqKbK&TsM5knRTUmI`cgW%Pl02lMeoB5Q%8*k>zO5!J!)rs{JI5`q~DS|d3 ze(V_$vSb^ zkXCQ6?=cEM-}Wr=q7Q`N_8s~zF9>zplaDBkqM5GK1m!vGnb8bV-FTDdXWGNOAb2yU zPK#!+n9s@Ps~HkX=h&Y*sE8SJ?e7f$&^MQiOUwWvICqTBX%On>^7jVB47oq@^YmR# z0rHacJab39P4e^XDO?b|dEB;$H&|U4W{cDeiOdVRU=@9`jrKc`qf5mAI%Nmzl(?t% zJ2dJ-07BgkTsw<8GKnSGPElP=9ys(%Z0(hd+b=cm0%+D!Hfy3irr&-kKXf;QzNO|} zNxuD#W(EK=b~H0$Chm7MGeFRGG&4w2zEsP;tj&@Y;7g2HX_lmjeis`w0L<9Myc_dH zzYE72Vg?9(yV#f^k=ZTVL)FR3u$#@4Mp3uBMK=KIcDD)8$Wa55yARXwMj`rkC%lQ9 z*yiq)9i;kX$=Hj#X)&)!^zCgk4?uJGwwVV)aBrJ=Ak^(`Gf!SH^!r$b0>F%Y99vy! zNPNH0@6%gDX?(uW?_()Slgzw*-8>VNWqV&6GZdk7-yt(o8P)AOXRcWZ*pJLp^-H|j z&pM?^@?F25JZ>*Xtw0fF|DdfS3)$big%{xMU-D!s?r&=WWzhDwwSZgJ!XLoRqE^TK z6G#&xil7}}f7$~C?Ew4J9yR``OW_Z+AqRkWV9ArQeqf^xIm+N2$dHr1;aK6|Y?=0r zEXxO5UpGtN9Ad2op!^UrYB36g;32J=LxE6tNcw9Q@*tx>)WTbQj}fHigQU#D8wlQ^ z7T%hrZw|A;1i*ENaa5+&iAN6OnA8wT4&y^2Lnt|n&{`9hH%r7kysTWh>hQ91>8iua z%B80cFDs|Iud-=TmC{pREostIU*!`ZD}(k`eQ%iB5Ilk}MWv_)!beBg^a8;dMd>6`ik&#|n0;28stRpViaua4E_KoClfWy7aN;;Z90LD9;@ zJI6`Dk`JBrDd^)kkW)$Avw|}nl@x!hP#+`6{>`soTd0CK?sy&X6v2|8sFlz@AJZa69#zXn6{1WqJXB}wv$rKH5f6O*KD+x&?LN?(x4t<28Uz7RjI zwT5EHpoI(Y?9v`Qy~AQY_9=?bFg6gN5MAA%x`sjVO+PvPuARRUH!`8k*TNJe<8 z-He2!jBjaGkiJ-B&rHXk-hh%u>5Da`M*3o{E}k`1N@uLKcY#4juGRIDGWud&sZ#o4 zU8z#~VqK|{zBsisOZwteT|dwveQ_#3MNlK@i_`37Y@SP!{nu$~3X;+nr?KX$Bz_Wa?wp#*RdW)GN1iyaYPPLpradg?j`*yt|-?!0sKI8AZ>xTT9ys~)I zt*%4s#{pPtWA7X5;05W%l&Q#x-5W8Ls@LtzOxn z`%bzkZ_Mj|Ypa+3!4>^mTfN4H1Nay$8hnM-Zc;_lz%zt3XuI;`TL~ z47!%`?@!5~YbpQ!ltkHA>*NJ?mh?>EF3(=RR209wy(=Y`CsH(uAS* z$8h508)kon2f)}J#y2InNcL1R(uVbDOpt1Z_3Ra^!I}}-s924(Yy_WUN;M?@qcZ#B zRMIu0_?Z(NAux|RH}*pXxp~zA2a_kmV5$e_87|}H1Qu} zxu+)nV=VV16Cal?)z->P9fx`H8{U|SPwYP8_I$&HiI)Co3O+GjiN=JSR!-!^WREBh zc~W=t9r<2UCfPEjsrMv1nbFjH5^2v0k(Z;AscyO-d`S1^d-4I@-@7BIbP_##4@4Wg!LJa}mcBc@pnCg0lmPQKXCB3WrW`S!yQv`&79Ye*z*^AluY z-;PV}=~^aB?RF(kys}-hrWHuS+m2jnU2HRgGrMoRGe2#@Ok3aMcX)p0&^YfgR+iYA z95!zk<>{2!-Agy;GiJ@UJc7@XP%_){i1=ZipUqQUYKi1Tv-vPVzJlia1-2U9cWuu1 zn>Dv&^GS}M%MGW>%i=S)h*4SOVb3Lm?jG61yZPPo@5=X_z}Gh>FJW&!zkpD-9L&sT z2|6Uo6ZRH%Z@nu&Y}UfEHhyL07jn(5@)8Rd_R*6mKJJ6PkaLy;Bb!*bsC(DD^RG-; zWTTgRY~&a9NF68VGK;w7ke_4)iHJ+OpS?Rjpthuxm%zBBN3u>1WR~#r44FJ^+;+-# z(@`tS#!fbQ;>K;~`c{p8-Pw%SZQRa$HYCRDHg4yEy0_GA+|DEuvzi)gAGb^QZCmm= z6Lv9Q>HcjO+dS+3Z5PImv{t?a^t*Pq+>?K0$*wk6;unE_*Z6puR3{^ISJE1>p8eh) z**FqsaTC`htdV5cw5f!o1+b=s@6-n3c zN17}K#f{zp-51`A`%{{(JG=wzxD$jy2QYb;MtR5wb@$nt?=;~ct5>&n2MtL(LW$T1 zk)F%1BISz3AKbluYkp|$;8I@Z<-rzYkU;)mU2KsD9Kv0KxE{bLI>dT8Ub*-~dgv$u zi6pl^5(?$Qg*;sc3;^1p=3WrB9V#bWs#E6Wp_=2tL-H`2kbsDKn4k=o!`SL6O{V8z zq@~g=Y*>%TPEc>k#5lsX;c>sZOfy-1zTYo1GaBN1Cw^J{v?6U-mkrRPZn@p?N7~sl z05guX(G;&b{E^lfAj~+@&YtC>gFm*Shq?f0N0l^Dd6aDzDC76O{wUio$fXCVxq1r# z@92^zE6maSs8L!?nMj(m=7+VZk>;^gJF-K7lLatpOwc1YIFdq1jDoP3P+ zX}lNl$Jjarf_IFqQ*tXp60czx0PWbaI++Z|GKN*91ixc#mn`=qr1ZM~1VB43(m3G( z!aFYh&N?D^$Jqu{?n+X+k6;6BU10$Mgq9T+J3%N}VTaowl&sLq4j!@ac>Vhu0#I^% zQo?0A4LY8;TGVKXiO09c?S+V<<#?XGlHtYq#Yx#2I=n>cBnF|3@_2r6vh_CrqfWMd z10i^_^|u~hoNV1Kf5XAAl#kh^;`lclfEw6AQfT`_5N52j{b8dFuT?fo093ciIsgQ1 z6~kUt%79vB!=zEbDW#P%x=xYWwIJ{QdFabeZ-9wiX z5VX}C9;s8PU`<(_jIK4Ml`@6b*c7fc#^JT5td8EWzx9G$GQ8H7)yeQ$+e1e>WmLD; zPDL7Jc&+2^SsO0HYh78LjIMR@$5YZ&Syb1tB&(GT{H?H4ZM|uX1M1Y0Cj;u#_^UPH z$$&alx2qrq)M<9~2B5mrN-Je-oYo`tfsCNj`1!F`$Etd|op&_KZxa3K+mt+6T~9Ba ODXZ)0MgMfK_x}LxgUx#Y diff --git a/clients/python/generate-proto.ps1 b/clients/python/generate-proto.ps1 index b336eee..a98eb29 100644 --- a/clients/python/generate-proto.ps1 +++ b/clients/python/generate-proto.ps1 @@ -1,15 +1,52 @@ Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' +# Pinned generator baseline. The committed Python bindings are stamped with +# GRPC_GENERATED_VERSION = 1.80.0 and "Protobuf Python Version: 6.31.1". A newer grpcio-tools +# stamps a higher GRPC_GENERATED_VERSION than the pinned grpcio runtime (pyproject.toml: +# grpcio>=1.80,<2), which makes _pb2_grpc import raise at runtime and breaks pytest. Regeneration +# MUST use this exact grpcio-tools version. See docs/ClientProtoGeneration.md and the repo memory +# project_python_client_regen_pin. +$PinnedGrpcioToolsVersion = '1.80.0' + $repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..') $protoRoot = Join-Path $repoRoot 'src\ZB.MOM.WW.MxGateway.Contracts\Protos' $outputRoot = Join-Path $PSScriptRoot 'src\zb_mom_ww_mxgateway\generated' -$python = 'C:\Users\dohertj2\AppData\Local\Programs\Python\Python312\python.exe' -if (-not (Test-Path $python)) { - throw "Python was not found at $python. See docs/ToolchainLinks.md." +function Resolve-Python { + # Prefer python on PATH (python on Windows, python3 on Linux/macOS), then the documented + # Windows install location. Avoids the previous hard-coded per-machine path. + foreach ($name in @('python', 'python3', 'python.exe')) { + $cmd = Get-Command $name -ErrorAction SilentlyContinue + if ($null -ne $cmd) { + return $cmd.Source + } + } + + if ($env:LOCALAPPDATA) { + $documentedPath = Join-Path $env:LOCALAPPDATA 'Programs\Python\Python312\python.exe' + if (Test-Path $documentedPath) { + return $documentedPath + } + } + + throw 'Could not find python on PATH. See docs/ToolchainLinks.md.' } +function Assert-GrpcioToolsVersion { + param([string]$Python) + + $version = (& $Python -c 'import grpc_tools; from importlib.metadata import version; print(version("grpcio-tools"))').Trim() + if ($version -ne $PinnedGrpcioToolsVersion) { + throw "grpcio-tools $version is installed, but regeneration is pinned to $PinnedGrpcioToolsVersion. " + + "Install the pin (python -m pip install 'grpcio-tools==$PinnedGrpcioToolsVersion') before regenerating, " + + "or the generated bindings will stamp a GRPC_GENERATED_VERSION the pinned grpcio runtime rejects." + } +} + +$python = Resolve-Python +Assert-GrpcioToolsVersion -Python $python + New-Item -ItemType Directory -Path $outputRoot -Force | Out-Null Get-ChildItem -Path (Join-Path $outputRoot '*_pb2.py') -File | Remove-Item Get-ChildItem -Path (Join-Path $outputRoot '*_pb2_grpc.py') -File | Remove-Item @@ -21,3 +58,7 @@ Get-ChildItem -Path (Join-Path $outputRoot '*_pb2_grpc.py') -File | Remove-Item mxaccess_gateway.proto ` mxaccess_worker.proto ` galaxy_repository.proto + +if ($LASTEXITCODE -ne 0) { + throw "grpc_tools.protoc failed with exit code $LASTEXITCODE." +} diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index bd09fbb..2b0e748 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -38,6 +38,9 @@ Issues = "https://gitea.dohertylan.com/dohertj2/mxaccessgw/issues" [project.optional-dependencies] dev = [ + # Runtime range for local dev, but REGENERATION is pinned to grpcio-tools==1.80.0 + # (protobuf 6.31.1): a newer version stamps a GRPC_GENERATED_VERSION the pinned grpcio + # runtime rejects and breaks pytest. generate-proto.ps1 asserts the exact pin. "grpcio-tools>=1.80,<2", "pytest>=9,<10", "pytest-asyncio>=1.3,<2", diff --git a/docs/ClientProtoGeneration.md b/docs/ClientProtoGeneration.md index 306209d..597869b 100644 --- a/docs/ClientProtoGeneration.md +++ b/docs/ClientProtoGeneration.md @@ -48,25 +48,55 @@ session. ## Descriptor Publishing Run this command after changing either source `.proto` file or the client proto -manifest: +manifest, with the **pinned protoc 34.1** (see [Toolchain Links](./ToolchainLinks.md)): ```powershell -scripts/publish-client-proto-inputs.ps1 +pwsh -File scripts/publish-client-proto-inputs.ps1 ``` The script writes `clients/proto/descriptors/mxaccessgw-client-v1.protoset` with imports and source information included. The descriptor is a generated artifact; do not edit -it by hand. +it by hand. Generating the committed artifact requires the pinned protoc version +so the checked-in bytes are reproducible; the script fails fast on a version +mismatch when generating. Use the check mode in CI or before committing: ```powershell -scripts/publish-client-proto-inputs.ps1 -Check +pwsh -File scripts/publish-client-proto-inputs.ps1 -Check ``` -`-Check` rebuilds the descriptor in a temporary path and fails when the checked -in descriptor is stale. +`-Check` rebuilds the descriptor and fails when the checked-in descriptor is +stale. The comparison normalizes both the committed and freshly built descriptor +through the same protoc with `source_code_info` stripped, so it is tolerant of +protoc-version encoding drift and does not false-fail across protoc releases +(it warns, rather than fails, when protoc is off the pin). + +The gateway test project carries an independent, protoc-free freshness guard: +`ClientProtoInputTests.Descriptor_ContainsEveryContractMessageAndField` reflects +over the in-process contract descriptors and fails if any contract message or +field is missing from the committed protoset. This is the primary CI gate for +descriptor staleness; a red test means "regenerate and commit the protoset." + +### Pinned generator versions + +Regeneration is reproducible only with the pinned toolchain. Regenerating with a +different version silently produces incompatible or noisy output, so the per-client +scripts assert the pin and resolve tools from `PATH`: + +| Generator | Pinned version | Guard | +|-----------|----------------|-------| +| protoc (descriptor set) | 34.1 | version assertion in `scripts/publish-client-proto-inputs.ps1` | +| `Grpc.Tools` (C# `Generated/`) | 2.80.0 (contracts csproj) | `scripts/check-codegen.ps1` git-diff of `Generated/` | +| `grpcio-tools` (Python) | 1.80.0 (protobuf runtime 6.31.1) | version assertion in `clients/python/generate-proto.ps1` | +| protobuf / grpc-java (Java) | `protobufVersion` / `grpcVersion` in `clients/java/build.gradle` | `checkGeneratedClean` gradle task | + +A newer `grpcio-tools` stamps a `GRPC_GENERATED_VERSION` above the pinned grpcio +runtime and breaks Python `pytest`; the Java protobuf plugin rewrites +`MxaccessGateway.java` with spurious protobuf-runtime-version churn on every build +(revert that one file when no `.proto` changed — see +[Gateway Testing](./GatewayTesting.md) "Continuous Integration"). ## Output Directories diff --git a/docs/Contracts.md b/docs/Contracts.md index 1690ab7..f0ee3d2 100644 --- a/docs/Contracts.md +++ b/docs/Contracts.md @@ -97,6 +97,24 @@ behavior. Generated C# output is written to `src/ZB.MOM.WW.MxGateway.Contracts/Generated/`. Do not hand-edit generated files. +The `Generated/` C# is **tracked and must be committed after any `.proto` change.** The +contracts project `Compile Remove`s `Generated/**/*.cs` and has `Grpc.Tools` regenerate them, +but `Grpc.Tools` skips regeneration when the committed output looks up to date. On the net10 +build the freshly regenerated code is compiled, so a stale `Generated/` is invisible there — +but the **net48 worker** consumes the committed `Generated/` and fails to build with `CS0246` +on any new type when the checked-in code lags the `.proto`. So after editing a `.proto` you must +regenerate and commit `Generated/`. If a build does not pick up your proto change, delete the +stale output to force a full regeneration: + +```bash +rm src/ZB.MOM.WW.MxGateway.Contracts/Generated/*.cs +dotnet build src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj +``` + +`scripts/check-codegen.ps1` enforces this in CI (it force-regenerates and fails on any +`git diff` against the committed `Generated/`); the Windows CI job's net48 worker build is the +definitive guard. + Client generation inputs are published through `clients/proto/proto-inputs.json` and the descriptor set under `clients/proto/descriptors/`. See @@ -124,12 +142,25 @@ gateway and test projects: dotnet build src/ZB.MOM.WW.MxGateway.slnx ``` -Regenerate the client descriptor after changing either `.proto` file: +Regenerate the client descriptor after changing either `.proto` file, using the pinned protoc +(34.1, see [Toolchain Links](./ToolchainLinks.md)): ```bash -powershell -ExecutionPolicy Bypass -File scripts/publish-client-proto-inputs.ps1 +pwsh -File scripts/publish-client-proto-inputs.ps1 ``` +Freshness is guarded two ways so a skipped regeneration cannot ship silently: + +- `ClientProtoInputTests.Descriptor_ContainsEveryContractMessageAndField` (gateway test project) + reflects over the in-process contract descriptors and fails if any message or field is missing + from the committed protoset. It is semantic (symbol presence), needs no protoc, and runs in the + Linux CI. A red test means "regenerate and commit the protoset." +- `pwsh -File scripts/publish-client-proto-inputs.ps1 -Check` rebuilds the descriptor and compares + it to the committed one. The comparison normalizes both sides through the same protoc with + `source_code_info` stripped, so it does not false-fail across protoc releases. +- `pwsh scripts/check-codegen.ps1` runs both the descriptor check and the `Generated/`-clean check + together; it is the single guard the CI pipeline invokes. + ## Related Documentation - [Client Proto Generation](./ClientProtoGeneration.md) diff --git a/docs/GatewayTesting.md b/docs/GatewayTesting.md index bdb04ba..6743272 100644 --- a/docs/GatewayTesting.md +++ b/docs/GatewayTesting.md @@ -408,6 +408,34 @@ Run the gateway test project after shared gateway test infrastructure changes: dotnet test src/ZB.MOM.WW.MxGateway.Tests/ZB.MOM.WW.MxGateway.Tests.csproj ``` +## Continuous Integration + +CI runs on Gitea Actions (`.gitea/workflows/ci.yml`; origin is Gitea at +`gitea.dohertylan.com`) on every push and pull request. The pipeline is split by +runtime because the x86 Worker cannot build on Linux: + +- **`portable`** (Linux runner) — builds `src/ZB.MOM.WW.MxGateway.NonWindows.slnx`, + runs the codegen/descriptor freshness guard (`scripts/check-codegen.ps1`), runs the + gateway fake-worker tests, and builds/tests the clients that run on Linux: .NET client + build, Go (`gofmt` + `go build` + `go test`), Rust (`cargo fmt --check` + `cargo test` + + `cargo clippy -D warnings`), and Python (`pytest`). +- **`java`** (Linux, JDK 17) — `gradle test`. The protobuf gradle plugin rewrites + `MxaccessGateway.java` with spurious protobuf-runtime-version churn on every build, so + when no `.proto` changed the job reverts that one file (`git checkout`) before asserting + the generated tree is clean. The dev Mac has no JRE, so Java verification is CI-only. +- **`windows`** (self-hosted windev runner) — the only host that builds the **x86 / + net48 Worker and Worker.Tests**; these are Windows-only and out of scope for the Linux + jobs. This job also proves the net48 build against the committed `Generated/` (the + definitive guard for the "regenerate and commit `Generated/`" rule). +- **`live-mxaccess`** (self-hosted, MXAccess-installed) — scheduled only + (`MXGATEWAY_RUN_LIVE_MXACCESS_TESTS=1`); never gates a push. + +The freshness guard `scripts/check-codegen.ps1` fails the build when the committed +client descriptor set or the C# `Generated/` no longer matches the current `.proto` +sources — the codegen drift class this repo has hit repeatedly (stale client +descriptors, net48 `CS0246` on unregenerated protos). See +[Client Proto Generation](./ClientProtoGeneration.md) and [Contracts](./Contracts.md). + ## Related Documentation - [Cross-Language Smoke Matrix](./CrossLanguageSmokeMatrix.md) diff --git a/scripts/check-codegen.ps1 b/scripts/check-codegen.ps1 new file mode 100644 index 0000000..62fe57f --- /dev/null +++ b/scripts/check-codegen.ps1 @@ -0,0 +1,96 @@ +#!/usr/bin/env pwsh +# Codegen freshness guard for CI (IPC-01, IPC-19, IPC-20, CLI-02). +# +# Three checks, all Linux/macOS-runnable (no Server build, no x86 worker): +# 1. Published client descriptor set matches the current .proto sources (delegates to +# publish-client-proto-inputs.ps1 -Check, which normalizes source_code_info so it is +# protoc-version tolerant). +# 2. The committed C# under Contracts/Generated matches a fresh regeneration. Grpc.Tools is +# pinned in the contracts csproj, so a clean checkout regenerates byte-identical output; a +# non-empty git diff means a .proto was edited without regenerating and committing Generated/ +# (which breaks the net48 worker build with CS0246 — see docs/Contracts.md). +# 3. The Rust crate's vendored protos (clients/rust/protos/*.proto — build inputs that make the +# crate buildable outside the repo, CLI-02) are byte-identical to the canonical Contracts +# protos. A drift means a .proto was edited without refreshing the vendored copies, which would +# publish a stale wire contract to crate consumers while the in-repo build stays correct. +# +# The x86 Worker + Worker.Tests are Windows-only and are guarded by the Windows CI job, not here. + +[CmdletBinding()] +param() + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot '..') +$generatedDir = Join-Path $repoRoot 'src/ZB.MOM.WW.MxGateway.Contracts/Generated' +$contractsProject = Join-Path $repoRoot 'src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj' +$failures = New-Object System.Collections.Generic.List[string] + +Write-Host '== Check 1/2: client descriptor set freshness ==' +try { + & (Join-Path $PSScriptRoot 'publish-client-proto-inputs.ps1') -Check + if ($LASTEXITCODE -ne 0) { + $failures.Add('Client descriptor set is stale (publish-client-proto-inputs.ps1 -Check failed).') + } +} +catch { + $failures.Add("Descriptor freshness check failed: $($_.Exception.Message)") +} + +Write-Host '' +Write-Host '== Check 2/2: Contracts/Generated matches a fresh regeneration ==' +try { + # Force a full regeneration: Grpc.Tools skips regen when the committed .cs look up to date, so + # remove them first (the documented "del Generated/*.cs to force regen" trick). + if (Test-Path $generatedDir) { + Get-ChildItem -Path $generatedDir -Filter '*.cs' -Recurse -File | Remove-Item -Force + } + + & dotnet build $contractsProject -c Release --nologo | Out-Host + if ($LASTEXITCODE -ne 0) { + $failures.Add('Contracts project failed to build during codegen check.') + } + + $diff = (& git -C $repoRoot status --porcelain -- 'src/ZB.MOM.WW.MxGateway.Contracts/Generated').Trim() + if (-not [string]::IsNullOrEmpty($diff)) { + Write-Host $diff + $failures.Add('Contracts/Generated differs from a fresh regeneration. Run `dotnet build` on the contracts project and commit Generated/ (required for the net48 worker build).') + } +} +catch { + $failures.Add("Generated codegen check failed: $($_.Exception.Message)") +} + +Write-Host '' +Write-Host '== Check 3/3: Rust vendored protos match canonical Contracts protos ==' +try { + $canonicalProtoDir = Join-Path $repoRoot 'src/ZB.MOM.WW.MxGateway.Contracts/Protos' + $vendoredProtoDir = Join-Path $repoRoot 'clients/rust/protos' + foreach ($vendored in Get-ChildItem -Path $vendoredProtoDir -Filter '*.proto' -File) { + $canonical = Join-Path $canonicalProtoDir $vendored.Name + if (-not (Test-Path $canonical)) { + $failures.Add("Vendored Rust proto has no canonical counterpart: $($vendored.Name).") + continue + } + $vendoredHash = (Get-FileHash -Algorithm SHA256 $vendored.FullName).Hash + $canonicalHash = (Get-FileHash -Algorithm SHA256 $canonical).Hash + if ($vendoredHash -ne $canonicalHash) { + $failures.Add("Rust vendored proto drifted from canonical: clients/rust/protos/$($vendored.Name). Refresh it from src/ZB.MOM.WW.MxGateway.Contracts/Protos/$($vendored.Name).") + } + } +} +catch { + $failures.Add("Rust vendored proto check failed: $($_.Exception.Message)") +} + +Write-Host '' +if ($failures.Count -gt 0) { + Write-Host 'Codegen freshness check FAILED:' -ForegroundColor Red + foreach ($failure in $failures) { + Write-Host " - $failure" -ForegroundColor Red + } + exit 1 +} + +Write-Host 'Codegen freshness check passed.' -ForegroundColor Green diff --git a/scripts/publish-client-proto-inputs.ps1 b/scripts/publish-client-proto-inputs.ps1 index 737ad7c..29deb5c 100644 --- a/scripts/publish-client-proto-inputs.ps1 +++ b/scripts/publish-client-proto-inputs.ps1 @@ -6,23 +6,46 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" +# Pinned protoc toolchain version used to generate the committed client descriptor set. +# See docs/ToolchainLinks.md. Generating (committing) the canonical artifact must use this +# exact version so the checked-in bytes are reproducible. The -Check comparison is made +# tolerant of protoc-version encoding drift (chiefly source_code_info, see IPC-20) by +# normalizing both the committed and the freshly built descriptor through the *same* protoc +# with source info stripped, so it does not false-fail across protoc releases. +$PinnedProtocVersion = "34.1" + $repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..") $protoRoot = Join-Path $repoRoot "src/ZB.MOM.WW.MxGateway.Contracts/Protos" $manifestPath = Join-Path $repoRoot "clients/proto/proto-inputs.json" $descriptorPath = Join-Path $repoRoot "clients/proto/descriptors/mxaccessgw-client-v1.protoset" +$protoFiles = @("mxaccess_gateway.proto", "mxaccess_worker.proto", "galaxy_repository.proto") function Resolve-Protoc { - $pathCommand = Get-Command "protoc.exe" -ErrorAction SilentlyContinue - if ($null -ne $pathCommand) { - return $pathCommand.Source + # Prefer protoc on PATH (protoc on Linux/macOS, protoc.exe on Windows), then fall back to + # the documented winget install location on Windows. + foreach ($name in @("protoc", "protoc.exe")) { + $pathCommand = Get-Command $name -ErrorAction SilentlyContinue + if ($null -ne $pathCommand) { + return $pathCommand.Source + } } - $documentedPath = Join-Path $env:LOCALAPPDATA "Microsoft/WinGet/Packages/Google.Protobuf_Microsoft.Winget.Source_8wekyb3d8bbwe/bin/protoc.exe" - if (Test-Path $documentedPath) { - return $documentedPath + if ($env:LOCALAPPDATA) { + $documentedPath = Join-Path $env:LOCALAPPDATA "Microsoft/WinGet/Packages/Google.Protobuf_Microsoft.Winget.Source_8wekyb3d8bbwe/bin/protoc.exe" + if (Test-Path $documentedPath) { + return $documentedPath + } } - throw "Could not find protoc.exe. See docs/ToolchainLinks.md for the documented protobuf toolchain path." + throw "Could not find protoc on PATH. See docs/ToolchainLinks.md for the documented protobuf toolchain (protoc $PinnedProtocVersion)." +} + +function Get-ProtocVersion { + param([string]$Protoc) + + # `protoc --version` prints e.g. "libprotoc 34.1". + $raw = (& $Protoc --version) | Select-Object -First 1 + return ($raw -replace '^libprotoc\s+', '').Trim() } function Ensure-Directory { @@ -33,7 +56,29 @@ function Ensure-Directory { } } -function Compare-FileBytes { +function New-SourceInfoFreeDescriptor { + # Re-emits an existing descriptor set with source_code_info stripped, by round-tripping it + # through protoc with --descriptor_set_in and *without* --include_source_info. Because both + # the committed and freshly built descriptors are normalized through the same protoc binary, + # any protoc-version-specific encoding cancels out and the byte comparison is stable. + param( + [string]$Protoc, + [string]$InputDescriptor, + [string]$OutputDescriptor + ) + + & $Protoc ` + "--descriptor_set_in=$InputDescriptor" ` + "--include_imports" ` + "--descriptor_set_out=$OutputDescriptor" ` + @protoFiles + + if ($LASTEXITCODE -ne 0) { + throw "protoc normalization (--descriptor_set_in) failed with exit code $LASTEXITCODE." + } +} + +function Test-FileBytesEqual { param( [string]$ExpectedPath, [string]$ActualPath @@ -66,31 +111,67 @@ foreach ($output in $manifest.generatedOutputs.PSObject.Properties.Value) { } $protoc = Resolve-Protoc -$outputPath = $descriptorPath -if ($Check) { - $outputPath = Join-Path ([System.IO.Path]::GetTempPath()) ("mxaccessgw-client-v1-" + [System.Guid]::NewGuid().ToString("N") + ".protoset") -} +$protocVersion = Get-ProtocVersion $protoc + +if ($Check) { + # Version drift no longer produces a false "stale" failure: the comparison is normalized + # below. A mismatch is surfaced as a warning so it is visible without blocking the gate. + if ($protocVersion -ne $PinnedProtocVersion) { + Write-Warning "protoc version '$protocVersion' differs from the pinned '$PinnedProtocVersion'. The freshness comparison is normalized and tolerant of this, but regeneration must use the pinned version (see docs/ClientProtoGeneration.md)." + } + + $freshDescriptor = Join-Path ([System.IO.Path]::GetTempPath()) ("mxaccessgw-client-v1-fresh-" + [System.Guid]::NewGuid().ToString("N") + ".protoset") + $committedNormalized = Join-Path ([System.IO.Path]::GetTempPath()) ("mxaccessgw-client-v1-committed-" + [System.Guid]::NewGuid().ToString("N") + ".protoset") + + try { + # Fresh descriptor from the current .proto sources, source info omitted (already normalized). + & $protoc ` + "--proto_path=$protoRoot" ` + "--include_imports" ` + "--descriptor_set_out=$freshDescriptor" ` + @protoFiles + + if ($LASTEXITCODE -ne 0) { + throw "protoc failed with exit code $LASTEXITCODE." + } + + if (-not (Test-Path $descriptorPath)) { + throw "Committed descriptor '$descriptorPath' does not exist. Run scripts/publish-client-proto-inputs.ps1 to create it." + } + + # Normalize the committed descriptor (which carries source info) through the same protoc. + New-SourceInfoFreeDescriptor -Protoc $protoc -InputDescriptor $descriptorPath -OutputDescriptor $committedNormalized + + if (-not (Test-FileBytesEqual -ExpectedPath $committedNormalized -ActualPath $freshDescriptor)) { + throw "Client proto descriptor is stale. Run scripts/publish-client-proto-inputs.ps1 with the pinned protoc ($PinnedProtocVersion) and commit the updated descriptor." + } + + Write-Host "Client proto descriptor is up to date." + } + finally { + foreach ($temp in @($freshDescriptor, $committedNormalized)) { + if (Test-Path $temp) { + Remove-Item -LiteralPath $temp + } + } + } +} +else { + # Generating the canonical committed artifact must use the pinned protoc so the bytes are reproducible. + if ($protocVersion -ne $PinnedProtocVersion) { + throw "protoc version '$protocVersion' does not match the pinned toolchain version '$PinnedProtocVersion'. Install the pinned protoc (see docs/ToolchainLinks.md) before regenerating the committed descriptor, or run with -Check to only verify freshness." + } -try { & $protoc ` "--proto_path=$protoRoot" ` "--include_imports" ` "--include_source_info" ` - "--descriptor_set_out=$outputPath" ` - "mxaccess_gateway.proto" ` - "mxaccess_worker.proto" ` - "galaxy_repository.proto" + "--descriptor_set_out=$descriptorPath" ` + @protoFiles if ($LASTEXITCODE -ne 0) { throw "protoc failed with exit code $LASTEXITCODE." } - if ($Check -and -not (Compare-FileBytes -ExpectedPath $descriptorPath -ActualPath $outputPath)) { - throw "Client proto descriptor is stale. Run scripts/publish-client-proto-inputs.ps1 and commit the updated descriptor." - } -} -finally { - if ($Check -and (Test-Path $outputPath)) { - Remove-Item -LiteralPath $outputPath - } + Write-Host "Wrote client proto descriptor to $descriptorPath (protoc $protocVersion)." } diff --git a/src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj b/src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj index b23971b..8cfb3fe 100644 --- a/src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj +++ b/src/ZB.MOM.WW.MxGateway.Contracts/ZB.MOM.WW.MxGateway.Contracts.csproj @@ -23,6 +23,13 @@ + diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Contracts/ClientProtoInputTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Contracts/ClientProtoInputTests.cs index 4848871..981b54f 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Contracts/ClientProtoInputTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Contracts/ClientProtoInputTests.cs @@ -1,5 +1,6 @@ using System.Text.Json; using Google.Protobuf; +using Google.Protobuf.Reflection; using ZB.MOM.WW.MxGateway.Contracts; using ZB.MOM.WW.MxGateway.Contracts.Proto; @@ -7,6 +8,102 @@ namespace ZB.MOM.WW.MxGateway.Tests.Contracts; public sealed class ClientProtoInputTests { + ///

+ /// Guards the published client descriptor set against silent staleness (IPC-01). Every message + /// and field compiled into the in-process contract (which the build regenerates from the current + /// .proto sources) must appear in the committed protoset. A missing symbol means the + /// descriptor was not regenerated after a proto change; run + /// scripts/publish-client-proto-inputs.ps1 and commit the refreshed protoset. + /// The check is semantic (symbol presence) rather than byte-wise, so it is independent of protoc + /// version and does not require protoc on the test runner. + /// + [Fact] + public void Descriptor_ContainsEveryContractMessageAndField() + { + DirectoryInfo repositoryRoot = FindRepositoryRoot(); + string descriptorPath = Path.Combine( + repositoryRoot.FullName, + "clients", + "proto", + "descriptors", + "mxaccessgw-client-v1.protoset"); + + Assert.True(File.Exists(descriptorPath), $"Expected descriptor set '{descriptorPath}' to exist."); + + FileDescriptorSet descriptorSet = FileDescriptorSet.Parser.ParseFrom(File.ReadAllBytes(descriptorPath)); + + HashSet publishedMessages = new(StringComparer.Ordinal); + HashSet publishedFields = new(StringComparer.Ordinal); + foreach (FileDescriptorProto file in descriptorSet.File) + { + foreach (DescriptorProto message in file.MessageType) + { + CollectPublishedSymbols(file.Package, message, publishedMessages, publishedFields); + } + } + + List missing = []; + foreach (FileDescriptor file in new[] { MxaccessGatewayReflection.Descriptor, MxaccessWorkerReflection.Descriptor }) + { + foreach (MessageDescriptor message in file.MessageTypes) + { + CollectMissingContractSymbols(message, publishedMessages, publishedFields, missing); + } + } + + Assert.True( + missing.Count == 0, + "Published client descriptor is stale; regenerate with scripts/publish-client-proto-inputs.ps1 and commit. " + + "Missing symbols: " + + string.Join(", ", missing)); + } + + private static void CollectPublishedSymbols( + string package, + DescriptorProto message, + HashSet messages, + HashSet fields) + { + string fullName = string.IsNullOrEmpty(package) ? message.Name : package + "." + message.Name; + messages.Add(fullName); + + foreach (FieldDescriptorProto field in message.Field) + { + fields.Add(fullName + "/" + field.Name); + } + + foreach (DescriptorProto nested in message.NestedType) + { + CollectPublishedSymbols(fullName, nested, messages, fields); + } + } + + private static void CollectMissingContractSymbols( + MessageDescriptor message, + HashSet publishedMessages, + HashSet publishedFields, + List missing) + { + if (!publishedMessages.Contains(message.FullName)) + { + missing.Add(message.FullName); + } + + foreach (FieldDescriptor field in message.Fields.InDeclarationOrder()) + { + string key = message.FullName + "/" + field.Name; + if (!publishedFields.Contains(key)) + { + missing.Add(key); + } + } + + foreach (MessageDescriptor nested in message.NestedTypes) + { + CollectMissingContractSymbols(nested, publishedMessages, publishedFields, missing); + } + } + /// Verifies that the proto inputs manifest declares current protocol versions and existing source files. [Fact] public void Manifest_DeclaresCurrentProtocolVersionsAndExistingInputs() From 8fb1a814d484bdf8dc1e16eae1e77b651fde2164 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 06:18:03 -0400 Subject: [PATCH 03/19] docs(archreview): P1 Wave 1 status (IPC-01/09/19/20, CLI-02 Done; TST-03 In review) --- archreview/remediation/00-tracking.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/archreview/remediation/00-tracking.md b/archreview/remediation/00-tracking.md index a6e5cb8..d19db7c 100644 --- a/archreview/remediation/00-tracking.md +++ b/archreview/remediation/00-tracking.md @@ -112,7 +112,7 @@ Full design + implementation for each row lives in the linked domain doc under i | ID | Sev | Tier | Eff | Dep | Status | Title | |---|---|:-:|:-:|---|---|---| -| IPC-01 | High | P1 | M | — | Not started | Published client descriptor set 7 weeks stale; nothing enforces freshness | +| IPC-01 | High | P1 | M | — | Done | Published client descriptor set 7 weeks stale; nothing enforces freshness | | IPC-02 | Medium | P1 | M | IPC-03 | Not started | Worker max frame size hard-coded, cannot follow gateway config | | IPC-03 | Medium | P1 | M | IPC-02 | Not started | gRPC max == pipe max with zero headroom; oversized write faults whole session | | IPC-04 | Medium | P1 | S | IPC-03 | Not started | `DrainEvents max_events=0` packs entire queue into one reply frame | @@ -120,7 +120,7 @@ Full design + implementation for each row lives in the linked domain doc under i | IPC-06 | Medium | P2 | S | — | Not started | `gateway.md` Worker Envelope sketch no longer matches the contract | | IPC-07 | Medium | P2 | S | — | Not started | `docs/Grpc.md` says six RPCs; there are seven | | IPC-08 | Medium | — | M | — | Not started | `WorkerCancel` defined and handled but never sent | -| IPC-09 | Medium | P1 | M | IPC-01 | Not started | Known codegen fragilities have no in-repo guards | +| IPC-09 | Medium | P1 | M | IPC-01 | Done | Known codegen fragilities have no in-repo guards | | IPC-10 | Low | — | S | — | Not started | Envelope `sequence` is write-only; monotonicity never validated | | IPC-11 | Low | — | S | — | Not started | No protocol version negotiation despite `supported_protocol_version` name | | IPC-12 | Low | — | S | — | Not started | Gateway frame writer has no write lock; integrity rests on undocumented invariant | @@ -130,8 +130,8 @@ Full design + implementation for each row lives in the linked domain doc under i | IPC-16 | Low | — | S | — | N/A | Correlation id carried twice per reply (Info) | | IPC-17 | Low | P2 | S | — | Not started | Two docs name the wrong Python generated-output directory | | IPC-18 | Low | — | S | IPC-01 | N/A | Generated-code hygiene verified good — preserve under change (positive) | -| IPC-19 | Low | P1 | S | IPC-01 | Not started | Generated/-must-be-committed rule for net48 undocumented and unguarded | -| IPC-20 | Low | P1 | S | IPC-01 | Not started | Descriptor `-Check` byte-compares source-info bytes; protoc-version-sensitive | +| IPC-19 | Low | P1 | S | IPC-01 | Done | Generated/-must-be-committed rule for net48 undocumented and unguarded | +| IPC-20 | Low | P1 | S | IPC-01 | Done | Descriptor `-Check` byte-compares source-info bytes; protoc-version-sensitive | | IPC-21 | Low | P2 | S | IPC-06 | Not started | `gateway.md` presents unimplemented `Session` RPC as live | | IPC-22 | Low | — | S | — | N/A | Public error-detail model stops at status codes plus prose (Info) | @@ -175,7 +175,7 @@ Full design + implementation for each row lives in the linked domain doc under i | ID | Sev | Tier | Eff | Dep | Status | Title | |---|---|:-:|:-:|---|---|---| | CLI-01 | High | P0 | M | — | Done | Go `Session.Events()` silently closes stream on 16-slot overflow | -| CLI-02 | High | P1 | M | — | Not started | Rust crate unbuildable outside the repo (`build.rs` path + `--no-verify`) | +| CLI-02 | High | P1 | M | — | Done | Rust crate unbuildable outside the repo (`build.rs` path + `--no-verify`) | | CLI-03 | High | P0 | M | — | Done | Rust `invoke` never validates HRESULT / MXSTATUS_PROXY | | CLI-04 | High | P2 | L | CLI-15 | Not started | Typed-command parity gap across all five clients | | CLI-05 | Medium | — | S | — | Not started | .NET session cannot be re-attached to an existing session id | @@ -215,7 +215,7 @@ Full design + implementation for each row lives in the linked domain doc under i |---|---|:-:|:-:|---|---|---| | TST-01 | High | P2 | L | TST-04 | Not started | Reconnect/replay has no e2e test and no client consumer | | TST-02 | High | P0 | M | TST-04 | Done | Reconnect owner re-validation not implemented | -| TST-03 | High | P1 | M | — | Not started | No CI exists | +| TST-03 | High | P1 | M | — | In review | No CI exists | | TST-04 | High | P2 | L | — | Not started | Session-resilience epic 16/28 tasks unfinished | | TST-05 | Medium | P1 | S | TST-03 | Not started | Real-worker control/COM paths verified opt-in only | | TST-06 | Medium | — | M | — | Not started | Dashboard live-data path untested | @@ -255,4 +255,5 @@ Findings the review flagged as one coordinated design pass — sequence them tog |---|---| | 2026-07-09 | Initial tracking doc generated from the six domain remediation designs. All 153 findings `Not started`. | | 2026-07-09 | P0 tier executed via parallel agents. GWC-01/02/03, TST-02, TST-12, CLI-01, CLI-03 → `Done` (NonWindows build clean; gateway 135/135 targeted tests pass, Go `go test ./...` + `-race` pass, Rust `test`/`clippy -D warnings` pass). | +| 2026-07-09 | P1 Wave 1 (CI + codegen freshness + Rust buildability) via parallel agents. IPC-01/09/19/20, CLI-02 → `Done`; TST-03 → `In review` (CI pipeline authored + YAML/layout-validated, but not yet executed on a Gitea runner — proven on first push). Verified on macOS: NonWindows build clean, `ClientProtoInputTests` 5/5, `publish-client-proto-inputs.ps1 -Check` exit 0, `cargo package` (no `--no-verify`) compiles standalone. Added a vendored-Rust-proto drift guard (Check 3) to `check-codegen.ps1` closing the CLI-02 static-copy risk. IPC-09 script guards not executed end-to-end (need pinned python/JRE toolchains); logic is PATH-resolution + version assertion. | | 2026-07-09 | WRK-01 → `Done`. Verified on Windows host (windev) via an isolated `origin/main` worktree: worker builds x86 clean, `StaRuntimeTests`+`WorkerPipeSessionTests` 33/33 pass. Fixed an `xUnit1030` build error (the new worker test used `.ConfigureAwait(false)` in `[Fact]` bodies) that the macOS tree could not surface. Also ran GWC-01's Windows-only `WorkerClientTests` on windev: 18/18 pass (incl. `ReadEventsAsync_SecondEnumerator_Throws`). All 8 P0 findings now `Done`. Not yet committed. | From 197731ae2d043105ae278bf5bee435ce0c1cdad1 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 06:37:10 -0400 Subject: [PATCH 04/19] =?UTF-8?q?chore(auth):=20consume=20ZB.MOM.WW.Auth?= =?UTF-8?q?=200.1.4=20=E2=80=94=20gain=20API-key=20expiry=20enforcement=20?= =?UTF-8?q?(archreview=20G-2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bump all four ZB.MOM.WW.Auth.* package refs 0.1.2 -> 0.1.4. The shared ApiKeyVerifier now rejects any key whose ExpiresUtc is in the past; existing keys have NULL expiry (never expire), so nothing breaks, and the auth SQLite DB auto-migrates to schema v3 (nullable expires_utc column) on first boot. Implement the two IApiKeyAdminStore members added in 0.1.3 (SetScopesAsync/SetEnabledAsync) on the test FakeApiKeyAdminStore. Build green; no new test failures (the macOS worker-pipe IPC failures are pre-existing/environmental, identical to the pre-bump baseline). --- ...B.MOM.WW.MxGateway.IntegrationTests.csproj | 4 ++-- .../ZB.MOM.WW.MxGateway.Server.csproj | 8 +++---- .../DashboardSnapshotServiceTests.cs | 21 +++++++++++++++++++ 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/src/ZB.MOM.WW.MxGateway.IntegrationTests/ZB.MOM.WW.MxGateway.IntegrationTests.csproj b/src/ZB.MOM.WW.MxGateway.IntegrationTests/ZB.MOM.WW.MxGateway.IntegrationTests.csproj index e699868..06c4207 100644 --- a/src/ZB.MOM.WW.MxGateway.IntegrationTests/ZB.MOM.WW.MxGateway.IntegrationTests.csproj +++ b/src/ZB.MOM.WW.MxGateway.IntegrationTests/ZB.MOM.WW.MxGateway.IntegrationTests.csproj @@ -22,8 +22,8 @@ (IntegrationTests-028). --> - - + + diff --git a/src/ZB.MOM.WW.MxGateway.Server/ZB.MOM.WW.MxGateway.Server.csproj b/src/ZB.MOM.WW.MxGateway.Server/ZB.MOM.WW.MxGateway.Server.csproj index ff87cb3..82bd3ea 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/ZB.MOM.WW.MxGateway.Server.csproj +++ b/src/ZB.MOM.WW.MxGateway.Server/ZB.MOM.WW.MxGateway.Server.csproj @@ -6,10 +6,10 @@ - - - - + + + + diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotServiceTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotServiceTests.cs index 50698aa..5f0e48c 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotServiceTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotServiceTests.cs @@ -623,6 +623,27 @@ public sealed class DashboardSnapshotServiceTests { return Task.FromResult(false); } + + /// Does nothing; the fake never changes scopes (added to IApiKeyAdminStore in Auth 0.1.3). + /// Key identifier. + /// Replacement scope set. + /// Cancellation token. + /// , always. + public Task SetScopesAsync(string keyId, IReadOnlySet scopes, CancellationToken ct) + { + return Task.FromResult(false); + } + + /// Does nothing; the fake never toggles key state (added to IApiKeyAdminStore in Auth 0.1.3). + /// Key identifier. + /// Desired enabled state. + /// Timestamp for the state change. + /// Cancellation token. + /// , always. + public Task SetEnabledAsync(string keyId, bool enabled, DateTimeOffset whenUtc, CancellationToken ct) + { + return Task.FromResult(false); + } } private class CountingApiKeyAdminStore(params ApiKeyListItem[] records) : FakeApiKeyAdminStore From c185f621a4bd9fc2f7b75e6ba8236f4bfc7fd330 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 06:40:57 -0400 Subject: [PATCH 05/19] fix(archreview): security config guards (SEC-01/04/06) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - SEC-01: derive AuthenticationOptions.SqlitePath / TlsOptions.SelfSignedCertPath defaults from SpecialFolder.CommonApplicationData (was Windows-literal strings that became CWD-relative files off-Windows); validator rejects non-rooted overrides; delete the stray C:\ProgramData\...gateway-auth.db files that materialized under src/; add a tree-hygiene test failing on any *.db under src/. - SEC-04: GatewayOptionsValidator fails startup when IsProduction() && DisableLogin. - SEC-06: validator rejects LDAP Transport=None in Production; document the MxGateway__Ldap__ServiceAccountPassword env override + dev-credential rotation. Galaxy SnapshotCachePath rooting could not be validator-enforced (bound by the external GalaxyRepository package, not GatewayOptions) — documented instead. archreview: SEC-01/04/06 (P1). Verified on the Auth-0.1.4 baseline: Server build clean, GatewayOptionsValidator + GatewayTreeHygiene tests 53/53. --- docs/Authentication.md | 2 + docs/GatewayConfiguration.md | 38 ++++- .../Configuration/AuthenticationOptions.cs | 14 +- .../Configuration/GatewayOptionsValidator.cs | 82 ++++++++++- .../Configuration/TlsOptions.cs | 16 ++- .../GatewayOptionsValidatorTests.cs | 135 ++++++++++++++++++ .../GatewayTreeHygieneTests.cs | 54 +++++++ 7 files changed, 327 insertions(+), 14 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/ProjectStructure/GatewayTreeHygieneTests.cs diff --git a/docs/Authentication.md b/docs/Authentication.md index 3fc0074..d864bb3 100644 --- a/docs/Authentication.md +++ b/docs/Authentication.md @@ -105,6 +105,8 @@ authorization code consumes. The gateway keeps API key state in a dedicated SQLite database. SQLite is sufficient because credential volume is small, the gateway runs as a single process, and the file is straightforward to back up and rotate independently of the main application data. +The database path is `GatewayOptions.Authentication.SqlitePath`. Its code default is derived from `Environment.GetFolderPath(SpecialFolder.CommonApplicationData)` (`C:\ProgramData\MxGateway\gateway-auth.db` on Windows, `/usr/share/MxGateway/gateway-auth.db` or the container equivalent elsewhere) so the credential store is never written relative to the launch working directory on a non-Windows host. The production hosts pin the explicit Windows path in `appsettings.json`. `GatewayOptionsValidator` rejects a non-rooted (relative) `SqlitePath` so a bad override fails fast at startup rather than scattering the store by launch CWD (SEC-01). + ### Connection factory `AuthSqliteConnectionFactory` reads `GatewayOptions.Authentication.SqlitePath`, ensures the parent directory exists, and builds a connection string in `ReadWriteCreate` mode so first-run installations can create the file without manual provisioning. Connection pooling is enabled and the connection string carries a non-zero `DefaultTimeout`: diff --git a/docs/GatewayConfiguration.md b/docs/GatewayConfiguration.md index 005eb5a..7304b7e 100644 --- a/docs/GatewayConfiguration.md +++ b/docs/GatewayConfiguration.md @@ -93,12 +93,15 @@ Environment variables use the normal .NET double-underscore form. For example, | Option | Default | Description | |--------|---------|-------------| | `MxGateway:Authentication:Mode` | `ApiKey` | Selects public gRPC authentication. Supported values are `ApiKey` and `Disabled`. `Disabled` bypasses API-key verification and is for local development only. | -| `MxGateway:Authentication:SqlitePath` | `C:\ProgramData\MxGateway\gateway-auth.db` | SQLite database path for API-key records and audit rows when API-key authentication is enabled. | +| `MxGateway:Authentication:SqlitePath` | derived from `CommonApplicationData` (`C:\ProgramData\MxGateway\gateway-auth.db` on Windows, `/usr/share/MxGateway/gateway-auth.db` or the container equivalent elsewhere) | SQLite database path for API-key records and audit rows when API-key authentication is enabled. The code default is built from `Environment.GetFolderPath(SpecialFolder.CommonApplicationData)` so the credential store never lands in the launch working directory on a non-Windows host; the production hosts pin the explicit Windows path in `appsettings.json`, which overrides the code default. | | `MxGateway:Authentication:PepperSecretName` | `MxGateway:ApiKeyPepper` | Configuration key used to read the HMAC pepper for API-key secret hashing. The dashboard effective configuration redacts this value. | | `MxGateway:Authentication:RunMigrationsOnStartup` | `true` | Runs SQLite auth schema migrations at gateway startup when API-key authentication is enabled. | When `Mode` is `ApiKey`, `SqlitePath` and `PepperSecretName` must be present. -`SqlitePath` must be a valid filesystem path. +`SqlitePath` must be a valid filesystem path and must be **rooted** (absolute): +the validator rejects a non-rooted path so a relative override cannot silently +resolve against the working directory and scatter the credential store by launch +CWD (SEC-01). ## Worker Options @@ -177,7 +180,7 @@ events (a "gap") and must re-snapshot; whatever is still retained is replayed. | `MxGateway:Dashboard:RecentSessionLimit` | `200` | Maximum number of session summaries projected into each dashboard snapshot. | | `MxGateway:Dashboard:ShowTagValues` | `false` | Reserved display control for tag values. The dashboard does not show full tag values by default. | | `MxGateway:Dashboard:GroupToRole` | _(empty)_ | LDAP group → dashboard role mapping. Keys are LDAP group names (short CN or full DN — leading-RDN match). Values must be `Admin` (read/write, API-key CRUD) or `Viewer` (read-only). A user whose LDAP groups don't intersect this map cannot sign in; with no mapping at all, only the loopback bypass admits anyone. | -| `MxGateway:Dashboard:DisableLogin` | `false` | Dev/test only. When `true`, replaces the cookie authentication handler with `DashboardAutoLoginAuthenticationHandler`, which auto-authenticates every dashboard request — including requests from remote browsers, not just loopback — as `AutoLoginUser` holding both `Administrator` and `Viewer` roles. No login form, LDAP bind, or cookie is involved. A loud one-time startup warning is logged. Differs from `AllowAnonymousLocalhost`: `DisableLogin` mints a real authenticated principal (so role-gated write affordances appear), whereas `AllowAnonymousLocalhost` satisfies the authorization requirement on loopback only without minting a principal (write affordances stay hidden). Never enable in production. | +| `MxGateway:Dashboard:DisableLogin` | `false` | Dev/test only. When `true`, replaces the cookie authentication handler with `DashboardAutoLoginAuthenticationHandler`, which auto-authenticates every dashboard request — including requests from remote browsers, not just loopback — as `AutoLoginUser` holding both `Administrator` and `Viewer` roles. No login form, LDAP bind, or cookie is involved. A loud one-time startup warning is logged. Differs from `AllowAnonymousLocalhost`: `DisableLogin` mints a real authenticated principal (so role-gated write affordances appear), whereas `AllowAnonymousLocalhost` satisfies the authorization requirement on loopback only without minting a principal (write affordances stay hidden). Never enable in production. **Production hard-stop (SEC-04):** when the host runs in the `Production` environment and `DisableLogin` is `true`, startup validation fails and the process aborts — the flag is only accepted outside Production, where the one-time startup warning still fires. | | `MxGateway:Dashboard:AutoLoginUser` | `(null)` | Username stamped on the synthetic principal when `DisableLogin` is `true`. Default `(null)` — a null or blank value falls back to `multi-role`. Has no effect when `DisableLogin` is `false`. | `SnapshotIntervalMilliseconds` must be greater than zero. `RecentFaultLimit` @@ -219,6 +222,31 @@ When the dashboard is enabled, three hubs are mapped under `/hubs/*`: token for the calling user; the Blazor pages use it via `DashboardHubConnectionFactory` to authenticate the SignalR connection. +## LDAP Options (Dashboard Login) + +The `MxGateway:Ldap` section configures the LDAP bind used by dashboard `/login` +(the gRPC API-key model is separate). The shipped defaults are the shared +dev/test GLAuth posture (`glauth.md`), not a production posture. + +| Option | Default | Description | +|--------|---------|-------------| +| `MxGateway:Ldap:Enabled` | `true` | Enables LDAP-backed dashboard login. | +| `MxGateway:Ldap:Server` | `localhost` | LDAP server host. Dev points at the shared GLAuth instance (`10.100.0.35`). | +| `MxGateway:Ldap:Port` | `3893` | LDAP server port. Must be between `1` and `65535`. | +| `MxGateway:Ldap:Transport` | `None` | Connection transport: `None` (plaintext), `StartTls` (upgrade), or `Ldaps` (implicit TLS). The dev default is `None` because the shared GLAuth instance has LDAPS disabled and binds send cleartext. **Production hard-stop (SEC-06):** when the host runs in the `Production` environment, `Transport` must not be `None` — startup validation fails otherwise. Deployed hosts must set `Ldaps` or `StartTls`. | +| `MxGateway:Ldap:AllowInsecure` | `true` | Permits a plaintext bind. Must be `true` when `Transport` is `None`; set `false` (with `Ldaps`/`StartTls`) in production. | +| `MxGateway:Ldap:SearchBase` | `dc=zb,dc=local` | Search base DN. | +| `MxGateway:Ldap:ServiceAccountDn` | `cn=serviceaccount,dc=zb,dc=local` | Bind DN for the search account. | +| `MxGateway:Ldap:ServiceAccountPassword` | `serviceaccount123` | Search-account password. **Dev-only committed value.** On the deployed hosts, do not ship this in `appsettings.json`; supply it out-of-band via the env-var override `MxGateway__Ldap__ServiceAccountPassword` (double-underscore form, as set on the NSSM-wrapped services). The committed `serviceaccount123` is a shared dev credential for the local GLAuth instance and should be rotated; production must use a secret it does not share with dev. | +| `MxGateway:Ldap:UserNameAttribute` | `cn` | LDAP attribute holding the login user name. | +| `MxGateway:Ldap:DisplayNameAttribute` | `cn` | LDAP attribute holding the display name. | +| `MxGateway:Ldap:GroupAttribute` | `memberOf` | LDAP attribute enumerating group membership (mapped to dashboard roles via `MxGateway:Dashboard:GroupToRole`). | + +When LDAP is enabled, `Server`, `SearchBase`, `ServiceAccountDn`, +`ServiceAccountPassword`, and the attribute names must be non-blank, and `Port` +must be in range. See `glauth.md` for the shared dev instance and the +dev→production hardening posture. + ## Protocol Options | Option | Default | Description | @@ -238,7 +266,7 @@ at startup. | `MxGateway:Galaxy:CommandTimeoutSeconds` | `60` | Per-command SQL timeout for all Galaxy browse RPCs. | | `MxGateway:Galaxy:DashboardRefreshIntervalSeconds` | `30` | Interval between background refreshes of the dashboard Galaxy summary cache. SQL is hit at most once per interval regardless of dashboard render rate. | | `MxGateway:Galaxy:PersistSnapshot` | `true` | Persists the latest successful Galaxy browse dataset to disk. When `true`, the cache reloads that snapshot at startup so clients can still browse last-known data while the Galaxy database is unreachable. The restored data is served with `Stale` status until a live query confirms it. | -| `MxGateway:Galaxy:SnapshotCachePath` | `C:\ProgramData\MxGateway\galaxy-snapshot.json` | File path for the persisted Galaxy browse snapshot. Ignored when `PersistSnapshot` is `false`. The snapshot is written atomically (temp file plus rename). | +| `MxGateway:Galaxy:SnapshotCachePath` | `C:\ProgramData\MxGateway\galaxy-snapshot.json` | File path for the persisted Galaxy browse snapshot. Ignored when `PersistSnapshot` is `false`. The snapshot is written atomically (temp file plus rename). Set an **absolute** path — this option is bound by the shared `ZB.MOM.WW.GalaxyRepository` package (not by `GatewayOptions`), so the gateway validator does not enforce rooting on it; a relative value would resolve against the launch working directory (SEC-01). | See [Galaxy Repository Browse](./GalaxyRepository.md) for the RPC surface and behavior. @@ -442,7 +470,7 @@ them. | Option | Default | Purpose | |---|---|---| -| `Tls:SelfSignedCertPath` | `C:\ProgramData\MxGateway\certs\gateway-selfsigned.pfx` | Where the generated certificate is persisted | +| `Tls:SelfSignedCertPath` | derived from `CommonApplicationData` (`C:\ProgramData\MxGateway\certs\gateway-selfsigned.pfx` on Windows, `/usr/share/MxGateway/certs/gateway-selfsigned.pfx` or the container equivalent elsewhere) | Where the generated certificate is persisted. The code default is built from `Environment.GetFolderPath(SpecialFolder.CommonApplicationData)` so a generated private key never lands in the launch working directory on a non-Windows host (SEC-01). Must be **rooted** (absolute); the validator rejects a non-rooted override. | | `Tls:ValidityYears` | `10` | Lifetime of the generated certificate (validated 1–100) | | `Tls:AdditionalDnsNames` | `[]` | Extra DNS SANs (e.g. a load-balancer name) | | `Tls:RegenerateIfExpired` | `true` | Replace an expired persisted certificate instead of failing | diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/AuthenticationOptions.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/AuthenticationOptions.cs index 534cdef..767daf1 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/AuthenticationOptions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/AuthenticationOptions.cs @@ -5,8 +5,18 @@ public sealed class AuthenticationOptions /// Gets the authentication mode. public AuthenticationMode Mode { get; init; } = AuthenticationMode.ApiKey; - /// Gets the SQLite database path for authentication credentials. - public string SqlitePath { get; init; } = @"C:\ProgramData\MxGateway\gateway-auth.db"; + /// + /// Gets the SQLite database path for authentication credentials. The default is + /// derived from + /// (C:\ProgramData on Windows, /usr/share or the container equivalent + /// elsewhere) so the credential store never lands in the launch working directory on a + /// non-Windows host. The production hosts override this with an explicit path in + /// appsettings.json; the validator rejects a non-rooted override. + /// + public string SqlitePath { get; init; } = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), + "MxGateway", + "gateway-auth.db"); /// Gets the secret manager name for API key pepper. public string PepperSecretName { get; init; } = "MxGateway:ApiKeyPepper"; diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs index e5b8813..686b168 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs @@ -9,15 +9,42 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase + /// Initializes a new instance of the class for the + /// dependency-injection path, deriving the production posture from the host environment. + /// + /// The host environment. + public GatewayOptionsValidator(IHostEnvironment environment) + { + ArgumentNullException.ThrowIfNull(environment); + _isProduction = environment.IsProduction(); + } + + /// + /// Initializes a new instance of the class for unit + /// tests and non-DI callers. Defaults to a non-production posture so the production-only + /// hard-stops do not fire; pass to exercise them. + /// + /// Whether to treat the host as running in Production. + internal GatewayOptionsValidator(bool isProduction = false) + { + _isProduction = isProduction; + } + /// protected override void Validate(ValidationBuilder builder, GatewayOptions options) { ValidateAuthentication(options.Authentication, builder); - ValidateLdap(options.Ldap, builder); + ValidateLdap(options.Ldap, builder, _isProduction); ValidateWorker(options.Worker, builder); ValidateSessions(options.Sessions, builder); ValidateEvents(options.Events, builder); - ValidateDashboard(options.Dashboard, builder); + ValidateDashboard(options.Dashboard, builder, _isProduction); ValidateProtocol(options.Protocol, builder); ValidateAlarms(options.Alarms, builder); ValidateTls(options.Tls, builder); @@ -41,6 +68,10 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase= 0, message); } + private static void AddIfNotRooted(string? value, string message, ValidationBuilder builder) + { + // Security-sensitive paths (the auth DB, the self-signed private key) must be absolute: + // a non-rooted value silently resolves against the launch working directory, so the store + // moves with the CWD and can leak into the source tree. Reject rather than auto-root — + // silent relocation of a credential store is worse than a boot error. Blank is handled by + // AddIfBlank; an empty value is not treated as non-rooted here. + if (string.IsNullOrWhiteSpace(value)) + { + return; + } + + if (!Path.IsPathRooted(value)) + { + builder.Add(message); + } + } + private static void AddIfInvalidPath(string? value, string message, ValidationBuilder builder) { if (string.IsNullOrWhiteSpace(value)) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/TlsOptions.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/TlsOptions.cs index 9be84a3..4099749 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/TlsOptions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/TlsOptions.cs @@ -7,9 +7,19 @@ namespace ZB.MOM.WW.MxGateway.Server.Configuration; /// public sealed class TlsOptions { - /// Path to the persisted self-signed PFX. Reused across restarts. - public string SelfSignedCertPath { get; init; } = - @"C:\ProgramData\MxGateway\certs\gateway-selfsigned.pfx"; + /// + /// Path to the persisted self-signed PFX. Reused across restarts. The default is derived + /// from (C:\ProgramData + /// on Windows, /usr/share or the container equivalent elsewhere) so a generated + /// private key never lands in the launch working directory on a non-Windows host. The + /// production hosts override this with an explicit path; the validator rejects a non-rooted + /// override. + /// + public string SelfSignedCertPath { get; init; } = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), + "MxGateway", + "certs", + "gateway-selfsigned.pfx"); /// Lifetime in years of a freshly generated certificate. public int ValidityYears { get; init; } = 10; diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs index e8646eb..159203c 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.Options; using ZB.MOM.WW.MxGateway.Server.Configuration; +using LdapTransport = ZB.MOM.WW.Auth.Abstractions.Ldap.LdapTransport; namespace ZB.MOM.WW.MxGateway.Tests.Configuration; @@ -548,4 +549,138 @@ public sealed class GatewayOptionsValidatorTests ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, options); Assert.True(result.Succeeded); } + + // ------------------------------------------------------------------------- + // SEC-01: security-sensitive paths must be rooted (absolute) + // ------------------------------------------------------------------------- + + private static GatewayOptions CloneWithAuthentication(GatewayOptions source, AuthenticationOptions authentication) + => new() + { + Authentication = authentication, + Ldap = source.Ldap, + Worker = source.Worker, + Sessions = source.Sessions, + Events = source.Events, + Dashboard = source.Dashboard, + Protocol = source.Protocol, + Alarms = source.Alarms, + Tls = source.Tls, + }; + + /// Verifies the default (CommonApplicationData-derived) auth DB path is rooted and passes validation. + [Fact] + public void Validate_Succeeds_WithDefaultRootedSqlitePath() + { + // The default AuthenticationOptions.SqlitePath is derived from CommonApplicationData, + // which is rooted on every host OS. + ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, ValidOptions()); + Assert.True(result.Succeeded); + } + + /// Verifies a non-rooted fails validation. + [Fact] + public void Validate_Fails_WhenSqlitePathNotRooted() + { + GatewayOptions options = CloneWithAuthentication( + ValidOptions(), + new AuthenticationOptions { SqlitePath = "gateway-auth.db" }); + ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains( + result.Failures!, + f => f.Contains("MxGateway:Authentication:SqlitePath") && f.Contains("rooted")); + } + + /// Verifies a non-rooted fails validation. + [Fact] + public void Validate_Fails_WhenSelfSignedCertPathNotRooted() + { + GatewayOptions options = CloneWithTls( + ValidOptions(), + new TlsOptions { SelfSignedCertPath = "gateway-selfsigned.pfx" }); + ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, options); + Assert.True(result.Failed); + Assert.Contains( + result.Failures!, + f => f.Contains("MxGateway:Tls:SelfSignedCertPath") && f.Contains("rooted")); + } + + // ------------------------------------------------------------------------- + // SEC-04: DisableLogin production guard + // ------------------------------------------------------------------------- + + private static GatewayOptions CloneWithDashboard(GatewayOptions source, DashboardOptions dashboard) + => new() + { + Authentication = source.Authentication, + Ldap = source.Ldap, + Worker = source.Worker, + Sessions = source.Sessions, + Events = source.Events, + Dashboard = dashboard, + Protocol = source.Protocol, + Alarms = source.Alarms, + Tls = source.Tls, + }; + + /// Verifies set to aborts startup in Production. + [Fact] + public void Validate_Fails_WhenDisableLoginTrueInProduction() + { + GatewayOptions options = CloneWithDashboard( + ValidOptions(), + new DashboardOptions { DisableLogin = true }); + ValidateOptionsResult result = new GatewayOptionsValidator(isProduction: true).Validate(null, options); + Assert.True(result.Failed); + Assert.Contains( + result.Failures!, + f => f.Contains("MxGateway:Dashboard:DisableLogin") && f.Contains("Production")); + } + + /// Verifies set to is accepted outside Production. + [Fact] + public void Validate_Succeeds_WhenDisableLoginTrueInDevelopment() + { + GatewayOptions options = CloneWithDashboard( + ValidOptions(), + new DashboardOptions { DisableLogin = true }); + ValidateOptionsResult result = new GatewayOptionsValidator(isProduction: false).Validate(null, options); + Assert.True(result.Succeeded); + } + + // ------------------------------------------------------------------------- + // SEC-06: LDAP plaintext transport production guard + // ------------------------------------------------------------------------- + + /// Verifies plaintext LDAP transport (None) aborts startup in Production. + [Fact] + public void Validate_Fails_WhenLdapTransportNoneInProduction() + { + // The class default LDAP options ship Transport=None + AllowInsecure=true (dev posture). + ValidateOptionsResult result = new GatewayOptionsValidator(isProduction: true).Validate(null, ValidOptions()); + Assert.True(result.Failed); + Assert.Contains( + result.Failures!, + f => f.Contains("MxGateway:Ldap:Transport") && f.Contains("Production")); + } + + /// Verifies plaintext LDAP transport (None) is accepted outside Production. + [Fact] + public void Validate_Succeeds_WhenLdapTransportNoneInDevelopment() + { + ValidateOptionsResult result = new GatewayOptionsValidator(isProduction: false).Validate(null, ValidOptions()); + Assert.True(result.Succeeded); + } + + /// Verifies secure LDAP transport (Ldaps) passes validation in Production. + [Fact] + public void Validate_Succeeds_WhenLdapTransportLdapsInProduction() + { + GatewayOptions options = CloneWithLdap( + ValidOptions(), + new LdapOptions { Enabled = true, Transport = LdapTransport.Ldaps, AllowInsecure = false }); + ValidateOptionsResult result = new GatewayOptionsValidator(isProduction: true).Validate(null, options); + Assert.True(result.Succeeded); + } } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/ProjectStructure/GatewayTreeHygieneTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/ProjectStructure/GatewayTreeHygieneTests.cs new file mode 100644 index 0000000..4505874 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/ProjectStructure/GatewayTreeHygieneTests.cs @@ -0,0 +1,54 @@ +namespace ZB.MOM.WW.MxGateway.Tests.ProjectStructure; + +/// +/// Repo-hygiene guards over the source tree. See SEC-01: a Windows-absolute default path that +/// resolves relative to the launch working directory silently materializes a SQLite credential +/// store inside the tree. This test fails if any such *.db file appears under src/. +/// +public sealed class GatewayTreeHygieneTests +{ + /// Verifies no SQLite database files are committed/materialized under the source tree (build output excluded). + [Fact] + public void SourceTree_ContainsNoSqliteDatabaseFiles() + { + DirectoryInfo sourceRoot = FindSourceRoot(); + + string[] databaseFiles = Directory + .EnumerateFiles(sourceRoot.FullName, "*.db", SearchOption.AllDirectories) + .Where(path => !IsBuildOutput(path)) + .ToArray(); + + Assert.True( + databaseFiles.Length == 0, + "Unexpected *.db file(s) found under the source tree (build output excluded): " + + string.Join(", ", databaseFiles) + + ". A SQLite DB in the tree usually means a Windows-absolute default path resolved " + + "relative to the working directory; see SEC-01."); + } + + // Build output (bin/obj) is regenerated by test/compile runs and is gitignored, so exclude it. + private static bool IsBuildOutput(string path) + { + string normalized = path.Replace('\\', '/'); + return normalized.Contains("/bin/", StringComparison.Ordinal) + || normalized.Contains("/obj/", StringComparison.Ordinal); + } + + // The directory holding ZB.MOM.WW.MxGateway.slnx is the src/ root of the working tree. + private static DirectoryInfo FindSourceRoot() + { + DirectoryInfo? current = new(AppContext.BaseDirectory); + + while (current is not null) + { + if (File.Exists(Path.Combine(current.FullName, "ZB.MOM.WW.MxGateway.slnx"))) + { + return current; + } + + current = current.Parent; + } + + throw new DirectoryNotFoundException("Could not locate src/ZB.MOM.WW.MxGateway.slnx from the test output directory."); + } +} From 970613eebd814e909d147f2c76c09866d62349b1 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 06:41:40 -0400 Subject: [PATCH 06/19] docs(archreview): P1 Wave 2a status (SEC-01/04/06 Done; SEC-10 Done via G-2 Auth 0.1.4) --- archreview/remediation/00-tracking.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/archreview/remediation/00-tracking.md b/archreview/remediation/00-tracking.md index d19db7c..3a928a9 100644 --- a/archreview/remediation/00-tracking.md +++ b/archreview/remediation/00-tracking.md @@ -139,16 +139,16 @@ Full design + implementation for each row lives in the linked domain doc under i | ID | Sev | Tier | Eff | Dep | Status | Title | |---|---|:-:|:-:|---|---|---| -| SEC-01 | Medium | P1 | M | — | Not started | Windows-absolute default paths become relative files off-Windows | +| SEC-01 | Medium | P1 | M | — | Done | Windows-absolute default paths become relative files off-Windows | | SEC-02 | Medium | P1 | S | — | Not started | `AllowAnonymousLocalhost` satisfies AdminOnly, not just Viewer | | SEC-03 | Medium | P2 | S | — | Not started | Dashboard cookie lost its `__Host-` prefix; four docs still promise it | -| SEC-04 | Medium | P1 | S | SEC-03 | Not started | `DisableLogin` has no production guard | +| SEC-04 | Medium | P1 | S | SEC-03 | Done | `DisableLogin` has no production guard | | SEC-05 | Medium | P1 | M | — | Not started | Hub bearer tokens irrevocable for 30 min, carried in query string | -| SEC-06 | Medium | P1 | M | — | Not started | LDAP plaintext-by-default with a committed service password | +| SEC-06 | Medium | P1 | M | — | Done | LDAP plaintext-by-default with a committed service password | | SEC-07 | Medium | P1 | S | — | Not started | `QueryActiveAlarmsRequest` missing from scope resolver; tests mislabeled | | SEC-08 | Medium | P1 | L | — | Not started | Per-RPC SQLite read + `last_used_utc` write; no verification cache | | SEC-09 | Medium | P2 | S | — | Not started | Dashboard design-doc `GroupToRole` sample now fails startup validation | -| SEC-10 | Medium | P1 | L | — | Not started | No API-key expiry | +| SEC-10 | Medium | P1 | L | — | Done | No API-key expiry | | SEC-11 | Medium | P1 | M | SEC-08 | Not started | No rate limiting or lockout on either auth surface | | SEC-12 | Medium | P1 | M | — | Not started | Dashboard Close/Kill bypass the canonical audit store | | SEC-13 | Low | — | S | SEC-30 | Not started | Redactor's credential-command list omits secured-bulk variants | @@ -255,5 +255,6 @@ Findings the review flagged as one coordinated design pass — sequence them tog |---|---| | 2026-07-09 | Initial tracking doc generated from the six domain remediation designs. All 153 findings `Not started`. | | 2026-07-09 | P0 tier executed via parallel agents. GWC-01/02/03, TST-02, TST-12, CLI-01, CLI-03 → `Done` (NonWindows build clean; gateway 135/135 targeted tests pass, Go `go test ./...` + `-race` pass, Rust `test`/`clippy -D warnings` pass). | +| 2026-07-09 | P1 Wave 2a (security). SEC-01/04/06 → `Done` (config path-rooting + production validator guards; Server build clean, validator+hygiene tests 53/53). SEC-10 → `Done`: the shared `ZB.MOM.WW.Auth.ApiKeys` gained optional `ExpiresUtc` (expired keys rejected, auth DB auto-migrates to schema v3) via a concurrent HistorianGateway-remediation session's "G-2"; this repo consumes it by bumping the four `Auth.*` refs 0.1.2→0.1.4 (commit 197731a). Remaining SEC-10 polish (`apikey create --expires` + dashboard staleness badge) tracked as a small follow-up. | | 2026-07-09 | P1 Wave 1 (CI + codegen freshness + Rust buildability) via parallel agents. IPC-01/09/19/20, CLI-02 → `Done`; TST-03 → `In review` (CI pipeline authored + YAML/layout-validated, but not yet executed on a Gitea runner — proven on first push). Verified on macOS: NonWindows build clean, `ClientProtoInputTests` 5/5, `publish-client-proto-inputs.ps1 -Check` exit 0, `cargo package` (no `--no-verify`) compiles standalone. Added a vendored-Rust-proto drift guard (Check 3) to `check-codegen.ps1` closing the CLI-02 static-copy risk. IPC-09 script guards not executed end-to-end (need pinned python/JRE toolchains); logic is PATH-resolution + version assertion. | | 2026-07-09 | WRK-01 → `Done`. Verified on Windows host (windev) via an isolated `origin/main` worktree: worker builds x86 clean, `StaRuntimeTests`+`WorkerPipeSessionTests` 33/33 pass. Fixed an `xUnit1030` build error (the new worker test used `.ConfigureAwait(false)` in `[Fact]` bodies) that the macOS tree could not surface. Also ran GWC-01's Windows-only `WorkerClientTests` on windev: 18/18 pass (incl. `ReadEventsAsync_SecondEnumerator_Throws`). All 8 P0 findings now `Done`. Not yet committed. | From 17f16ea18161313cfa131842752a10baf76e1f08 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 07:31:35 -0400 Subject: [PATCH 07/19] fix(archreview): security authz+hub cluster (SEC-07/05/08/11) + validation hardening SEC-07: add QueryActiveAlarmsRequest -> events:read scope arm; fix two tests that constructed StreamAlarmsRequest instead of QueryActiveAlarmsRequest. SEC-05: shorten hub-token lifetime 30m -> 5m; document that the ?access_token= query carriage must never be request-logged. SEC-08: gateway-side CachingApiKeyVerifier (short TTL, keyed on a hash of the presented secret) skips the per-call store read+last_used write; CoalescingMarkApiKeyStore bounds last_used writes to <=1/key/min; identity constraints are cached. Invalidation is wired at the gateway admin sites (revoke/rotate/delete); short TTL backstops out-of-process CLI. SEC-11: fixed-window rate limit on POST /auth/login + a per-peer (key-id) failure limiter checked before VerifyAsync; new MxGateway:Security options bound + validated. Also fixes regressions from the SEC-01/04/06 commit (c185f62) that a narrow test filter missed (all now covered by a full-suite checkpoint): - Rooting check is cross-platform: accepts Windows C:\/UNC forms on Unix so the shipped appsettings path validates on the macOS dev box, still rejecting bare filenames. - AddGatewayConfiguration TryAdds a non-production IHostEnvironment fallback so the validator resolves in minimal test/tooling containers; the real host + apikey CLI register the actual environment first (TryAdd no-op there). - Test assembly defaults ASPNETCORE_ENVIRONMENT=Development (ModuleInitializer) so full-host tests exercise wiring instead of tripping the SEC-04/06 production guards. - GatewayOptionsTests asserts the SEC-01 CommonApplicationData-derived default (platform-correct). archreview: SEC-07/05/08/11 (P1). Verified: NonWindows build clean; full gateway suite 747 passed / 42 failed, where all 42 are the pre-existing macOS named-pipe-harness failures (Unix-socket path limit) and 0 are validation/regression failures. --- docs/Authentication.md | 32 +++ docs/Authorization.md | 11 +- docs/GatewayConfiguration.md | 23 +- docs/GatewayDashboardDesign.md | 20 +- docs/Metrics.md | 12 ++ ...onfigurationServiceCollectionExtensions.cs | 26 +++ .../Configuration/GatewayOptions.cs | 6 + .../Configuration/GatewayOptionsValidator.cs | 68 +++++- .../Configuration/SecurityOptions.cs | 65 ++++++ .../DashboardApiKeyManagementService.cs | 15 +- ...DashboardEndpointRouteBuilderExtensions.cs | 38 ++++ .../HubTokenAuthenticationHandler.cs | 10 + .../Dashboard/HubTokenService.cs | 24 ++- .../GatewayApplication.cs | 24 +++ .../AuthStoreServiceCollectionExtensions.cs | 85 ++++++++ .../Authentication/CachingApiKeyVerifier.cs | 159 ++++++++++++++ .../CoalescingMarkApiKeyStore.cs | 101 +++++++++ .../GatewayApiKeyIdentityMapper.cs | 34 ++- .../Authorization/ApiKeyFailureLimiter.cs | 152 +++++++++++++ .../GatewayGrpcAuthorizationInterceptor.cs | 47 +++- .../Authorization/GatewayGrpcScopeResolver.cs | 1 + ...uthorizationServiceCollectionExtensions.cs | 7 + .../Configuration/GatewayOptionsTests.cs | 10 +- .../GatewayOptionsValidatorTests.cs | 88 ++++++++ .../Dashboard/DashboardLoginRateLimitTests.cs | 66 ++++++ .../Gateway/Dashboard/HubTokenServiceTests.cs | 59 +++++ .../CachingApiKeyVerifierTests.cs | 201 ++++++++++++++++++ ...atewayGrpcAuthorizationInterceptorTests.cs | 126 ++++++++++- .../TestHostEnvironmentInitializer.cs | 29 +++ 29 files changed, 1521 insertions(+), 18 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Server/Configuration/SecurityOptions.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/CachingApiKeyVerifier.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/CoalescingMarkApiKeyStore.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardLoginRateLimitTests.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TestHostEnvironmentInitializer.cs diff --git a/docs/Authentication.md b/docs/Authentication.md index d864bb3..29a17a3 100644 --- a/docs/Authentication.md +++ b/docs/Authentication.md @@ -101,6 +101,38 @@ return ApiKeyVerificationResult.Success(new ApiKeyIdentity( `DisplayName`, `Scopes`, and `Constraints`) and is the type downstream authorization code consumes. +### Hot-path caching and last-used coalescing + +Left unmediated, every authenticated gRPC call costs a SQLite read plus a +`last_used_utc` **write** (the library verifier couples `MarkKeyUsed` into +`VerifyAsync`), which makes the auth store the throughput ceiling on the +bulk-read workload. The gateway layers two decorators over the shared library's +registrations (in `AuthStoreServiceCollectionExtensions`) — it does not edit the +library: + +- **`CachingApiKeyVerifier`** wraps `IApiKeyVerifier` with an `IMemoryCache` + entry per successful verification, keyed on a SHA-256 hash of the presented + token (never the plaintext secret). A cache hit within + `MxGateway:Security:ApiKeyVerificationCacheSeconds` (default 15 s) returns the + cached result without touching the store, so both the read and the coupled + write are skipped. Only successes are cached; failures always reach the inner + verifier. On a gateway-initiated revoke/rotate/delete the dashboard admin + service calls `IApiKeyCacheInvalidator.Invalidate(keyId)`, evicting the cached + entry immediately. The short TTL is the backstop for out-of-band mutations + (a direct DB edit, or a revoke run by the separate `apikey` CLI process, whose + in-memory cache is not the running gateway's cache). +- **`CoalescingMarkApiKeyStore`** wraps `IApiKeyStore` and forwards at most one + `MarkUsed` write per key per `MxGateway:Security:ApiKeyLastUsedCoalesceSeconds` + (default 60 s), so even under a cache miss the `last_used_utc` write is bounded + to roughly one per key per minute rather than one per RPC. `last_used_utc` is a + coarse staleness hint, not an audit record (audit rows are written separately), + so bounded staleness of up to one window is acceptable. + +`GatewayApiKeyIdentityMapper` additionally memoizes the constraints-JSON +deserialization by blob, so the per-call parse on the mapped identity collapses to +a dictionary lookup. Both windows are configurable and may be set to `0` to +disable the respective mechanism; see [GatewayConfiguration](./GatewayConfiguration.md). + ## Storage The gateway keeps API key state in a dedicated SQLite database. SQLite is sufficient because credential volume is small, the gateway runs as a single process, and the file is straightforward to back up and rotate independently of the main application data. diff --git a/docs/Authorization.md b/docs/Authorization.md index c693317..87c62de 100644 --- a/docs/Authorization.md +++ b/docs/Authorization.md @@ -89,6 +89,12 @@ The flow is: The status codes are deliberately distinct: `Unauthenticated` signals "we do not know who you are," and `PermissionDenied` signals "we know who you are, but you cannot do this." Treating the two as the same code would make troubleshooting harder for client implementations. +### Rate limiting the auth surface (SEC-11) + +Before the verification store read, the helper checks a cheap in-process per-peer failure counter (`ApiKeyFailureLimiter`). A peer that has accumulated more than `MxGateway:Security:ApiKeyFailureLimit` failed attempts inside the sliding `ApiKeyFailureWindowSeconds` window is short-circuited with `StatusCode.ResourceExhausted` — so online guessing of API-key secrets cannot spend a SQLite read (and, in a naive design, a cache miss) per attempt. The peer is keyed on the presented key id where the token parses, falling back to the transport peer address; keying on key id throttles a single abusive credential without penalizing co-located clients behind a shared NAT. A successful verification resets the peer's counter. The counter is a bounded LRU (`ApiKeyFailureTrackedPeers`) so it cannot grow without limit. `ResourceExhausted` reveals only that throttling is in effect, not whether any particular secret was valid, preserving the opaque-failure property. + +The dashboard login surface is throttled independently: `POST /auth/login` carries a fixed-window ASP.NET Core rate-limiter policy keyed per remote IP (`MxGateway:Security:LoginRateLimit*`), rejecting a burst with HTTP 429 before the LDAP bind is relayed to the directory. See [GatewayConfiguration](./GatewayConfiguration.md#security-options). + ## Scope Resolution `GatewayGrpcScopeResolver` is a stateless singleton that switches on the runtime request type. Top-level RPC requests map directly: @@ -104,6 +110,7 @@ public string ResolveRequiredScope(object request) MxCommandRequest commandRequest => ResolveCommandScope(commandRequest.Command?.Kind ?? MxCommandKind.Unspecified), AcknowledgeAlarmRequest => GatewayScopes.InvokeWrite, StreamAlarmsRequest => GatewayScopes.EventsRead, + QueryActiveAlarmsRequest => GatewayScopes.EventsRead, TestConnectionRequest or GetLastDeployTimeRequest or DiscoverHierarchyRequest or @@ -113,7 +120,7 @@ public string ResolveRequiredScope(object request) } ``` -The `_ => GatewayScopes.Admin` fallback is intentional: any future request type that the resolver does not recognize fails closed, requiring the strongest scope until the resolver is updated. `AcknowledgeAlarm` is treated as a write — it mutates alarm state, mirroring `MxCommandKind.Write*` — and `StreamAlarms` shares the alarm/event surface with `StreamEvents` and `MxCommandKind.DrainEvents`, so it carries `events:read`. Both alarm RPCs are session-less: the scope check is the only authorization gate, since there is no per-session ownership to enforce. +The `_ => GatewayScopes.Admin` fallback is intentional: any future request type that the resolver does not recognize fails closed, requiring the strongest scope until the resolver is updated. `AcknowledgeAlarm` is treated as a write — it mutates alarm state, mirroring `MxCommandKind.Write*` — and `StreamAlarms` and `QueryActiveAlarms` share the alarm/event surface with `StreamEvents` and `MxCommandKind.DrainEvents`, so they carry `events:read` (the active-alarm snapshot is the same data reachable through the event surface). All three alarm RPCs are session-less: the scope check is the only authorization gate, since there is no per-session ownership to enforce. `MxCommandRequest` is special because it multiplexes many MxAccess operations through a single RPC. The resolver inspects the embedded `MxCommandKind` so each operation gets its own scope: @@ -205,7 +212,7 @@ blocking constraint; secured values and raw credentials are never logged. |----------|-------|--------------| | `SessionOpen` | `session:open` | `OpenSessionRequest` | | `SessionClose` | `session:close` | `CloseSessionRequest` | -| `EventsRead` | `events:read` | `StreamEventsRequest`, `StreamAlarmsRequest`, `MxCommandKind.DrainEvents` | +| `EventsRead` | `events:read` | `StreamEventsRequest`, `StreamAlarmsRequest`, `QueryActiveAlarmsRequest`, `MxCommandKind.DrainEvents` | | `InvokeRead` | `invoke:read` | `MxCommandRequest` for read-style command kinds (`Register`, `AddItem`, `Advise`, `ReadBulk`, and any kind not otherwise mapped) | | `InvokeWrite` | `invoke:write` | `AcknowledgeAlarmRequest`, `MxCommandKind.Write`, `MxCommandKind.Write2`, `MxCommandKind.WriteBulk`, `MxCommandKind.Write2Bulk` | | `InvokeSecure` | `invoke:secure` | `MxCommandKind.WriteSecured`, `MxCommandKind.WriteSecured2`, `MxCommandKind.WriteSecuredBulk`, `MxCommandKind.WriteSecured2Bulk`, `MxCommandKind.AuthenticateUser` | diff --git a/docs/GatewayConfiguration.md b/docs/GatewayConfiguration.md index 7304b7e..3b1eaec 100644 --- a/docs/GatewayConfiguration.md +++ b/docs/GatewayConfiguration.md @@ -218,9 +218,13 @@ When the dashboard is enabled, three hubs are mapped under `/hubs/*`: side-effect; the dashboard only sees events while a gRPC client is also subscribed to that session. -`GET /hubs/token` (cookie-only) mints a 30-minute data-protected bearer +`GET /hubs/token` (cookie-only) mints a 5-minute data-protected bearer token for the calling user; the Blazor pages use it via `DashboardHubConnectionFactory` to authenticate the SignalR connection. +The factory refreshes the token on every (re)connect, so the short lifetime +(SEC-05) is transparent to clients. The token is not server-side revocable; +its short lifetime bounds exposure of a captured token (see +[GatewayDashboardDesign](./GatewayDashboardDesign.md)). ## LDAP Options (Dashboard Login) @@ -258,6 +262,23 @@ The protocol option is exposed for diagnostics and explicit deployment configuration, not for compatibility negotiation. A mismatch fails validation at startup. +## Security Options + +Bound from `MxGateway:Security`. These knobs tune the authentication hot path +(SEC-08 verification cache / last-used coalescing) and the auth-surface rate +limiting (SEC-11). Defaults are safe; leave them unless profiling or a threat +model requires otherwise. + +| Option | Default | Description | +|--------|---------|-------------| +| `MxGateway:Security:ApiKeyVerificationCacheSeconds` | `15` | TTL, in seconds, of a cached successful API-key verification. A cache hit within this window skips the per-call SQLite read (and the coupled `last_used` write). Gateway-initiated revoke/rotate/delete invalidate the entry immediately; this TTL is the backstop for out-of-band mutations (a direct DB edit, or a revoke run by the separate `apikey` CLI process). `0` disables the cache. Must be zero or greater. | +| `MxGateway:Security:ApiKeyLastUsedCoalesceSeconds` | `60` | Coalescing window, in seconds, for the `last_used_utc` write. The library verifier writes `last_used` on every successful verification; this bounds the write to at most one per key per window, so a hammered key does not churn the WAL. `0` forwards every write. Must be zero or greater. | +| `MxGateway:Security:LoginRateLimitPermitLimit` | `10` | Maximum `POST /auth/login` attempts permitted per remote IP within `LoginRateLimitWindowSeconds` before requests are rejected with HTTP 429. Throttles LDAP credential stuffing before the bind is relayed to the directory. Must be greater than zero. | +| `MxGateway:Security:LoginRateLimitWindowSeconds` | `60` | Fixed-window length, in seconds, for the per-IP login rate limit. Must be greater than zero. | +| `MxGateway:Security:ApiKeyFailureLimit` | `10` | Failed API-key verifications, per peer, within `ApiKeyFailureWindowSeconds` that trip the in-process short-circuit. Once tripped, the gRPC auth path rejects further attempts with `ResourceExhausted` **before** the store read; a successful verification resets the peer's counter. The peer is keyed on the presented key id (falling back to the transport address) so a single abusive credential behind a shared NAT is throttled without locking out co-located clients. Must be greater than zero. | +| `MxGateway:Security:ApiKeyFailureWindowSeconds` | `60` | Sliding-window length, in seconds, over which API-key verification failures are counted per peer. Must be greater than zero. | +| `MxGateway:Security:ApiKeyFailureTrackedPeers` | `4096` | Maximum distinct peers tracked by the failure counter (a bounded LRU) so a spray of unique peer keys cannot grow memory without limit. Must be greater than zero. | + ## Galaxy Options | Option | Default | Description | diff --git a/docs/GatewayDashboardDesign.md b/docs/GatewayDashboardDesign.md index ed49e4b..9778a74 100644 --- a/docs/GatewayDashboardDesign.md +++ b/docs/GatewayDashboardDesign.md @@ -486,9 +486,13 @@ dashboard mints short-lived bearer tokens for the connection: and role claims to JSON, encrypts with the ASP.NET Core data-protection time-limited protector under purpose `ZB.MOM.WW.MxGateway.Dashboard.HubToken.v1`, and returns the protected - string. Lifetime is 30 minutes. + string. Lifetime is **5 minutes**. 3. The SignalR client passes the token as either `Authorization: Bearer …` or - `?access_token=…` (WebSocket upgrade query string). + `?access_token=…` (WebSocket upgrade query string). The query-string form is + the standard SignalR carriage for WebSocket upgrades, which cannot attach a + custom header; because it is easy to capture (proxy logs, browser history), + it must never be request-logged (see the remarks on + `HubTokenAuthenticationHandler`). 4. `HubTokenAuthenticationHandler` validates the protected payload and rebuilds the `ClaimsPrincipal` with the carried roles. 5. The hubs' `[Authorize(Policy = HubClientsPolicy)]` accepts the resulting @@ -496,7 +500,17 @@ dashboard mints short-lived bearer tokens for the connection: `DashboardHubConnectionFactory` (scoped to the Blazor circuit) wraps the HubConnectionBuilder and supplies a fresh token via `AccessTokenProvider` on -every (re)connect. +every (re)connect, so the short 5-minute lifetime is transparent to clients. + +Caveat — logout does not revoke outstanding tokens. Logout clears the dashboard +cookie, but hub bearer tokens are self-contained, data-protection-encrypted, and +carry no server-side revocation state (no jti denylist). A token captured before +logout remains valid until it expires, and a role change or key revocation does +not take effect on an already-issued token until then. The 5-minute lifetime is +the deliberate mitigation: it bounds that exposure window without the cost of a +revocation store. Server-side revocation is deferred until per-session hub ACLs +land (see the per-session-ACL note), at which point tokens gain session/role +binding and a denylist becomes worthwhile. ## Configuration diff --git a/docs/Metrics.md b/docs/Metrics.md index 202b00a..64836a4 100644 --- a/docs/Metrics.md +++ b/docs/Metrics.md @@ -203,6 +203,18 @@ metrics.RecordEventStreamSend(publicEvent.Family.ToString(), stopwatch.Elapsed); `Dashboard/DashboardSnapshotService.cs` calls `_metrics.GetSnapshot()` once per `GetSnapshot` invocation and projects it into the dashboard transport types together with the session registry view. The dashboard receives a single, internally consistent snapshot per tick rather than reading individual counters at separate times. See [Gateway Dashboard Design](./GatewayDashboardDesign.md) and [Dashboard Interface Design](./DashboardInterfaceDesign.md) for the projection rules and wire format. +## Auth Store Write Churn + +The per-RPC authentication path is a hidden write source: the shared verifier +refreshes `last_used_utc` on every authenticated call. To keep that off the +auth-store throughput ceiling, the gateway caches successful verifications for a +short TTL and coalesces the `last_used_utc` write to at most one per key per +window (`MxGateway:Security:ApiKeyVerificationCacheSeconds` / +`ApiKeyLastUsedCoalesceSeconds`, defaults 15 s / 60 s). This bounds WAL churn on +the bulk-read workload without dropping any metric — auth activity is still +observable via command-throughput counters and audit rows. See +[Authentication](./Authentication.md#hot-path-caching-and-last-used-coalescing). + ## Related Documentation - [Gateway Dashboard Design](./GatewayDashboardDesign.md) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayConfigurationServiceCollectionExtensions.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayConfigurationServiceCollectionExtensions.cs index 814e9ac..863db3b 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayConfigurationServiceCollectionExtensions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayConfigurationServiceCollectionExtensions.cs @@ -1,4 +1,7 @@ using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.FileProviders; +using Microsoft.Extensions.Hosting; using ZB.MOM.WW.Configuration; namespace ZB.MOM.WW.MxGateway.Server.Configuration; @@ -12,6 +15,14 @@ public static class GatewayConfigurationServiceCollectionExtensions public static IServiceCollection AddGatewayConfiguration( this IServiceCollection services, IConfiguration configuration) { + // GatewayOptionsValidator depends on IHostEnvironment to gate the production-only rules + // (SEC-04/06). The real host and the apikey CLI both build through WebApplicationBuilder, + // which registers IHostEnvironment before this call — so TryAdd is a no-op there. Minimal + // containers (unit tests, tooling) have none; fall back to a non-production environment so + // the validator still activates and the production guards stay off outside a real + // deployment (the fail-fast guards only make sense against a real host environment). + services.TryAddSingleton(new FallbackHostEnvironment()); + services.AddValidatedOptions( configuration, GatewayOptions.SectionName); @@ -19,4 +30,19 @@ public static class GatewayConfigurationServiceCollectionExtensions return services; } + + /// + /// Non-production used only when no real host registered one + /// (minimal test/tooling containers). The real host's registration always wins via TryAdd. + /// + private sealed class FallbackHostEnvironment : IHostEnvironment + { + public string EnvironmentName { get; set; } = Environments.Development; + + public string ApplicationName { get; set; } = typeof(FallbackHostEnvironment).Assembly.GetName().Name ?? "MxGateway"; + + public string ContentRootPath { get; set; } = AppContext.BaseDirectory; + + public IFileProvider ContentRootFileProvider { get; set; } = new NullFileProvider(); + } } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptions.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptions.cs index f3d41e5..3063604 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptions.cs @@ -46,4 +46,10 @@ public sealed class GatewayOptions /// Gets self-signed TLS certificate auto-generation options. public TlsOptions Tls { get; init; } = new(); + + /// + /// Gets security hot-path options (API-key verification cache / last-used coalescing and + /// login / API-key rate limiting). + /// + public SecurityOptions Security { get; init; } = new(); } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs index 686b168..34c50e7 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs @@ -48,6 +48,43 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase + /// Determines whether is an absolute path for any platform, + /// not just the host running the validator. This matters on the macOS dev box, where the + /// production appsettings.json ships Windows-absolute paths (C:\ProgramData\...) + /// that reports as non-rooted on Unix. The intent of the + /// rooting check is to reject bare filenames that resolve against the launch working directory, + /// so a valid Windows drive-qualified or UNC path must pass regardless of the current OS. + /// + private static bool IsRootedForAnyPlatform(string value) + { + // Rooted on the current OS (Unix "/...", or a Windows drive/UNC path when on Windows). + if (Path.IsPathRooted(value)) + { + return true; + } + + // Windows drive-qualified path ("C:\..." or "C:/...") checked on a non-Windows host. + if (value.Length >= 3 + && char.IsLetter(value[0]) + && value[1] == ':' + && (value[2] == '\\' || value[2] == '/')) + { + return true; + } + + // Windows UNC path ("\\server\share") checked on a non-Windows host. + return value.StartsWith(@"\\", StringComparison.Ordinal); + } + private static void AddIfInvalidPath(string? value, string message, ValidationBuilder builder) { if (string.IsNullOrWhiteSpace(value)) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/SecurityOptions.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/SecurityOptions.cs new file mode 100644 index 0000000..c664edb --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/SecurityOptions.cs @@ -0,0 +1,65 @@ +namespace ZB.MOM.WW.MxGateway.Server.Configuration; + +/// +/// Security hot-path options bound from MxGateway:Security. Groups the API-key verification +/// cache and last_used write-coalescing knobs (SEC-08) with the login and API-key +/// rate-limiting knobs (SEC-11). Defaults preserve safe, low-overhead behaviour without requiring +/// operators to configure anything. +/// +public sealed class SecurityOptions +{ + /// The configuration sub-section this binds from. + public const string SectionName = "MxGateway:Security"; + + /// + /// Gets the time-to-live, in seconds, of a cached successful API-key verification. A cache hit + /// within this window skips the per-call SQLite read (and the library verifier's coupled + /// last_used write). Explicit invalidation on gateway-initiated revoke/rotate is the + /// primary correctness mechanism; this short TTL is the backstop for out-of-band mutations. + /// Set to 0 to disable caching. Default is 15 seconds. + /// + public int ApiKeyVerificationCacheSeconds { get; init; } = 15; + + /// + /// Gets the coalescing window, in seconds, for the last_used_utc write. The library + /// verifier couples the write into every verification; this decorator forwards at most one + /// MarkUsed per key per window, so a hammered key produces at most one write per window + /// rather than one per authenticated RPC. Set to 0 to forward every write. Default is + /// 60 seconds (at most one write per key per minute). + /// + public int ApiKeyLastUsedCoalesceSeconds { get; init; } = 60; + + /// + /// Gets the maximum number of POST /auth/login attempts permitted per remote IP within + /// before requests are rejected with HTTP 429. Default + /// is 10. + /// + public int LoginRateLimitPermitLimit { get; init; } = 10; + + /// + /// Gets the fixed-window length, in seconds, for the POST /auth/login per-IP rate limit. + /// Default is 60 seconds. + /// + public int LoginRateLimitWindowSeconds { get; init; } = 60; + + /// + /// Gets the number of consecutive failed API-key verifications, per peer, within + /// that trips the in-process short-circuit. Once tripped, + /// the gRPC auth path rejects further attempts before the store read; a successful verification + /// resets the peer's counter. Default is 10. + /// + public int ApiKeyFailureLimit { get; init; } = 10; + + /// + /// Gets the sliding-window length, in seconds, over which API-key verification failures are + /// counted per peer. Default is 60 seconds. + /// + public int ApiKeyFailureWindowSeconds { get; init; } = 60; + + /// + /// Gets the maximum number of distinct peers tracked by the API-key failure counter. The counter + /// is a bounded LRU so a spray of unique peer keys cannot grow memory without limit. Default is + /// 4096. + /// + public int ApiKeyFailureTrackedPeers { get; init; } = 4096; +} diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardApiKeyManagementService.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardApiKeyManagementService.cs index 315c05a..6215e83 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardApiKeyManagementService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardApiKeyManagementService.cs @@ -15,7 +15,8 @@ public sealed class DashboardApiKeyManagementService( ApiKeyAdminCommands adminCommands, IApiKeyAdminStore adminStore, IAuditWriter auditWriter, - IHttpContextAccessor httpContextAccessor) : IDashboardApiKeyManagementService + IHttpContextAccessor httpContextAccessor, + IApiKeyCacheInvalidator? cacheInvalidator = null) : IDashboardApiKeyManagementService { private const string UnauthorizedMessage = "Sign in with an authorized LDAP account to manage API keys."; private const string PepperUnavailableMarker = "pepper unavailable"; @@ -100,6 +101,10 @@ public sealed class DashboardApiKeyManagementService( .RevokeKeyAsync(normalizedKeyId, RemoteAddress(), cancellationToken) .ConfigureAwait(false); + // SEC-08: drop any cached verification for this key in THIS gateway process so the revoke + // takes effect immediately rather than after the verification-cache TTL elapses. + cacheInvalidator?.Invalidate(normalizedKeyId); + await WriteDashboardAuditAsync( user, normalizedKeyId, @@ -138,6 +143,10 @@ public sealed class DashboardApiKeyManagementService( .RotateKeyAsync(normalizedKeyId, RemoteAddress(), cancellationToken) .ConfigureAwait(false); + // SEC-08: the old secret's cached verification must not outlive the rotate — evict it so + // the superseded secret stops authenticating within this process immediately. + cacheInvalidator?.Invalidate(normalizedKeyId); + bool succeeded = rotated.Token is not null; await WriteDashboardAuditAsync( @@ -182,6 +191,10 @@ public sealed class DashboardApiKeyManagementService( .DeleteAsync(normalizedKeyId, cancellationToken) .ConfigureAwait(false); + // SEC-08: a deleted key is only reachable after a prior revoke, but evict defensively so no + // cached verification survives the delete. + cacheInvalidator?.Invalidate(normalizedKeyId); + await WriteDashboardAuditAsync( user, normalizedKeyId, diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardEndpointRouteBuilderExtensions.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardEndpointRouteBuilderExtensions.cs index bf68cd1..dea4e5f 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardEndpointRouteBuilderExtensions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardEndpointRouteBuilderExtensions.cs @@ -1,4 +1,5 @@ using System.Text.Encodings.Web; +using System.Threading.RateLimiting; using Microsoft.AspNetCore.Antiforgery; using Microsoft.AspNetCore.Authentication; using ZB.MOM.WW.MxGateway.Server.Configuration; @@ -10,6 +11,41 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard; /// Endpoint extensions for registering the gateway dashboard routes. public static class DashboardEndpointRouteBuilderExtensions { + /// + /// The named rate-limiter policy applied to the POST /auth/login route (SEC-11). Registered + /// in GatewayApplication via . + /// + internal const string LoginRateLimiterPolicy = "dashboard-login"; + + /// + /// Builds the fixed-window rate-limit partition for a login request, keyed on the remote IP. + /// Shared by the production policy registration and the tests so both exercise identical limits. + /// + /// The inbound request. + /// The bound security options carrying the login-limit knobs. + /// The per-IP fixed-window partition. + internal static RateLimitPartition GetLoginRateLimitPartition( + HttpContext httpContext, + SecurityOptions security) + { + string partitionKey = httpContext.Connection.RemoteIpAddress?.ToString() ?? "unknown"; + return RateLimitPartition.GetFixedWindowLimiter( + partitionKey, + _ => new FixedWindowRateLimiterOptions + { + PermitLimit = security.LoginRateLimitPermitLimit, + Window = TimeSpan.FromSeconds(security.LoginRateLimitWindowSeconds), + QueueLimit = 0, + AutoReplenishment = true, + }); + } + + // Test seam: a standalone partitioned limiter over the production partition logic, so a test can + // acquire leases and assert the (permit+1)th is rejected without an HTTP server. + internal static PartitionedRateLimiter CreateLoginRateLimiter(SecurityOptions security) + => PartitionedRateLimiter.Create( + ctx => GetLoginRateLimitPartition(ctx, security)); + /// Maps all gateway dashboard routes including login, logout, and Razor components. /// The endpoint route builder. /// The route builder for chaining. @@ -40,6 +76,8 @@ public static class DashboardEndpointRouteBuilderExtensions (HttpContext httpContext, IAntiforgery antiforgery, IDashboardAuthenticator authenticator) => PostLoginAsync(httpContext, antiforgery, authenticator)) .AllowAnonymous() + // SEC-11: throttle credential stuffing before the LDAP bind is relayed to the directory. + .RequireRateLimiting(LoginRateLimiterPolicy) .WithName("DashboardLoginPost"); endpoints.MapPost( diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/HubTokenAuthenticationHandler.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/HubTokenAuthenticationHandler.cs index af9d9ca..0a5622e 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/HubTokenAuthenticationHandler.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/HubTokenAuthenticationHandler.cs @@ -10,6 +10,16 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard; /// string (used by WebSocket upgrade requests that can't carry custom headers) /// and validating it via . /// +/// +/// Carrying the token in the ?access_token= query string is the standard SignalR pattern: +/// the WebSocket upgrade handshake cannot attach a custom Authorization header, so the JS +/// client appends the token to the URL. Because a query string is far easier to capture than a +/// header (proxy access logs, browser history, Referer), this value must NEVER be +/// request-logged. Serilog HTTP request logging is intentionally not enabled; if it is ever added, +/// scrub the access_token query parameter before the URL reaches a sink. The short token +/// lifetime () is the backstop that bounds exposure of a +/// leaked token. +/// public sealed class HubTokenAuthenticationHandler : AuthenticationHandler { private readonly HubTokenService _tokens; diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/HubTokenService.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/HubTokenService.cs index 773fb74..168f818 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/HubTokenService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/HubTokenService.cs @@ -26,7 +26,15 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard; public sealed class HubTokenService { private const string ProtectorPurpose = "ZB.MOM.WW.MxGateway.Dashboard.HubToken.v1"; - private static readonly TimeSpan TokenLifetime = TimeSpan.FromMinutes(30); + + // Hub bearer tokens are single-purpose, data-protection-encrypted, and NOT server-side + // revocable. A short lifetime bounds the exposure window of a token captured from a proxy + // or log after logout (the cookie is cleared on logout, but outstanding tokens are not), and + // bounds how long a stale role set survives a role change. Five minutes is transparent to + // clients because DashboardHubConnectionFactory mints a fresh token on every (re)connect; + // see docs/GatewayDashboardDesign.md. Heavier jti-denylist revocation is deliberately + // deferred until per-session hub ACLs land (SEC-25), when tokens gain session binding. + internal static readonly TimeSpan TokenLifetime = TimeSpan.FromMinutes(5); private readonly ITimeLimitedDataProtector _protector; @@ -41,14 +49,24 @@ public sealed class HubTokenService /// Issues a bearer token carrying the user's identity and roles. /// The claims principal representing the user. /// The data-protected bearer token string. - public string Issue(ClaimsPrincipal user) + public string Issue(ClaimsPrincipal user) => Issue(user, TokenLifetime); + + /// + /// Issues a bearer token that expires after the supplied lifetime. Test seam so a caller can + /// mint an already-expired token deterministically without wall-clock delay; production callers + /// use and get . + /// + /// The claims principal representing the user. + /// The lifetime applied to the token; a non-positive value yields an already-expired token. + /// The data-protected bearer token string. + internal string Issue(ClaimsPrincipal user, TimeSpan lifetime) { ArgumentNullException.ThrowIfNull(user); HubTokenPayload payload = new( user.Identity?.Name, user.FindFirstValue(ClaimTypes.NameIdentifier), [.. user.FindAll(ClaimTypes.Role).Select(c => c.Value)]); - return _protector.Protect(JsonSerializer.Serialize(payload), TokenLifetime); + return _protector.Protect(JsonSerializer.Serialize(payload), lifetime); } /// Validates a token and returns the equivalent claims principal; null when invalid or expired. diff --git a/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs b/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs index 3dda7f5..131ef19 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/GatewayApplication.cs @@ -41,6 +41,9 @@ public static class GatewayApplication app.UseStaticFiles(); app.UseAuthentication(); app.UseAuthorization(); + // SEC-11: the rate limiter must run after routing has selected the endpoint (so its + // RequireRateLimiting policy metadata is visible) and before the endpoint executes. + app.UseRateLimiter(); app.UseAntiforgery(); app.MapGatewayEndpoints(); @@ -68,6 +71,7 @@ public static class GatewayApplication builder.Services.AddGatewayConfiguration(builder.Configuration); builder.Services.AddSqliteAuthStore(builder.Configuration); builder.Services.AddGatewayGrpcAuthorization(); + AddLoginRateLimiter(builder); builder.Services.AddHealthChecks() .AddTypeActivatedCheck( "auth-store", @@ -101,6 +105,26 @@ public static class GatewayApplication return builder; } + // SEC-11: register the named fixed-window rate-limiter policy applied to POST /auth/login. The + // limit knobs are read from MxGateway:Security at startup; the per-request partition is keyed on + // the remote IP (see DashboardEndpointRouteBuilderExtensions.GetLoginRateLimitPartition). + private static void AddLoginRateLimiter(WebApplicationBuilder builder) + { + SecurityOptions security = + builder.Configuration.GetSection(SecurityOptions.SectionName).Get() + ?? new SecurityOptions(); + + builder.Services.AddRateLimiter(options => + { + options.RejectionStatusCode = StatusCodes.Status429TooManyRequests; + options.AddPolicy( + DashboardEndpointRouteBuilderExtensions.LoginRateLimiterPolicy, + httpContext => DashboardEndpointRouteBuilderExtensions.GetLoginRateLimitPartition( + httpContext, + security)); + }); + } + private static void ConfigureSelfSignedTls(WebApplicationBuilder builder) { if (!Security.Tls.KestrelTlsInspector.RequiresGeneratedCertificate(builder.Configuration)) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/AuthStoreServiceCollectionExtensions.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/AuthStoreServiceCollectionExtensions.cs index e9138b7..5b3bc65 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/AuthStoreServiceCollectionExtensions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/AuthStoreServiceCollectionExtensions.cs @@ -1,4 +1,6 @@ +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; using ZB.MOM.WW.Audit; using ZB.MOM.WW.Auth.Abstractions.ApiKeys; @@ -6,6 +8,7 @@ using ZB.MOM.WW.Auth.ApiKeys; using ZB.MOM.WW.Auth.ApiKeys.Admin; using ZB.MOM.WW.Auth.ApiKeys.DependencyInjection; using ZB.MOM.WW.Auth.ApiKeys.Sqlite; +using ZB.MOM.WW.MxGateway.Server.Configuration; using ZB.MOM.WW.MxGateway.Server.Security.Audit; namespace ZB.MOM.WW.MxGateway.Server.Security.Authentication; @@ -66,6 +69,34 @@ public static class AuthStoreServiceCollectionExtensions // migrator and the migration hosted service. services.AddZbApiKeyAuth(effectiveConfig, AuthenticationSectionPath); + // SEC-08 hot-path decorators. Every gRPC call previously did a SQLite read plus a + // last_used_utc WRITE via IApiKeyVerifier.VerifyAsync (the library verifier couples the + // mark into verification). Two gateway-side decorators cut that cost without editing the + // external library: + // * CoalescingMarkApiKeyStore wraps the library's IApiKeyStore and coalesces the + // last_used write to at most one per key per MxGateway:Security window (default 60s), + // so the library verifier's per-call mark no longer churns the WAL. + // * CachingApiKeyVerifier wraps the library's IApiKeyVerifier with a short-TTL memory + // cache (MxGateway:Security:ApiKeyVerificationCacheSeconds, default 15s) keyed on a + // SHA-256 hash of the presented token, so a cache hit skips the store read entirely. + // Invalidation of a cached verification on a gateway-initiated revoke/rotate/delete is + // wired at the dashboard admin call site (DashboardApiKeyManagementService) via + // IApiKeyCacheInvalidator; the short TTL is the backstop for out-of-band mutations. + // Read the security knobs straight off IConfiguration rather than IOptions: + // touching IOptions.Value would force the whole-options validation pipeline + // (which needs IHostEnvironment) at auth-store resolution, breaking the bare-DI auth tests. + // GatewayOptions is still validated at startup, which covers these same values. + SecurityOptions security = configuration.GetSection(SecurityOptions.SectionName).Get() + ?? new SecurityOptions(); + services.AddMemoryCache(); + DecorateSingleton( + services, + (sp, inner) => new CoalescingMarkApiKeyStore( + inner, + security, + sp.GetService() ?? TimeProvider.System)); + DecorateVerifierWithCache(services, security); + services.AddSingleton(sp => new SqliteCanonicalAuditStore(sp.GetRequiredService())); // Resolve the logger defensively: the production host always registers ILogger, but the @@ -103,4 +134,58 @@ public static class AuthStoreServiceCollectionExtensions return services; } + + /// + /// Replaces the last registration of with a singleton that wraps + /// it. The wrapped (inner) service is created once, preserving singleton semantics. Used to layer + /// the SEC-08 store decorator over the external library's registration without editing it. + /// + private static void DecorateSingleton( + IServiceCollection services, + Func decorator) + where TService : class + { + Func innerFactory = CaptureInnerFactory(services); + services.AddSingleton(sp => decorator(sp, innerFactory(sp))); + } + + // Wraps the library's IApiKeyVerifier with CachingApiKeyVerifier and exposes the same instance + // as IApiKeyCacheInvalidator so admin call sites can drop cached verifications on revoke/rotate. + private static void DecorateVerifierWithCache(IServiceCollection services, SecurityOptions security) + { + Func innerFactory = CaptureInnerFactory(services); + services.AddSingleton(sp => new CachingApiKeyVerifier( + innerFactory(sp), + sp.GetRequiredService(), + security)); + services.AddSingleton(sp => sp.GetRequiredService()); + services.AddSingleton(sp => sp.GetRequiredService()); + } + + // Removes the current last registration of TService and returns a factory that materialises that + // original implementation exactly once when first resolved. The removed descriptor may be a + // type, factory or instance registration. + private static Func CaptureInnerFactory(IServiceCollection services) + where TService : class + { + ServiceDescriptor descriptor = services.LastOrDefault(d => d.ServiceType == typeof(TService)) + ?? throw new InvalidOperationException( + $"No existing registration for {typeof(TService)} to decorate; register the library provider first."); + services.Remove(descriptor); + + if (descriptor.ImplementationInstance is TService instance) + { + return _ => instance; + } + + if (descriptor.ImplementationFactory is not null) + { + return sp => (TService)descriptor.ImplementationFactory(sp); + } + + Type implementationType = descriptor.ImplementationType + ?? throw new InvalidOperationException( + $"Registration for {typeof(TService)} has no implementation type, factory or instance to decorate."); + return sp => (TService)ActivatorUtilities.CreateInstance(sp, implementationType); + } } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/CachingApiKeyVerifier.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/CachingApiKeyVerifier.cs new file mode 100644 index 0000000..fe3ae52 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/CachingApiKeyVerifier.cs @@ -0,0 +1,159 @@ +using System.Collections.Concurrent; +using System.Security.Cryptography; +using System.Text; +using Microsoft.Extensions.Caching.Memory; +using ZB.MOM.WW.Auth.Abstractions.ApiKeys; +using ZB.MOM.WW.MxGateway.Server.Configuration; + +namespace ZB.MOM.WW.MxGateway.Server.Security.Authentication; + +/// +/// Lets gateway-side admin call sites drop a cached API-key verification when they revoke, rotate +/// or delete a key, so a mutation applied through the running gateway process takes effect +/// immediately rather than after the cache TTL elapses. +/// +public interface IApiKeyCacheInvalidator +{ + /// Removes every cached verification for the given key id. + /// The key id whose cached verifications should be dropped. + void Invalidate(string keyId); +} + +/// +/// An decorator that caches successful verifications for a short, +/// configurable TTL (), keyed on a +/// SHA-256 hash of the presented token. A cache hit returns the cached result without calling the +/// inner (library) verifier, so it avoids both the per-call SQLite read and the last_used +/// write the library verifier couples into . +/// +/// +/// +/// Only successful verifications are cached. Failures and unparseable headers always fall through +/// to the inner verifier — caching a failure risks pinning a transiently-wrong negative, and the +/// SEC-11 per-peer failure counter (not this cache) is what bounds brute-force cost. +/// +/// +/// The cache key is the hex SHA-256 of the presented token (which embeds both the key id and the +/// secret). Hashing means the plaintext secret is never stored in memory. The hash is a cache index +/// only; the authentic constant-time secret comparison remains inside the inner verifier, reached +/// on every cache miss. +/// +/// +/// Correctness on mutation is provided by two mechanisms: gateway-initiated revoke/rotate/delete +/// call directly (see DashboardApiKeyManagementService), and the +/// short TTL is the backstop for out-of-band mutations (a direct DB edit, or a revoke issued by the +/// separate apikey CLI process, whose in-memory cache is not this process's cache). +/// +/// +public sealed class CachingApiKeyVerifier : IApiKeyVerifier, IApiKeyCacheInvalidator +{ + private const string CacheKeyPrefix = "mxgw:apikeyverif:"; + + private readonly IApiKeyVerifier _inner; + private readonly IMemoryCache _cache; + private readonly TimeSpan _ttl; + + // keyId -> set of cache keys currently held for that id, so a revoke/rotate can evict every + // secret variant (e.g. an old secret still within TTL after a rotate). + private readonly ConcurrentDictionary> _keyIdIndex = + new(StringComparer.Ordinal); + + /// Initializes a new instance of the class. + /// The wrapped verifier (the library verifier) reached on a cache miss. + /// The shared memory cache. + /// Security options carrying the verification-cache TTL. + public CachingApiKeyVerifier(IApiKeyVerifier inner, IMemoryCache cache, SecurityOptions security) + : this( + inner, + cache, + TimeSpan.FromSeconds((security ?? throw new ArgumentNullException(nameof(security))) + .ApiKeyVerificationCacheSeconds)) + { + } + + // Test/explicit-TTL seam. + internal CachingApiKeyVerifier(IApiKeyVerifier inner, IMemoryCache cache, TimeSpan ttl) + { + ArgumentNullException.ThrowIfNull(inner); + ArgumentNullException.ThrowIfNull(cache); + _inner = inner; + _cache = cache; + _ttl = ttl; + } + + /// + public async Task VerifyAsync(string authorizationHeader, CancellationToken ct) + { + if (_ttl <= TimeSpan.Zero || !TryComputeCacheKey(authorizationHeader, out string cacheKey)) + { + return await _inner.VerifyAsync(authorizationHeader, ct).ConfigureAwait(false); + } + + if (_cache.TryGetValue(cacheKey, out ApiKeyVerification? cached) && cached is not null) + { + return cached; + } + + ApiKeyVerification result = await _inner.VerifyAsync(authorizationHeader, ct).ConfigureAwait(false); + + if (result.Succeeded && result.Identity is not null) + { + _cache.Set(cacheKey, result, new MemoryCacheEntryOptions + { + AbsoluteExpirationRelativeToNow = _ttl, + }); + IndexCacheKey(result.Identity.KeyId, cacheKey); + } + + return result; + } + + /// + public void Invalidate(string keyId) + { + if (string.IsNullOrEmpty(keyId)) + { + return; + } + + if (_keyIdIndex.TryRemove(keyId, out ConcurrentDictionary? cacheKeys)) + { + foreach (string cacheKey in cacheKeys.Keys) + { + _cache.Remove(cacheKey); + } + } + } + + private void IndexCacheKey(string keyId, string cacheKey) + { + ConcurrentDictionary set = _keyIdIndex.GetOrAdd( + keyId, + static _ => new ConcurrentDictionary(StringComparer.Ordinal)); + set.TryAdd(cacheKey, 0); + } + + private static bool TryComputeCacheKey(string? authorizationHeader, out string cacheKey) + { + cacheKey = string.Empty; + if (string.IsNullOrEmpty(authorizationHeader)) + { + return false; + } + + ReadOnlySpan header = authorizationHeader.AsSpan().Trim(); + const string bearer = "Bearer "; + ReadOnlySpan token = header.StartsWith(bearer, StringComparison.OrdinalIgnoreCase) + ? header[bearer.Length..].Trim() + : header; + + if (token.IsEmpty) + { + return false; + } + + byte[] hash = SHA256.HashData(Encoding.UTF8.GetBytes(token.ToString())); + cacheKey = CacheKeyPrefix + Convert.ToHexString(hash); + return true; + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/CoalescingMarkApiKeyStore.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/CoalescingMarkApiKeyStore.cs new file mode 100644 index 0000000..9270530 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/CoalescingMarkApiKeyStore.cs @@ -0,0 +1,101 @@ +using System.Collections.Concurrent; +using ZB.MOM.WW.Auth.Abstractions.ApiKeys; +using ZB.MOM.WW.MxGateway.Server.Configuration; + +namespace ZB.MOM.WW.MxGateway.Server.Security.Authentication; + +/// +/// An decorator that coalesces writes to at +/// most one per key per window. Lookups +/// pass through unchanged. +/// +/// +/// The library verifier couples the last_used_utc write into every successful verification +/// by calling . On the bulk-read hot path that turns a +/// per-RPC database write into the throughput ceiling. This decorator sits under the library +/// verifier: it forwards the first mark for a key, then drops subsequent marks until the window +/// elapses, so last_used_utc is refreshed at most once per key per window while the read +/// path stays correct. last_used_utc is a coarse staleness indicator, not an audit record +/// (audit rows are written separately), so bounded staleness of up to one window is acceptable. +/// +public sealed class CoalescingMarkApiKeyStore : IApiKeyStore +{ + private readonly IApiKeyStore _inner; + private readonly TimeSpan _window; + private readonly TimeProvider _clock; + + // keyId -> UtcTicks of the last forwarded mark. Bounded by the number of API keys, which is + // small; no eviction needed. + private readonly ConcurrentDictionary _lastMarkedTicks = new(StringComparer.Ordinal); + + /// Initializes a new instance of the class. + /// The wrapped store. + /// Security options carrying the coalescing window. + /// The time provider. + public CoalescingMarkApiKeyStore(IApiKeyStore inner, SecurityOptions security, TimeProvider clock) + : this( + inner, + TimeSpan.FromSeconds((security ?? throw new ArgumentNullException(nameof(security))) + .ApiKeyLastUsedCoalesceSeconds), + clock) + { + } + + // Test/explicit-window seam. + internal CoalescingMarkApiKeyStore(IApiKeyStore inner, TimeSpan window, TimeProvider clock) + { + ArgumentNullException.ThrowIfNull(inner); + ArgumentNullException.ThrowIfNull(clock); + _inner = inner; + _window = window; + _clock = clock; + } + + /// + public Task FindByKeyIdAsync(string keyId, CancellationToken ct) + => _inner.FindByKeyIdAsync(keyId, ct); + + /// + public Task FindActiveByKeyIdAsync(string keyId, CancellationToken ct) + => _inner.FindActiveByKeyIdAsync(keyId, ct); + + /// + public Task MarkUsedAsync(string keyId, DateTimeOffset whenUtc, CancellationToken ct) + { + ArgumentNullException.ThrowIfNull(keyId); + + if (_window <= TimeSpan.Zero) + { + return _inner.MarkUsedAsync(keyId, whenUtc, ct); + } + + long now = _clock.GetUtcNow().UtcTicks; + + while (true) + { + if (_lastMarkedTicks.TryGetValue(keyId, out long last)) + { + if (now - last < _window.Ticks) + { + // Within the coalescing window — drop the write. + return Task.CompletedTask; + } + + if (_lastMarkedTicks.TryUpdate(keyId, now, last)) + { + return _inner.MarkUsedAsync(keyId, whenUtc, ct); + } + + // Lost the race to another writer; re-evaluate. + continue; + } + + if (_lastMarkedTicks.TryAdd(keyId, now)) + { + return _inner.MarkUsedAsync(keyId, whenUtc, ct); + } + + // Lost the race to another writer; re-evaluate. + } + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/GatewayApiKeyIdentityMapper.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/GatewayApiKeyIdentityMapper.cs index ed55248..f507a79 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/GatewayApiKeyIdentityMapper.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/GatewayApiKeyIdentityMapper.cs @@ -1,3 +1,5 @@ +using System.Collections.Concurrent; + using LibApiKeyIdentity = ZB.MOM.WW.Auth.Abstractions.ApiKeys.ApiKeyIdentity; namespace ZB.MOM.WW.MxGateway.Server.Security.Authentication; @@ -17,6 +19,36 @@ namespace ZB.MOM.WW.MxGateway.Server.Security.Authentication; /// public static class GatewayApiKeyIdentityMapper { + // SEC-08: memoize the constraints deserialization keyed by the raw JSON blob so the per-call + // JSON parse on the auth hot path collapses to a dictionary lookup. Distinct blobs are bounded + // by the number of API keys (small); ApiKeyConstraints is an immutable record, so a parsed + // instance is safe to share across callers. The cap is a defensive backstop against a pathological + // spread of distinct blobs — past it, we simply parse without caching rather than grow unbounded. + private const int MaxCachedConstraintBlobs = 1024; + private static readonly ConcurrentDictionary ConstraintCache = + new(StringComparer.Ordinal); + + private static ApiKeyConstraints DeserializeConstraints(string? constraintsJson) + { + if (string.IsNullOrWhiteSpace(constraintsJson)) + { + return ApiKeyConstraints.Empty; + } + + if (ConstraintCache.TryGetValue(constraintsJson, out ApiKeyConstraints? cached)) + { + return cached; + } + + ApiKeyConstraints parsed = ApiKeyConstraintSerializer.Deserialize(constraintsJson); + if (ConstraintCache.Count < MaxCachedConstraintBlobs) + { + ConstraintCache.TryAdd(constraintsJson, parsed); + } + + return parsed; + } + /// /// Converts a shared API-key identity into the gateway identity, deserializing the opaque /// constraints JSON into . @@ -38,6 +70,6 @@ public static class GatewayApiKeyIdentityMapper KeyPrefix: "mxgw", DisplayName: identity.DisplayName, Scopes: identity.Scopes, - Constraints: ApiKeyConstraintSerializer.Deserialize(constraintsJson)); + Constraints: DeserializeConstraints(constraintsJson)); } } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs new file mode 100644 index 0000000..730e4c4 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/ApiKeyFailureLimiter.cs @@ -0,0 +1,152 @@ +using System.Collections.Concurrent; +using ZB.MOM.WW.MxGateway.Server.Configuration; + +namespace ZB.MOM.WW.MxGateway.Server.Security.Authorization; + +/// +/// Cheap, in-process per-peer sliding-window failure counter for the gRPC auth path (SEC-11). It is +/// checked BEFORE the API-key verification store read and short-circuits a peer that has exceeded +/// failed attempts within +/// ; a successful verification resets the +/// peer's counter. +/// +/// +/// +/// The peer key is the API key id when the presented token parses, falling back to the transport +/// peer address otherwise. Keying on key id (per the NAT caveat in glauth.md) means a single +/// abusive credential behind a shared NAT is throttled without locking out unrelated clients on the +/// same address. +/// +/// +/// The tracked-peer set is a bounded LRU () so +/// a spray of unique peer keys cannot grow memory without limit. Successful peers are removed on +/// reset, so in steady state the map holds only peers with recent failures — the common success path +/// is a lock-free dictionary miss. +/// +/// +public sealed class ApiKeyFailureLimiter +{ + private readonly int _limit; + private readonly long _windowTicks; + private readonly int _maxPeers; + private readonly TimeProvider _clock; + + private readonly ConcurrentDictionary _peers = new(StringComparer.Ordinal); + + /// Initializes a new instance of the class. + /// Security options carrying the failure-limit knobs. + /// The time provider. + public ApiKeyFailureLimiter(SecurityOptions security, TimeProvider clock) + : this( + (security ?? throw new ArgumentNullException(nameof(security))).ApiKeyFailureLimit, + TimeSpan.FromSeconds(security.ApiKeyFailureWindowSeconds), + security.ApiKeyFailureTrackedPeers, + clock) + { + } + + // Test/explicit seam. + internal ApiKeyFailureLimiter(int limit, TimeSpan window, int maxPeers, TimeProvider clock) + { + ArgumentNullException.ThrowIfNull(clock); + _limit = limit; + _windowTicks = window.Ticks; + _maxPeers = maxPeers; + _clock = clock; + } + + /// Returns whether the peer has reached the failure limit within the current window. + /// The peer key (key id or peer address). + /// when the peer should be short-circuited. + public bool IsBlocked(string peer) + { + ArgumentNullException.ThrowIfNull(peer); + if (_limit <= 0) + { + return false; + } + + if (!_peers.TryGetValue(peer, out PeerState? state)) + { + return false; + } + + long now = _clock.GetUtcNow().UtcTicks; + lock (state) + { + Prune(state, now); + return state.FailureTicks.Count >= _limit; + } + } + + /// Records a failed verification attempt for the peer. + /// The peer key (key id or peer address). + public void RecordFailure(string peer) + { + ArgumentNullException.ThrowIfNull(peer); + if (_limit <= 0) + { + return; + } + + long now = _clock.GetUtcNow().UtcTicks; + PeerState state = _peers.GetOrAdd(peer, static _ => new PeerState()); + lock (state) + { + Prune(state, now); + state.FailureTicks.Enqueue(now); + state.LastActivityTicks = now; + } + + EvictIfOverCapacity(); + } + + /// Clears the peer's failure count after a successful verification. + /// The peer key (key id or peer address). + public void Reset(string peer) + { + ArgumentNullException.ThrowIfNull(peer); + _peers.TryRemove(peer, out _); + } + + private void Prune(PeerState state, long now) + { + while (state.FailureTicks.Count > 0 && now - state.FailureTicks.Peek() >= _windowTicks) + { + state.FailureTicks.Dequeue(); + } + } + + private void EvictIfOverCapacity() + { + // Best-effort eviction: only runs when the map exceeds the cap (rare, since only peers with + // recent failures are tracked). Removes the least-recently-active peer. Racy under + // concurrency, which is acceptable for a bound rather than an exact policy. + while (_peers.Count > _maxPeers) + { + string? oldest = null; + long oldestTicks = long.MaxValue; + foreach (KeyValuePair entry in _peers) + { + long activity = Volatile.Read(ref entry.Value.LastActivityTicks); + if (activity < oldestTicks) + { + oldestTicks = activity; + oldest = entry.Key; + } + } + + if (oldest is null || !_peers.TryRemove(oldest, out _)) + { + break; + } + } + } + + private sealed class PeerState + { + public Queue FailureTicks { get; } = new(); + + public long LastActivityTicks; + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcAuthorizationInterceptor.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcAuthorizationInterceptor.cs index 97533d5..0a187ef 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcAuthorizationInterceptor.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcAuthorizationInterceptor.cs @@ -15,7 +15,8 @@ public sealed class GatewayGrpcAuthorizationInterceptor( IApiKeyVerifier apiKeyVerifier, GatewayGrpcScopeResolver scopeResolver, IGatewayRequestIdentityAccessor identityAccessor, - IOptions options) : Interceptor + IOptions options, + ApiKeyFailureLimiter failureLimiter) : Interceptor { /// public override async Task UnaryServerHandler( @@ -63,6 +64,19 @@ public sealed class GatewayGrpcAuthorizationInterceptor( string? authorizationHeader = context.RequestHeaders.GetValue("authorization"); + // SEC-11: short-circuit a peer that has already failed too many times inside the sliding + // window BEFORE the verification store read, so online guessing cannot spend a SQLite read + // per attempt. The peer key prefers the presented key id over the transport address (NAT + // caveat). ResourceExhausted signals throttling without revealing whether any particular + // secret was valid. + string peerKey = ResolvePeerKey(authorizationHeader, context); + if (failureLimiter.IsBlocked(peerKey)) + { + throw new RpcException(new Status( + StatusCode.ResourceExhausted, + "Too many authentication attempts. Try again later.")); + } + // The shared verifier owns parse + pepper + lookup + revocation + constant-time compare, // returning a discriminated failure rather than throwing. Every authentication failure maps // to Unauthenticated with an opaque message; the client never learns which stage failed. @@ -72,11 +86,16 @@ public sealed class GatewayGrpcAuthorizationInterceptor( if (!verification.Succeeded || verification.Identity is null) { + failureLimiter.RecordFailure(peerKey); throw new RpcException(new Status( StatusCode.Unauthenticated, "Missing or invalid API key.")); } + // Successful authentication clears the peer's failure count so a legitimate client that + // fat-fingered a few attempts is not penalised once it recovers. + failureLimiter.Reset(peerKey); + ApiKeyIdentity identity = GatewayApiKeyIdentityMapper.ToGatewayIdentity(verification.Identity); string requiredScope = scopeResolver.ResolveRequiredScope(request); @@ -89,4 +108,30 @@ public sealed class GatewayGrpcAuthorizationInterceptor( return identity; } + + // Resolves the failure-limiter partition key: the presented key id (token shape + // mxgw__) when the header parses, otherwise the transport peer address. Keying on + // key id throttles a single abusive credential without locking out co-located clients behind NAT. + private static string ResolvePeerKey(string? authorizationHeader, ServerCallContext context) + { + if (!string.IsNullOrWhiteSpace(authorizationHeader)) + { + ReadOnlySpan header = authorizationHeader.AsSpan().Trim(); + const string bearer = "Bearer "; + ReadOnlySpan token = header.StartsWith(bearer, StringComparison.OrdinalIgnoreCase) + ? header[bearer.Length..].Trim() + : header; + + // mxgw__: the key id is the second underscore-delimited segment. The + // secret may itself contain underscores, but the key id is unaffected. + string tokenText = token.ToString(); + string[] parts = tokenText.Split('_'); + if (parts.Length >= 3 && parts[0].Length > 0 && parts[1].Length > 0) + { + return "key:" + parts[1]; + } + } + + return "peer:" + context.Peer; + } } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcScopeResolver.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcScopeResolver.cs index 08bcd83..36e756e 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcScopeResolver.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GatewayGrpcScopeResolver.cs @@ -20,6 +20,7 @@ public sealed class GatewayGrpcScopeResolver MxCommandRequest commandRequest => ResolveCommandScope(commandRequest.Command?.Kind ?? MxCommandKind.Unspecified), AcknowledgeAlarmRequest => GatewayScopes.InvokeWrite, StreamAlarmsRequest => GatewayScopes.EventsRead, + QueryActiveAlarmsRequest => GatewayScopes.EventsRead, TestConnectionRequest or GetLastDeployTimeRequest or DiscoverHierarchyRequest or diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GrpcAuthorizationServiceCollectionExtensions.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GrpcAuthorizationServiceCollectionExtensions.cs index 38ccd4d..ae697f3 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GrpcAuthorizationServiceCollectionExtensions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authorization/GrpcAuthorizationServiceCollectionExtensions.cs @@ -19,6 +19,13 @@ public static class GrpcAuthorizationServiceCollectionExtensions services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + // SEC-11 per-peer failure counter, checked before the verification store read. Bind the knobs + // from IConfiguration directly (not IOptions) to avoid coupling this + // registration to the whole-options validation pipeline. + services.AddSingleton(sp => new ApiKeyFailureLimiter( + sp.GetRequiredService().GetSection(SecurityOptions.SectionName).Get() + ?? new SecurityOptions(), + sp.GetService() ?? TimeProvider.System)); services.AddSingleton(); services .AddOptions() diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsTests.cs index 25e08ac..26e80e7 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsTests.cs @@ -14,7 +14,15 @@ public sealed class GatewayOptionsTests GatewayOptions options = BindOptions(new Dictionary()); Assert.Equal(AuthenticationMode.ApiKey, options.Authentication.Mode); - Assert.Equal(@"C:\ProgramData\MxGateway\gateway-auth.db", options.Authentication.SqlitePath); + // SEC-01: the default is derived from CommonApplicationData (C:\ProgramData on Windows, + // /usr/share on Unix) rather than a Windows literal, so assert against the same derivation + // to stay platform-correct. + Assert.Equal( + Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData), + "MxGateway", + "gateway-auth.db"), + options.Authentication.SqlitePath); Assert.Equal("MxGateway:ApiKeyPepper", options.Authentication.PepperSecretName); Assert.True(options.Authentication.RunMigrationsOnStartup); diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs index 159203c..d2614a3 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs @@ -683,4 +683,92 @@ public sealed class GatewayOptionsValidatorTests ValidateOptionsResult result = new GatewayOptionsValidator(isProduction: true).Validate(null, options); Assert.True(result.Succeeded); } + + // ---- SEC-11: MxGateway:Security validation ---- + + private static GatewayOptions WithSecurity(SecurityOptions security) => new() { Security = security }; + + /// Verifies the default security options pass validation. + [Fact] + public void Validate_Succeeds_WithDefaultSecurityOptions() + { + ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, WithSecurity(new SecurityOptions())); + Assert.True(result.Succeeded); + } + + /// Verifies a zero verification-cache TTL is allowed (disables caching). + [Fact] + public void Validate_Succeeds_WhenVerificationCacheSecondsZero() + { + ValidateOptionsResult result = new GatewayOptionsValidator().Validate( + null, + WithSecurity(new SecurityOptions { ApiKeyVerificationCacheSeconds = 0 })); + Assert.True(result.Succeeded); + } + + /// Verifies a negative verification-cache TTL fails validation. + [Fact] + public void Validate_Fails_WhenVerificationCacheSecondsNegative() + { + ValidateOptionsResult result = new GatewayOptionsValidator().Validate( + null, + WithSecurity(new SecurityOptions { ApiKeyVerificationCacheSeconds = -1 })); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, f => f.Contains("ApiKeyVerificationCacheSeconds")); + } + + /// Verifies a negative last-used coalesce window fails validation. + [Fact] + public void Validate_Fails_WhenLastUsedCoalesceSecondsNegative() + { + ValidateOptionsResult result = new GatewayOptionsValidator().Validate( + null, + WithSecurity(new SecurityOptions { ApiKeyLastUsedCoalesceSeconds = -5 })); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, f => f.Contains("ApiKeyLastUsedCoalesceSeconds")); + } + + /// Verifies a zero login rate-limit permit fails validation. + [Fact] + public void Validate_Fails_WhenLoginRateLimitPermitLimitZero() + { + ValidateOptionsResult result = new GatewayOptionsValidator().Validate( + null, + WithSecurity(new SecurityOptions { LoginRateLimitPermitLimit = 0 })); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, f => f.Contains("LoginRateLimitPermitLimit")); + } + + /// Verifies a zero login rate-limit window fails validation. + [Fact] + public void Validate_Fails_WhenLoginRateLimitWindowSecondsZero() + { + ValidateOptionsResult result = new GatewayOptionsValidator().Validate( + null, + WithSecurity(new SecurityOptions { LoginRateLimitWindowSeconds = 0 })); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, f => f.Contains("LoginRateLimitWindowSeconds")); + } + + /// Verifies a zero API-key failure limit fails validation. + [Fact] + public void Validate_Fails_WhenApiKeyFailureLimitZero() + { + ValidateOptionsResult result = new GatewayOptionsValidator().Validate( + null, + WithSecurity(new SecurityOptions { ApiKeyFailureLimit = 0 })); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, f => f.Contains("ApiKeyFailureLimit")); + } + + /// Verifies a zero tracked-peer cap fails validation. + [Fact] + public void Validate_Fails_WhenApiKeyFailureTrackedPeersZero() + { + ValidateOptionsResult result = new GatewayOptionsValidator().Validate( + null, + WithSecurity(new SecurityOptions { ApiKeyFailureTrackedPeers = 0 })); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, f => f.Contains("ApiKeyFailureTrackedPeers")); + } } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardLoginRateLimitTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardLoginRateLimitTests.cs new file mode 100644 index 0000000..523496a --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardLoginRateLimitTests.cs @@ -0,0 +1,66 @@ +using System.Net; +using System.Threading.RateLimiting; +using Microsoft.AspNetCore.Http; +using ZB.MOM.WW.MxGateway.Server.Configuration; +using ZB.MOM.WW.MxGateway.Server.Dashboard; + +namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Dashboard; + +/// +/// SEC-11: the POST /auth/login fixed-window per-IP rate limiter. Exercised through the same +/// partition factory the production policy registers, so the test pins the real permit limit and +/// per-IP partitioning without standing up an HTTP server. +/// +public sealed class DashboardLoginRateLimitTests +{ + /// A burst beyond the permit limit from one IP has its excess attempts rejected (HTTP 429 in the pipeline). + [Fact] + public void LoginRateLimiter_BurstBeyondPermitLimit_RejectsExcessFromSameIp() + { + SecurityOptions security = new() + { + LoginRateLimitPermitLimit = 3, + LoginRateLimitWindowSeconds = 60, + }; + using PartitionedRateLimiter limiter = + DashboardEndpointRouteBuilderExtensions.CreateLoginRateLimiter(security); + HttpContext context = ContextWithIp("10.0.0.7"); + + for (int i = 0; i < 3; i++) + { + using RateLimitLease lease = limiter.AttemptAcquire(context); + Assert.True(lease.IsAcquired); + } + + using RateLimitLease rejected = limiter.AttemptAcquire(context); + Assert.False(rejected.IsAcquired); + } + + /// Distinct client IPs are partitioned independently — one IP's burst does not starve another. + [Fact] + public void LoginRateLimiter_DistinctIps_PartitionedIndependently() + { + SecurityOptions security = new() + { + LoginRateLimitPermitLimit = 1, + LoginRateLimitWindowSeconds = 60, + }; + using PartitionedRateLimiter limiter = + DashboardEndpointRouteBuilderExtensions.CreateLoginRateLimiter(security); + + using RateLimitLease first = limiter.AttemptAcquire(ContextWithIp("10.0.0.1")); + using RateLimitLease exhaustedForFirst = limiter.AttemptAcquire(ContextWithIp("10.0.0.1")); + using RateLimitLease second = limiter.AttemptAcquire(ContextWithIp("10.0.0.2")); + + Assert.True(first.IsAcquired); + Assert.False(exhaustedForFirst.IsAcquired); + Assert.True(second.IsAcquired); + } + + private static HttpContext ContextWithIp(string ip) + { + DefaultHttpContext context = new(); + context.Connection.RemoteIpAddress = IPAddress.Parse(ip); + return context; + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/HubTokenServiceTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/HubTokenServiceTests.cs index 8ad4f87..4d726fc 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/HubTokenServiceTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/HubTokenServiceTests.cs @@ -115,4 +115,63 @@ public sealed class HubTokenServiceTests Assert.Null(service.Validate("this-is-not-a-protected-payload")); } + + /// + /// Issue/validate round-trip: a freshly minted token (default ) + /// validates and reconstructs the caller's identity and roles. + /// + [Fact] + public void IssueThenValidate_FreshToken_RoundTripsIdentityAndRoles() + { + HubTokenService service = new(new EphemeralDataProtectionProvider()); + ClaimsIdentity identity = new( + [ + new Claim(ClaimTypes.Name, "bob"), + new Claim(ClaimTypes.NameIdentifier, "bob-id"), + new Claim(ClaimTypes.Role, DashboardRoles.Viewer), + new Claim(ClaimTypes.Role, DashboardRoles.Admin), + ], + authenticationType: "test", + nameType: ClaimTypes.Name, + roleType: ClaimTypes.Role); + + string token = service.Issue(new ClaimsPrincipal(identity)); + ClaimsPrincipal? result = service.Validate(token); + + Assert.NotNull(result); + Assert.Equal("bob", result.Identity?.Name); + Assert.True(result.IsInRole(DashboardRoles.Viewer)); + Assert.True(result.IsInRole(DashboardRoles.Admin)); + } + + /// + /// The default token lifetime is the short (5-minute) window mandated by SEC-05, not the + /// former 30-minute window. Pins the value so a regression that widens the exposure window + /// of an irrevocable, query-string-carried token is caught in CI. + /// + [Fact] + public void TokenLifetime_IsFiveMinutes() + { + Assert.Equal(TimeSpan.FromMinutes(5), HubTokenService.TokenLifetime); + } + + /// + /// A token whose lifetime has elapsed is rejected by . + /// Uses the internal lifetime-issuing seam with a negative lifetime so the token is already + /// expired at mint time — deterministic, no wall-clock delay. + /// + [Fact] + public void Validate_ExpiredToken_ReturnsNull() + { + HubTokenService service = new(new EphemeralDataProtectionProvider()); + ClaimsIdentity identity = new( + [new Claim(ClaimTypes.Name, "carol")], + authenticationType: "test"); + + string expiredToken = service.Issue( + new ClaimsPrincipal(identity), + TimeSpan.FromMinutes(-1)); + + Assert.Null(service.Validate(expiredToken)); + } } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs new file mode 100644 index 0000000..5429448 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/CachingApiKeyVerifierTests.cs @@ -0,0 +1,201 @@ +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Time.Testing; +using ZB.MOM.WW.Auth.Abstractions.ApiKeys; +using ZB.MOM.WW.MxGateway.Server.Security.Authentication; + +using LibApiKeyIdentity = ZB.MOM.WW.Auth.Abstractions.ApiKeys.ApiKeyIdentity; + +namespace ZB.MOM.WW.MxGateway.Tests.Security.Authentication; + +/// +/// SEC-08 hot-path decorators. Covers both mechanisms: +/// (read/verification coalescing plus revoke/rotate invalidation) and +/// (the last_used write coalescing that keeps the +/// per-RPC database write off the throughput ceiling). +/// +public sealed class CachingApiKeyVerifierTests +{ + private const string Header = "Bearer mxgw_operator01_super-secret"; + + /// A cache hit within the TTL returns the cached result and never calls the inner verifier. + [Fact] + public async Task VerifyAsync_RepeatedWithinTtl_CallsInnerOnce() + { + FakeVerifier inner = new(Success("operator01")); + using MemoryCache cache = NewCache(); + CachingApiKeyVerifier verifier = new(inner, cache, TimeSpan.FromSeconds(15)); + + ApiKeyVerification first = await verifier.VerifyAsync(Header, CancellationToken.None); + ApiKeyVerification second = await verifier.VerifyAsync(Header, CancellationToken.None); + + Assert.True(first.Succeeded); + Assert.True(second.Succeeded); + Assert.Equal(1, inner.CallCount); + } + + /// Different presented secrets are cached under distinct keys (no cross-secret aliasing). + [Fact] + public async Task VerifyAsync_DifferentTokens_NotAliased() + { + FakeVerifier inner = new(Success("operator01")); + using MemoryCache cache = NewCache(); + CachingApiKeyVerifier verifier = new(inner, cache, TimeSpan.FromSeconds(15)); + + await verifier.VerifyAsync("Bearer mxgw_operator01_secret-a", CancellationToken.None); + await verifier.VerifyAsync("Bearer mxgw_operator01_secret-b", CancellationToken.None); + + Assert.Equal(2, inner.CallCount); + } + + /// Failed verifications are never cached; every attempt reaches the inner verifier. + [Fact] + public async Task VerifyAsync_FailedVerification_NotCached() + { + FakeVerifier inner = new(Failure(ApiKeyFailure.SecretMismatch)); + using MemoryCache cache = NewCache(); + CachingApiKeyVerifier verifier = new(inner, cache, TimeSpan.FromSeconds(15)); + + await verifier.VerifyAsync(Header, CancellationToken.None); + await verifier.VerifyAsync(Header, CancellationToken.None); + + Assert.Equal(2, inner.CallCount); + } + + /// A TTL of zero disables caching: the inner verifier is called on every request. + [Fact] + public async Task VerifyAsync_ZeroTtl_DisablesCache() + { + FakeVerifier inner = new(Success("operator01")); + using MemoryCache cache = NewCache(); + CachingApiKeyVerifier verifier = new(inner, cache, TimeSpan.Zero); + + await verifier.VerifyAsync(Header, CancellationToken.None); + await verifier.VerifyAsync(Header, CancellationToken.None); + + Assert.Equal(2, inner.CallCount); + } + + /// Invalidating a key id (revoke/rotate) drops its cached verification, forcing a re-verify. + [Fact] + public async Task Invalidate_DropsCachedEntry_ForcesReverify() + { + FakeVerifier inner = new(Success("operator01")); + using MemoryCache cache = NewCache(); + CachingApiKeyVerifier verifier = new(inner, cache, TimeSpan.FromSeconds(30)); + + await verifier.VerifyAsync(Header, CancellationToken.None); + Assert.Equal(1, inner.CallCount); + + ((IApiKeyCacheInvalidator)verifier).Invalidate("operator01"); + + await verifier.VerifyAsync(Header, CancellationToken.None); + Assert.Equal(2, inner.CallCount); + } + + /// + /// The store decorator coalesces repeated MarkUsed writes for the same key inside the + /// window down to a single forwarded write — the ≤1/min guarantee for last_used_utc. + /// + [Fact] + public async Task CoalescingStore_RepeatedMarksWithinWindow_ForwardsOnce() + { + FakeStore inner = new(); + FakeTimeProvider clock = new(DateTimeOffset.UtcNow); + CoalescingMarkApiKeyStore store = new(inner, TimeSpan.FromMinutes(1), clock); + + // Ten rapid authenticated calls in the same minute. + for (int i = 0; i < 10; i++) + { + await store.MarkUsedAsync("operator01", clock.GetUtcNow(), CancellationToken.None); + clock.Advance(TimeSpan.FromSeconds(5)); + } + + Assert.Equal(1, inner.MarkUsedCount); + } + + /// After the window elapses the next mark is forwarded again (staleness is bounded, not frozen). + [Fact] + public async Task CoalescingStore_AfterWindow_ForwardsAgain() + { + FakeStore inner = new(); + FakeTimeProvider clock = new(DateTimeOffset.UtcNow); + CoalescingMarkApiKeyStore store = new(inner, TimeSpan.FromMinutes(1), clock); + + await store.MarkUsedAsync("operator01", clock.GetUtcNow(), CancellationToken.None); + clock.Advance(TimeSpan.FromSeconds(61)); + await store.MarkUsedAsync("operator01", clock.GetUtcNow(), CancellationToken.None); + + Assert.Equal(2, inner.MarkUsedCount); + } + + /// Distinct keys are coalesced independently. + [Fact] + public async Task CoalescingStore_DistinctKeys_TrackedSeparately() + { + FakeStore inner = new(); + FakeTimeProvider clock = new(DateTimeOffset.UtcNow); + CoalescingMarkApiKeyStore store = new(inner, TimeSpan.FromMinutes(1), clock); + + await store.MarkUsedAsync("key-a", clock.GetUtcNow(), CancellationToken.None); + await store.MarkUsedAsync("key-b", clock.GetUtcNow(), CancellationToken.None); + await store.MarkUsedAsync("key-a", clock.GetUtcNow(), CancellationToken.None); + + Assert.Equal(2, inner.MarkUsedCount); + } + + /// A zero window disables coalescing: every mark is forwarded. + [Fact] + public async Task CoalescingStore_ZeroWindow_ForwardsEveryMark() + { + FakeStore inner = new(); + FakeTimeProvider clock = new(DateTimeOffset.UtcNow); + CoalescingMarkApiKeyStore store = new(inner, TimeSpan.Zero, clock); + + await store.MarkUsedAsync("operator01", clock.GetUtcNow(), CancellationToken.None); + await store.MarkUsedAsync("operator01", clock.GetUtcNow(), CancellationToken.None); + + Assert.Equal(2, inner.MarkUsedCount); + } + + private static MemoryCache NewCache() => new(new MemoryCacheOptions()); + + private static ApiKeyVerification Success(string keyId) => new( + Succeeded: true, + Identity: new LibApiKeyIdentity( + KeyId: keyId, + DisplayName: "Operator Key", + Scopes: new HashSet(StringComparer.Ordinal), + Constraints: null), + Failure: null); + + private static ApiKeyVerification Failure(ApiKeyFailure failure) => + new(Succeeded: false, Identity: null, Failure: failure); + + private sealed class FakeVerifier(ApiKeyVerification result) : IApiKeyVerifier + { + public int CallCount { get; private set; } + + public Task VerifyAsync(string authorizationHeader, CancellationToken ct) + { + CallCount++; + return Task.FromResult(result); + } + } + + private sealed class FakeStore : IApiKeyStore + { + public int MarkUsedCount { get; private set; } + + public Task FindByKeyIdAsync(string keyId, CancellationToken ct) + => Task.FromResult(null); + + public Task FindActiveByKeyIdAsync(string keyId, CancellationToken ct) + => Task.FromResult(null); + + public Task MarkUsedAsync(string keyId, DateTimeOffset whenUtc, CancellationToken ct) + { + MarkUsedCount++; + return Task.CompletedTask; + } + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authorization/GatewayGrpcAuthorizationInterceptorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authorization/GatewayGrpcAuthorizationInterceptorTests.cs index 0e66945..796739b 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authorization/GatewayGrpcAuthorizationInterceptorTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authorization/GatewayGrpcAuthorizationInterceptorTests.cs @@ -327,7 +327,7 @@ public sealed class GatewayGrpcAuthorizationInterceptorTests RpcException exception = await Assert.ThrowsAsync( () => interceptor.ServerStreamingServerHandler( - new StreamAlarmsRequest(), + new QueryActiveAlarmsRequest(), new RecordingServerStreamWriter(), ContextWithAuthorization("Bearer mxgw_operator01_secret"), (_, _, _) => Task.CompletedTask)); @@ -347,7 +347,7 @@ public sealed class GatewayGrpcAuthorizationInterceptorTests RecordingServerStreamWriter streamWriter = new(); await interceptor.ServerStreamingServerHandler( - new StreamAlarmsRequest(), + new QueryActiveAlarmsRequest(), streamWriter, ContextWithAuthorization("Bearer mxgw_operator01_secret"), async (_, writer, _) => @@ -358,6 +358,102 @@ public sealed class GatewayGrpcAuthorizationInterceptorTests Assert.Single(streamWriter.Messages); } + /// + /// SEC-11: once a peer has exceeded the failure limit, the interceptor short-circuits with + /// BEFORE calling the verifier, so an online guessing + /// loop stops spending a store read per attempt. A verifier that always fails is used; after the + /// limit is reached the verifier is no longer invoked. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task UnaryServerHandler_ExceedsFailureLimit_ShortCircuitsBeforeVerify() + { + CountingFailureVerifier verifier = new(Failure(ApiKeyFailure.SecretMismatch)); + ApiKeyFailureLimiter limiter = new( + limit: 3, + window: TimeSpan.FromMinutes(1), + maxPeers: 16, + clock: TimeProvider.System); + GatewayGrpcAuthorizationInterceptor interceptor = CreateInterceptor( + verifier, + new GatewayRequestIdentityAccessor(), + failureLimiter: limiter); + + // The first three attempts reach the verifier and fail (recording a failure each time). + for (int attempt = 0; attempt < 3; attempt++) + { + RpcException failure = await Assert.ThrowsAsync( + () => interceptor.UnaryServerHandler( + new OpenSessionRequest(), + ContextWithAuthorization("Bearer mxgw_operator01_bad-secret"), + (_, _) => Task.FromResult(new OpenSessionReply()))); + Assert.Equal(StatusCode.Unauthenticated, failure.StatusCode); + } + + Assert.Equal(3, verifier.CallCount); + + // The fourth attempt is short-circuited: ResourceExhausted, and the verifier is NOT called. + RpcException throttled = await Assert.ThrowsAsync( + () => interceptor.UnaryServerHandler( + new OpenSessionRequest(), + ContextWithAuthorization("Bearer mxgw_operator01_bad-secret"), + (_, _) => Task.FromResult(new OpenSessionReply()))); + + Assert.Equal(StatusCode.ResourceExhausted, throttled.StatusCode); + Assert.Equal(3, verifier.CallCount); + } + + /// + /// SEC-11: a successful verification resets the peer's failure counter, so accumulated failures + /// from a fat-fingered secret do not lock out a client that subsequently authenticates. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task UnaryServerHandler_SuccessResetsFailureCounter() + { + ApiKeyFailureLimiter limiter = new( + limit: 3, + window: TimeSpan.FromMinutes(1), + maxPeers: 16, + clock: TimeProvider.System); + + // Two failures against the same key id, then a success (which resets), then two more + // failures — without the reset the fifth attempt would be blocked at the limit of 3. + GatewayGrpcAuthorizationInterceptor failing = CreateInterceptor( + new FakeApiKeyVerifier(Failure(ApiKeyFailure.SecretMismatch)), + new GatewayRequestIdentityAccessor(), + failureLimiter: limiter); + GatewayGrpcAuthorizationInterceptor succeeding = CreateInterceptor( + new FakeApiKeyVerifier(SuccessWithScopes(GatewayScopes.SessionOpen)), + new GatewayRequestIdentityAccessor(), + failureLimiter: limiter); + + for (int i = 0; i < 2; i++) + { + await Assert.ThrowsAsync( + () => failing.UnaryServerHandler( + new OpenSessionRequest(), + ContextWithAuthorization("Bearer mxgw_operator01_bad"), + (_, _) => Task.FromResult(new OpenSessionReply()))); + } + + await succeeding.UnaryServerHandler( + new OpenSessionRequest(), + ContextWithAuthorization("Bearer mxgw_operator01_good"), + (_, _) => Task.FromResult(new OpenSessionReply { SessionId = "s" })); + + // Post-reset: two more failures still map to Unauthenticated (not ResourceExhausted). + for (int i = 0; i < 2; i++) + { + RpcException ex = await Assert.ThrowsAsync( + () => failing.UnaryServerHandler( + new OpenSessionRequest(), + ContextWithAuthorization("Bearer mxgw_operator01_bad"), + (_, _) => Task.FromResult(new OpenSessionReply()))); + Assert.Equal(StatusCode.Unauthenticated, ex.StatusCode); + } + } + private static MxAccessGatewayService CreateService( ISessionManager sessionManager, IGatewayRequestIdentityAccessor identityAccessor) @@ -377,7 +473,8 @@ public sealed class GatewayGrpcAuthorizationInterceptorTests private static GatewayGrpcAuthorizationInterceptor CreateInterceptor( IApiKeyVerifier apiKeyVerifier, IGatewayRequestIdentityAccessor identityAccessor, - AuthenticationMode authenticationMode = AuthenticationMode.ApiKey) + AuthenticationMode authenticationMode = AuthenticationMode.ApiKey, + ApiKeyFailureLimiter? failureLimiter = null) { return new GatewayGrpcAuthorizationInterceptor( apiKeyVerifier, @@ -389,7 +486,12 @@ public sealed class GatewayGrpcAuthorizationInterceptorTests { Mode = authenticationMode } - })); + }), + failureLimiter ?? new ApiKeyFailureLimiter( + limit: 1000, + window: TimeSpan.FromMinutes(1), + maxPeers: 1000, + clock: TimeProvider.System)); } private static ApiKeyVerification SuccessWithScopes(params string[] scopes) @@ -531,6 +633,22 @@ public sealed class GatewayGrpcAuthorizationInterceptorTests } } + private sealed class CountingFailureVerifier(ApiKeyVerification result) : IApiKeyVerifier + { + /// Gets the number of times the verifier was invoked. + public int CallCount { get; private set; } + + /// Returns the configured result and counts the invocation. + /// The authorization header to verify. + /// Cancellation token. + /// The configured verification result. + public Task VerifyAsync(string authorizationHeader, CancellationToken ct) + { + CallCount++; + return Task.FromResult(result); + } + } + private sealed class FakeApiKeyVerifier(ApiKeyVerification result) : IApiKeyVerifier { /// Gets whether the verifier was called. diff --git a/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TestHostEnvironmentInitializer.cs b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TestHostEnvironmentInitializer.cs new file mode 100644 index 0000000..580de0e --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TestHostEnvironmentInitializer.cs @@ -0,0 +1,29 @@ +using System.Runtime.CompilerServices; + +namespace ZB.MOM.WW.MxGateway.Tests.TestSupport; + +/// +/// Defaults the host environment to Development for the whole test assembly. +/// +/// +/// Many tests build the full gateway host through GatewayApplication.Build against the dev +/// appsettings.json (which ships Ldap:Transport=None and other dev-only defaults). +/// An unset environment resolves to Production, where the SEC-04/06 production guards fail startup +/// by design — so those host-building tests would trip the guards. Setting the environment to +/// Development once, before any test runs, keeps that suite exercising app wiring rather than +/// production-config validation. Tests that specifically assert production behavior construct +/// GatewayOptionsValidator with its isProduction constructor (no host environment) and +/// are unaffected; a test that needs Production can still pass --environment=Production, which +/// overrides this default. +/// +internal static class TestHostEnvironmentInitializer +{ + [ModuleInitializer] + internal static void SetDevelopmentEnvironmentDefault() + { + if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT"))) + { + Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); + } + } +} From c2df4a0aaed225fb54714906e133ec347b3d5069 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 07:32:14 -0400 Subject: [PATCH 08/19] docs(archreview): P1 Wave 2b status (SEC-05/07/08/11 Done) --- archreview/remediation/00-tracking.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/archreview/remediation/00-tracking.md b/archreview/remediation/00-tracking.md index 3a928a9..0d2a49d 100644 --- a/archreview/remediation/00-tracking.md +++ b/archreview/remediation/00-tracking.md @@ -143,13 +143,13 @@ Full design + implementation for each row lives in the linked domain doc under i | SEC-02 | Medium | P1 | S | — | Not started | `AllowAnonymousLocalhost` satisfies AdminOnly, not just Viewer | | SEC-03 | Medium | P2 | S | — | Not started | Dashboard cookie lost its `__Host-` prefix; four docs still promise it | | SEC-04 | Medium | P1 | S | SEC-03 | Done | `DisableLogin` has no production guard | -| SEC-05 | Medium | P1 | M | — | Not started | Hub bearer tokens irrevocable for 30 min, carried in query string | +| SEC-05 | Medium | P1 | M | — | Done | Hub bearer tokens irrevocable for 30 min, carried in query string | | SEC-06 | Medium | P1 | M | — | Done | LDAP plaintext-by-default with a committed service password | -| SEC-07 | Medium | P1 | S | — | Not started | `QueryActiveAlarmsRequest` missing from scope resolver; tests mislabeled | -| SEC-08 | Medium | P1 | L | — | Not started | Per-RPC SQLite read + `last_used_utc` write; no verification cache | +| SEC-07 | Medium | P1 | S | — | Done | `QueryActiveAlarmsRequest` missing from scope resolver; tests mislabeled | +| SEC-08 | Medium | P1 | L | — | Done | Per-RPC SQLite read + `last_used_utc` write; no verification cache | | SEC-09 | Medium | P2 | S | — | Not started | Dashboard design-doc `GroupToRole` sample now fails startup validation | | SEC-10 | Medium | P1 | L | — | Done | No API-key expiry | -| SEC-11 | Medium | P1 | M | SEC-08 | Not started | No rate limiting or lockout on either auth surface | +| SEC-11 | Medium | P1 | M | SEC-08 | Done | No rate limiting or lockout on either auth surface | | SEC-12 | Medium | P1 | M | — | Not started | Dashboard Close/Kill bypass the canonical audit store | | SEC-13 | Low | — | S | SEC-30 | Not started | Redactor's credential-command list omits secured-bulk variants | | SEC-14 | Low | — | S | SEC-02,20 | Not started | `/metrics` + `/health` unauthenticated; a metric leaks session ids | @@ -255,6 +255,7 @@ Findings the review flagged as one coordinated design pass — sequence them tog |---|---| | 2026-07-09 | Initial tracking doc generated from the six domain remediation designs. All 153 findings `Not started`. | | 2026-07-09 | P0 tier executed via parallel agents. GWC-01/02/03, TST-02, TST-12, CLI-01, CLI-03 → `Done` (NonWindows build clean; gateway 135/135 targeted tests pass, Go `go test ./...` + `-race` pass, Rust `test`/`clippy -D warnings` pass). | +| 2026-07-09 | P1 Wave 2b (security authz+hub). SEC-05/07/08/11 → `Done` (hub-token lifetime; QueryActiveAlarms scope arm; gateway-side verification cache + last-used coalescing; login rate limit + per-peer gRPC failure limiter). Full-suite checkpoint caught + fixed regressions the earlier narrow SEC-01/04/06 filter missed: cross-platform path-rooting, an `IHostEnvironment` fallback for minimal DI containers, a test-assembly `ASPNETCORE_ENVIRONMENT=Development` default, and a platform-correct default-path assertion. Suite: 747 passed / 42 failed, all 42 pre-existing macOS named-pipe-harness env failures. | | 2026-07-09 | P1 Wave 2a (security). SEC-01/04/06 → `Done` (config path-rooting + production validator guards; Server build clean, validator+hygiene tests 53/53). SEC-10 → `Done`: the shared `ZB.MOM.WW.Auth.ApiKeys` gained optional `ExpiresUtc` (expired keys rejected, auth DB auto-migrates to schema v3) via a concurrent HistorianGateway-remediation session's "G-2"; this repo consumes it by bumping the four `Auth.*` refs 0.1.2→0.1.4 (commit 197731a). Remaining SEC-10 polish (`apikey create --expires` + dashboard staleness badge) tracked as a small follow-up. | | 2026-07-09 | P1 Wave 1 (CI + codegen freshness + Rust buildability) via parallel agents. IPC-01/09/19/20, CLI-02 → `Done`; TST-03 → `In review` (CI pipeline authored + YAML/layout-validated, but not yet executed on a Gitea runner — proven on first push). Verified on macOS: NonWindows build clean, `ClientProtoInputTests` 5/5, `publish-client-proto-inputs.ps1 -Check` exit 0, `cargo package` (no `--no-verify`) compiles standalone. Added a vendored-Rust-proto drift guard (Check 3) to `check-codegen.ps1` closing the CLI-02 static-copy risk. IPC-09 script guards not executed end-to-end (need pinned python/JRE toolchains); logic is PATH-resolution + version assertion. | | 2026-07-09 | WRK-01 → `Done`. Verified on Windows host (windev) via an isolated `origin/main` worktree: worker builds x86 clean, `StaRuntimeTests`+`WorkerPipeSessionTests` 33/33 pass. Fixed an `xUnit1030` build error (the new worker test used `.ConfigureAwait(false)` in `[Fact]` bodies) that the macOS tree could not surface. Also ran GWC-01's Windows-only `WorkerClientTests` on windev: 18/18 pass (incl. `ReadEventsAsync_SecondEnumerator_Throws`). All 8 P0 findings now `Done`. Not yet committed. | From 74e815c4d72389e6ab244dc2019d078820ba4d5c Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 07:47:51 -0400 Subject: [PATCH 09/19] fix(archreview-p1): SEC-02/12/20 dashboard + observability hardening - SEC-02: DashboardAuthorizationHandler restricts the loopback and Authentication:Mode=Disabled bypasses to read-only. They now satisfy only a Viewer-bearing requirement (AnyDashboardRole), never AdminOnly, so anonymous localhost can view the dashboard but cannot reach API-key CRUD or session Close/Kill at the policy layer (previously guarded only by service re-checks). - SEC-12: DashboardSessionAdminService emits canonical AuditEvents through IAuditWriter (actions dashboard-close-session / dashboard-kill-worker, category SessionAdmin) on Success/Failure/Denied, mirroring the API-key audit path so destructive session actions leave durable, queryable rows. - SEC-20: drop the unbounded session_id tag from the exported mxgateway.heartbeats.failed counter (per-session detail stays in the snapshot/log). Docs updated same-change: CLAUDE.md (read-only loopback + 5-min bearer), GatewayDashboardDesign.md (bypass scoping + session-admin audit), Metrics.md. Server build clean; 30/30 targeted + 295/295 Dashboard/Security/App/Metrics sweep. --- CLAUDE.md | 2 +- archreview/remediation/00-tracking.md | 7 +- docs/GatewayDashboardDesign.md | 24 +++- docs/Metrics.md | 2 +- .../DashboardAuthorizationHandler.cs | 20 +++- .../Dashboard/DashboardSessionAdminService.cs | 105 +++++++++++++++++- .../Metrics/GatewayMetrics.cs | 6 +- .../DashboardAuthorizationHandlerTests.cs | 53 ++++++++- .../DashboardSessionAdminServiceTests.cs | 98 +++++++++++++++- .../Metrics/GatewayMetricsTests.cs | 50 +++++++++ 10 files changed, 347 insertions(+), 20 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 56fd95e..3f617f7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -123,7 +123,7 @@ Gateway gRPC clients authenticate with an API key in metadata: `authorization: B Session event streaming is **owner-scoped**: the API key that opened a session is recorded on the session, and every `StreamEvents` attach/reattach is rejected with `PermissionDenied` unless the caller's key id matches the owner. Possessing the `event` scope and knowing a session id is not sufficient — this closes the reconnect/fan-out trust boundary (detach-grace and replay retention are on by default) so an `event`-scoped key cannot attach to another key's retained session. -Dashboard auth is LDAP-backed (separate from the gRPC API-key model). `/login` binds against `MxGateway:Ldap` and maps the user's LDAP groups to `Admin` or `Viewer` via `MxGateway:Dashboard:GroupToRole`, then issues an HTTP-only secure `__Host-MxGatewayDashboard` cookie. SignalR hubs at `/hubs/{snapshot,alarms,events}` accept either the cookie or a 30-minute bearer minted at `/hubs/token`. `Dashboard:AllowAnonymousLocalhost` bypasses auth on loopback when enabled. `Dashboard:DisableLogin` (default `false`) auto-authenticates every dashboard request — including remote browsers — as `Dashboard:AutoLoginUser` (default `multi-role`) with both Admin and Viewer roles; dev/test only, never enable in production. +Dashboard auth is LDAP-backed (separate from the gRPC API-key model). `/login` binds against `MxGateway:Ldap` and maps the user's LDAP groups to `Admin` or `Viewer` via `MxGateway:Dashboard:GroupToRole`, then issues an HTTP-only secure `__Host-MxGatewayDashboard` cookie. SignalR hubs at `/hubs/{snapshot,alarms,events}` accept either the cookie or a 5-minute bearer minted at `/hubs/token`. `Dashboard:AllowAnonymousLocalhost` (default `true`) grants **read-only** loopback access — it satisfies the Viewer requirement but never the Admin-only requirement, so anonymous localhost can view the dashboard but not reach API-key CRUD or session Close/Kill (`Authentication:Mode=Disabled` is scoped the same way). `Dashboard:DisableLogin` (default `false`) auto-authenticates every dashboard request — including remote browsers — as `Dashboard:AutoLoginUser` (default `multi-role`) with both Admin and Viewer roles; dev/test only, never enable in production. ## Process / Platform Notes diff --git a/archreview/remediation/00-tracking.md b/archreview/remediation/00-tracking.md index 0d2a49d..004223d 100644 --- a/archreview/remediation/00-tracking.md +++ b/archreview/remediation/00-tracking.md @@ -140,7 +140,7 @@ Full design + implementation for each row lives in the linked domain doc under i | ID | Sev | Tier | Eff | Dep | Status | Title | |---|---|:-:|:-:|---|---|---| | SEC-01 | Medium | P1 | M | — | Done | Windows-absolute default paths become relative files off-Windows | -| SEC-02 | Medium | P1 | S | — | Not started | `AllowAnonymousLocalhost` satisfies AdminOnly, not just Viewer | +| SEC-02 | Medium | P1 | S | — | Done | `AllowAnonymousLocalhost` satisfies AdminOnly, not just Viewer | | SEC-03 | Medium | P2 | S | — | Not started | Dashboard cookie lost its `__Host-` prefix; four docs still promise it | | SEC-04 | Medium | P1 | S | SEC-03 | Done | `DisableLogin` has no production guard | | SEC-05 | Medium | P1 | M | — | Done | Hub bearer tokens irrevocable for 30 min, carried in query string | @@ -150,7 +150,7 @@ Full design + implementation for each row lives in the linked domain doc under i | SEC-09 | Medium | P2 | S | — | Not started | Dashboard design-doc `GroupToRole` sample now fails startup validation | | SEC-10 | Medium | P1 | L | — | Done | No API-key expiry | | SEC-11 | Medium | P1 | M | SEC-08 | Done | No rate limiting or lockout on either auth surface | -| SEC-12 | Medium | P1 | M | — | Not started | Dashboard Close/Kill bypass the canonical audit store | +| SEC-12 | Medium | P1 | M | — | Done | Dashboard Close/Kill bypass the canonical audit store | | SEC-13 | Low | — | S | SEC-30 | Not started | Redactor's credential-command list omits secured-bulk variants | | SEC-14 | Low | — | S | SEC-02,20 | Not started | `/metrics` + `/health` unauthenticated; a metric leaks session ids | | SEC-15 | Low | — | S | — | Not started | GET `/logout` skips antiforgery | @@ -158,7 +158,7 @@ Full design + implementation for each row lives in the linked domain doc under i | SEC-17 | Low | — | S | — | Not started | Interceptor does not override client-streaming/duplex handlers | | SEC-18 | Low | — | S | — | Not started | Pepper-unavailable detection matches library message text | | SEC-19 | Low | — | S | — | Not started | Canonical audit store re-issues `CREATE TABLE` per write/read | -| SEC-20 | Low | P1 | S | — | Not started | `session_id` metric tag is unbounded cardinality | +| SEC-20 | Low | P1 | S | — | Done | `session_id` metric tag is unbounded cardinality | | SEC-21 | Low | — | M | — | Not started | Snapshot publisher works every second regardless of audience | | SEC-22 | Low | P2 | M | — | Not started | `docs/Authentication.md` documents types no longer in this repo | | SEC-23 | Low | — | S | SEC-03,04 | Not started | Validator ignores `DisableLogin`/`AutoLoginUser`/`RequireHttpsCookie`/`CookieName` | @@ -255,6 +255,7 @@ Findings the review flagged as one coordinated design pass — sequence them tog |---|---| | 2026-07-09 | Initial tracking doc generated from the six domain remediation designs. All 153 findings `Not started`. | | 2026-07-09 | P0 tier executed via parallel agents. GWC-01/02/03, TST-02, TST-12, CLI-01, CLI-03 → `Done` (NonWindows build clean; gateway 135/135 targeted tests pass, Go `go test ./...` + `-race` pass, Rust `test`/`clippy -D warnings` pass). | +| 2026-07-09 | P1 S-misc (dashboard/observability hardening). SEC-02/12/20 → `Done`. SEC-02: `DashboardAuthorizationHandler` restricts the loopback + `Authentication:Mode=Disabled` bypasses to read-only (they satisfy a Viewer-bearing requirement but never `AdminOnly`), closing the policy-layer gap where anonymous localhost was authorized for Admin surfaces. SEC-12: `DashboardSessionAdminService` now emits canonical `AuditEvent`s (`dashboard-close-session`/`dashboard-kill-worker`, category `SessionAdmin`) through `IAuditWriter` on Success/Failure/Denied, so Close/Kill land durable audit rows. SEC-20: dropped the unbounded `session_id` tag from the exported `mxgateway.heartbeats.failed` counter. Docs updated same-change (CLAUDE.md, GatewayDashboardDesign.md, Metrics.md). Server build clean (0 warnings); targeted classes 30/30 pass; broader Dashboard+Security+GatewayApplication+Metrics sweep 295/295 pass. | | 2026-07-09 | P1 Wave 2b (security authz+hub). SEC-05/07/08/11 → `Done` (hub-token lifetime; QueryActiveAlarms scope arm; gateway-side verification cache + last-used coalescing; login rate limit + per-peer gRPC failure limiter). Full-suite checkpoint caught + fixed regressions the earlier narrow SEC-01/04/06 filter missed: cross-platform path-rooting, an `IHostEnvironment` fallback for minimal DI containers, a test-assembly `ASPNETCORE_ENVIRONMENT=Development` default, and a platform-correct default-path assertion. Suite: 747 passed / 42 failed, all 42 pre-existing macOS named-pipe-harness env failures. | | 2026-07-09 | P1 Wave 2a (security). SEC-01/04/06 → `Done` (config path-rooting + production validator guards; Server build clean, validator+hygiene tests 53/53). SEC-10 → `Done`: the shared `ZB.MOM.WW.Auth.ApiKeys` gained optional `ExpiresUtc` (expired keys rejected, auth DB auto-migrates to schema v3) via a concurrent HistorianGateway-remediation session's "G-2"; this repo consumes it by bumping the four `Auth.*` refs 0.1.2→0.1.4 (commit 197731a). Remaining SEC-10 polish (`apikey create --expires` + dashboard staleness badge) tracked as a small follow-up. | | 2026-07-09 | P1 Wave 1 (CI + codegen freshness + Rust buildability) via parallel agents. IPC-01/09/19/20, CLI-02 → `Done`; TST-03 → `In review` (CI pipeline authored + YAML/layout-validated, but not yet executed on a Gitea runner — proven on first push). Verified on macOS: NonWindows build clean, `ClientProtoInputTests` 5/5, `publish-client-proto-inputs.ps1 -Check` exit 0, `cargo package` (no `--no-verify`) compiles standalone. Added a vendored-Rust-proto drift guard (Check 3) to `check-codegen.ps1` closing the CLI-02 static-copy risk. IPC-09 script guards not executed end-to-end (need pinned python/JRE toolchains); logic is PATH-resolution + version assertion. | diff --git a/docs/GatewayDashboardDesign.md b/docs/GatewayDashboardDesign.md index 9778a74..412f6d2 100644 --- a/docs/GatewayDashboardDesign.md +++ b/docs/GatewayDashboardDesign.md @@ -257,6 +257,15 @@ gated by a confirmation dialog before it reaches `ISessionManager`. is killed immediately with no graceful-shutdown attempt. The session is removed from the registry and the open-session slot is released either way. +Both actions write a canonical `AuditEvent` through `IAuditWriter` into the +`audit_event` store (category `SessionAdmin`, actions `dashboard-close-session` +and `dashboard-kill-worker`) in addition to the operational `ILogger` line — so a +worker killed mid-production leaves a durable, queryable row rather than only a +rotatable log entry. The event records the LDAP actor, the session id (`Target`), +the remote address, and an outcome of `Success`, `Failure`, or `Denied` +(an unauthorized attempt is still audited as `Denied`). This mirrors the API-key +management audit path. + ### Workers page Show: @@ -436,11 +445,16 @@ Three authorization policies are registered: cookie OR a `MxGateway.Dashboard.HubToken` bearer (used by WebSocket upgrades where the cookie can't be forwarded). -Two environmental bypasses still apply: `MxGateway:Authentication:Mode = Disabled` -authorizes every request, and `MxGateway:Dashboard:AllowAnonymousLocalhost` -(default `true`) authorizes any loopback request without a role check. Remote -requests always require an authenticated principal carrying at least the -Viewer role. +Two environmental bypasses still apply, both scoped to **read-only** access: +`MxGateway:Authentication:Mode = Disabled` and `MxGateway:Dashboard:AllowAnonymousLocalhost` +(default `true`, loopback only) each satisfy a requirement that includes the Viewer +role (`AnyDashboardRole`) but **never** the `AdminOnly` requirement. Destructive/admin +surfaces (API-key CRUD, session Close, worker Kill) still demand a real `Administrator` +role claim, so the "anonymous localhost is read-only" contract holds at the policy layer +rather than only in downstream service re-checks. Remote requests (outside the `Disabled` +bypass) always require an authenticated principal carrying at least the Viewer role. The +loopback test trusts `Connection.RemoteIpAddress`; if forwarded-headers middleware is ever +added upstream, `DashboardAuthorizationHandler.IsLoopbackRequest()` must be revisited. ### DisableLogin dev bypass diff --git a/docs/Metrics.md b/docs/Metrics.md index 64836a4..6b40611 100644 --- a/docs/Metrics.md +++ b/docs/Metrics.md @@ -44,7 +44,7 @@ All counters are `Counter`. Tag values come from the call sites listed und | `mxgateway.faults` | `category` | Faults reported by session, event, or worker code paths. The category is a `SessionManagerErrorCode` or `WorkerClientErrorCode` name. | | `mxgateway.workers.killed` | `reason` | Forced terminations of worker processes. | | `mxgateway.workers.exited` | `reason` | Clean or fault-driven worker exits. | -| `mxgateway.heartbeats.failed` | `session_id` | Worker heartbeat misses tracked per session. | +| `mxgateway.heartbeats.failed` | *(none)* | Aggregate worker heartbeat misses. The exported counter carries **no** `session_id` tag — per-session attribution is unbounded cardinality (every session mints a new time series), so it is kept out of the exporter; use the dashboard snapshot or the log scope for per-session detail. | | `mxgateway.grpc.streams.disconnected` | `reason` | Detachments of the dashboard or client gRPC event stream. | | `mxgateway.retries.attempted` | `area` | Resilience retries executed by gateway components. | diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardAuthorizationHandler.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardAuthorizationHandler.cs index 8e157bd..3692867 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardAuthorizationHandler.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardAuthorizationHandler.cs @@ -11,6 +11,18 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard; /// (b) authentication is fully disabled, or (c) the request is from loopback /// and MxGateway:Dashboard:AllowAnonymousLocalhost is on. /// +/// +/// The environment bypasses in (b) and (c) grant read-only access only: they +/// satisfy a requirement that includes (i.e. +/// ) but never the +/// requirement. Destructive/admin +/// surfaces still require a real Admin role claim, preserving the documented +/// "anonymous localhost is read-only" contract at the policy layer rather than by accident +/// in downstream service re-checks. The loopback test trusts +/// Connection.RemoteIpAddress; if forwarded-headers middleware is ever added upstream, +/// must be revisited so a spoofed X-Forwarded-For +/// cannot masquerade as loopback. +/// public sealed class DashboardAuthorizationHandler( IHttpContextAccessor httpContextAccessor, IOptions options) : AuthorizationHandler @@ -22,14 +34,18 @@ public sealed class DashboardAuthorizationHandler( { GatewayOptions gatewayOptions = options.Value; - if (gatewayOptions.Authentication.Mode == AuthenticationMode.Disabled) + // Environment bypasses are read-only: they satisfy the Viewer requirement + // (AnyDashboardRole) but never AdminOnly, which lacks Viewer in RequiredRoles. + bool grantsReadOnly = requirement.RequiredRoles.Contains(DashboardRoles.Viewer); + + if (grantsReadOnly && gatewayOptions.Authentication.Mode == AuthenticationMode.Disabled) { context.Succeed(requirement); return Task.CompletedTask; } - if (gatewayOptions.Dashboard.AllowAnonymousLocalhost && IsLoopbackRequest()) + if (grantsReadOnly && gatewayOptions.Dashboard.AllowAnonymousLocalhost && IsLoopbackRequest()) { context.Succeed(requirement); diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSessionAdminService.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSessionAdminService.cs index b19b1a1..595530b 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSessionAdminService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSessionAdminService.cs @@ -1,6 +1,8 @@ using System.Security.Claims; +using System.Text.Json; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using ZB.MOM.WW.Audit; using ZB.MOM.WW.MxGateway.Server.Sessions; namespace ZB.MOM.WW.MxGateway.Server.Dashboard; @@ -8,21 +10,35 @@ namespace ZB.MOM.WW.MxGateway.Server.Dashboard; /// /// Default implementation of : gates /// destructive session actions on the role, -/// audit-logs successful operations, and converts -/// (and any other unexpected exceptions) into -/// so the Blazor pages never see a raw exception. +/// records each attempt as a canonical through +/// (in addition to the operational line), and converts +/// (and any other unexpected exceptions) into +/// so the Blazor pages never see a raw exception. /// /// /// The constant dashboard-admin-kill is the reason passed to /// and forwarded as the /// reason tag on the mxgateway.workers.killed counter and in -/// the worker-kill audit log entries. +/// the worker-kill audit log entries. Destructive dashboard actions write durable, +/// queryable audit rows (actions dashboard-close-session / dashboard-kill-worker) +/// to the canonical audit_event store, mirroring the API-key management path so a +/// worker killed mid-production is not visible only in a rotatable line. /// public sealed class DashboardSessionAdminService( ISessionManager sessionManager, IHttpContextAccessor httpContextAccessor, + IAuditWriter auditWriter, ILogger? logger = null) : IDashboardSessionAdminService { + /// Canonical for dashboard session-admin actions. + internal const string SessionAdminCategory = "SessionAdmin"; + + /// Canonical for a dashboard session close. + internal const string CloseSessionAction = "dashboard-close-session"; + + /// Canonical for a dashboard worker kill. + internal const string KillWorkerAction = "dashboard-kill-worker"; + private const string UnauthorizedMessage = "Sign in as an Admin to close sessions or kill workers."; private const string KillReason = "dashboard-admin-kill"; @@ -46,6 +62,8 @@ public sealed class DashboardSessionAdminService( { if (!CanManage(user)) { + await WriteAuditAsync(user, CloseSessionAction, sessionId, AuditOutcome.Denied, "unauthorized", cancellationToken) + .ConfigureAwait(false); return DashboardSessionAdminResult.Fail(UnauthorizedMessage); } @@ -68,6 +86,15 @@ public sealed class DashboardSessionAdminService( ResolveRemoteAddress(), result.AlreadyClosed); + await WriteAuditAsync( + user, + CloseSessionAction, + sessionId, + AuditOutcome.Success, + result.AlreadyClosed ? "alreadyClosed" : "closed", + cancellationToken) + .ConfigureAwait(false); + return DashboardSessionAdminResult.Success( result.AlreadyClosed ? $"Session {sessionId} was already closed." @@ -75,6 +102,8 @@ public sealed class DashboardSessionAdminService( } catch (SessionManagerException exception) when (exception.ErrorCode == SessionManagerErrorCode.SessionNotFound) { + await WriteAuditAsync(user, CloseSessionAction, sessionId, AuditOutcome.Failure, "not-found", cancellationToken) + .ConfigureAwait(false); return DashboardSessionAdminResult.Fail($"Session {sessionId} was not found."); } catch (SessionManagerException exception) @@ -84,6 +113,8 @@ public sealed class DashboardSessionAdminService( "Dashboard admin {Actor} close failed for session {SessionId}.", actor, sessionId); + await WriteAuditAsync(user, CloseSessionAction, sessionId, AuditOutcome.Failure, exception.Message, cancellationToken) + .ConfigureAwait(false); return DashboardSessionAdminResult.Fail( $"Close failed: {exception.Message}"); } @@ -98,6 +129,8 @@ public sealed class DashboardSessionAdminService( "Dashboard admin {Actor} close failed unexpectedly for session {SessionId}.", actor, sessionId); + await WriteAuditAsync(user, CloseSessionAction, sessionId, AuditOutcome.Failure, "unexpected", cancellationToken) + .ConfigureAwait(false); return DashboardSessionAdminResult.Fail( $"Close failed unexpectedly for session {sessionId}. See the gateway log for details."); } @@ -111,6 +144,8 @@ public sealed class DashboardSessionAdminService( { if (!CanManage(user)) { + await WriteAuditAsync(user, KillWorkerAction, sessionId, AuditOutcome.Denied, "unauthorized", cancellationToken) + .ConfigureAwait(false); return DashboardSessionAdminResult.Fail(UnauthorizedMessage); } @@ -133,6 +168,15 @@ public sealed class DashboardSessionAdminService( ResolveRemoteAddress(), result.AlreadyClosed); + await WriteAuditAsync( + user, + KillWorkerAction, + sessionId, + AuditOutcome.Success, + result.AlreadyClosed ? "alreadyClosed" : "killed", + cancellationToken) + .ConfigureAwait(false); + return DashboardSessionAdminResult.Success( result.AlreadyClosed ? $"Session {sessionId} was already closed." @@ -140,6 +184,8 @@ public sealed class DashboardSessionAdminService( } catch (SessionManagerException exception) when (exception.ErrorCode == SessionManagerErrorCode.SessionNotFound) { + await WriteAuditAsync(user, KillWorkerAction, sessionId, AuditOutcome.Failure, "not-found", cancellationToken) + .ConfigureAwait(false); return DashboardSessionAdminResult.Fail($"Session {sessionId} was not found."); } catch (SessionManagerException exception) @@ -149,6 +195,8 @@ public sealed class DashboardSessionAdminService( "Dashboard admin {Actor} kill failed for session {SessionId}.", actor, sessionId); + await WriteAuditAsync(user, KillWorkerAction, sessionId, AuditOutcome.Failure, exception.Message, cancellationToken) + .ConfigureAwait(false); return DashboardSessionAdminResult.Fail( $"Kill failed: {exception.Message}"); } @@ -163,6 +211,8 @@ public sealed class DashboardSessionAdminService( "Dashboard admin {Actor} kill failed unexpectedly for session {SessionId}.", actor, sessionId); + await WriteAuditAsync(user, KillWorkerAction, sessionId, AuditOutcome.Failure, "unexpected", cancellationToken) + .ConfigureAwait(false); return DashboardSessionAdminResult.Fail( $"Kill failed unexpectedly for session {sessionId}. See the gateway log for details."); } @@ -177,4 +227,51 @@ public sealed class DashboardSessionAdminService( { return httpContextAccessor.HttpContext?.Connection.RemoteIpAddress?.ToString(); } + + /// + /// Writes a canonical for a dashboard session-admin action through the + /// best-effort (failures are swallowed/logged by the writer, so this + /// never throws and never masks the operation result). is the LDAP + /// operator, the session id, and is wrapped + /// as the detail field of the JSON extension bag. + /// + private async Task WriteAuditAsync( + ClaimsPrincipal user, + string action, + string sessionId, + AuditOutcome outcome, + string? detail, + CancellationToken cancellationToken) + { + AuditEvent auditEvent = new() + { + EventId = Guid.NewGuid(), + OccurredAtUtc = DateTimeOffset.UtcNow, + Actor = ResolveActor(user), + Action = action, + Outcome = outcome, + Category = SessionAdminCategory, + Target = sessionId, + SourceNode = ResolveRemoteAddress(), + CorrelationId = ParseCorrelationId(), + DetailsJson = WrapDetail(detail), + }; + + await auditWriter.WriteAsync(auditEvent, cancellationToken).ConfigureAwait(false); + } + + /// + /// Derives a correlation id from the request trace identifier when it is a well-formed GUID; + /// otherwise null (the default HttpContext.TraceIdentifier is the connection:request form, + /// not a GUID, so it correlates to null rather than fabricating one). + /// + private Guid? ParseCorrelationId() => + Guid.TryParse(httpContextAccessor.HttpContext?.TraceIdentifier, out Guid correlationId) + ? correlationId + : null; + + private static string? WrapDetail(string? detail) => + detail is null + ? null + : JsonSerializer.Serialize(new Dictionary { ["detail"] = detail }); } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetrics.cs b/src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetrics.cs index 2164661..c42468f 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetrics.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Metrics/GatewayMetrics.cs @@ -351,7 +351,11 @@ public sealed class GatewayMetrics : IDisposable _heartbeatFailures++; } - _heartbeatFailuresCounter.Add(1, new KeyValuePair("session_id", sessionId)); + // Exported without a session_id tag: per-session attribution is unbounded cardinality + // (every session ever opened mints a new exporter time series, bloating Prometheus/OTLP + // storage on long-running gateways). Per-session detail stays available via the dashboard + // snapshot and the log scope; the exported counter tracks only the aggregate. + _heartbeatFailuresCounter.Add(1); } /// diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardAuthorizationHandlerTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardAuthorizationHandlerTests.cs index 9d4ab2d..7423c2e 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardAuthorizationHandlerTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardAuthorizationHandlerTests.cs @@ -128,11 +128,58 @@ public sealed class DashboardAuthorizationHandlerTests Assert.False(context.HasSucceeded); } + /// + /// Verifies that the anonymous-localhost bypass is read-only: a loopback request with + /// AllowAnonymousLocalhost on is denied the + /// requirement, so destructive/admin surfaces are never reachable without a real Admin role. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task HandleAsync_AnonymousLocalhostAllowed_DoesNotSatisfyAdminPolicy() + { + AuthorizationHandlerContext context = await AuthorizeAsync( + new ClaimsPrincipal(new ClaimsIdentity()), + IPAddress.Loopback, + allowAnonymousLocalhost: true, + DashboardAuthorizationRequirement.AdminOnly); + + Assert.False(context.HasSucceeded); + } + + /// + /// Verifies that with authentication fully disabled the environment bypass stays read-only: + /// a remote request satisfies + /// (Viewer) but is denied . + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task HandleAsync_AuthenticationDisabled_GrantsViewerButNotAdmin() + { + AuthorizationHandlerContext viewerContext = await AuthorizeAsync( + new ClaimsPrincipal(new ClaimsIdentity()), + IPAddress.Parse("10.0.0.5"), + allowAnonymousLocalhost: false, + DashboardAuthorizationRequirement.AnyDashboardRole, + authenticationMode: AuthenticationMode.Disabled); + + Assert.True(viewerContext.HasSucceeded); + + AuthorizationHandlerContext adminContext = await AuthorizeAsync( + new ClaimsPrincipal(new ClaimsIdentity()), + IPAddress.Parse("10.0.0.5"), + allowAnonymousLocalhost: false, + DashboardAuthorizationRequirement.AdminOnly, + authenticationMode: AuthenticationMode.Disabled); + + Assert.False(adminContext.HasSucceeded); + } + private static async Task AuthorizeAsync( ClaimsPrincipal principal, IPAddress remoteAddress, bool allowAnonymousLocalhost, - DashboardAuthorizationRequirement requirement) + DashboardAuthorizationRequirement requirement, + AuthenticationMode authenticationMode = AuthenticationMode.ApiKey) { DefaultHttpContext httpContext = new(); httpContext.Connection.RemoteIpAddress = remoteAddress; @@ -140,6 +187,10 @@ public sealed class DashboardAuthorizationHandlerTests new HttpContextAccessor { HttpContext = httpContext }, Options.Create(new GatewayOptions { + Authentication = new AuthenticationOptions + { + Mode = authenticationMode, + }, Dashboard = new DashboardOptions { AllowAnonymousLocalhost = allowAnonymousLocalhost, diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSessionAdminServiceTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSessionAdminServiceTests.cs index 08d7657..f254ba1 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSessionAdminServiceTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSessionAdminServiceTests.cs @@ -1,6 +1,8 @@ +using System.Collections.Concurrent; using System.Diagnostics.CodeAnalysis; using System.Security.Claims; using Microsoft.AspNetCore.Http; +using ZB.MOM.WW.Audit; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Server.Dashboard; using ZB.MOM.WW.MxGateway.Server.Sessions; @@ -198,14 +200,91 @@ public sealed class DashboardSessionAdminServiceTests Assert.False(string.IsNullOrWhiteSpace(result.Message)); } - private static DashboardSessionAdminService CreateService(ISessionManager sessionManager) + /// + /// Verifies that a successful close writes a canonical dashboard-close-session + /// — actor, session-id target, and Success outcome — to the + /// audit store, not only the operational log (SEC-12). + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task CloseSessionAsync_AdminClose_WritesCanonicalAuditEvent() + { + FakeSessionManager sessionManager = new(); + RecordingAuditWriter auditWriter = new(); + DashboardSessionAdminService service = CreateService(sessionManager, auditWriter); + + DashboardSessionAdminResult result = await service.CloseSessionAsync( + CreateUser(DashboardRoles.Admin), + "session-1", + CancellationToken.None); + + Assert.True(result.Succeeded); + AuditEvent audit = Assert.Single(auditWriter.Events); + Assert.Equal(DashboardSessionAdminService.CloseSessionAction, audit.Action); + Assert.Equal(DashboardSessionAdminService.SessionAdminCategory, audit.Category); + Assert.Equal("session-1", audit.Target); + Assert.Equal("tester", audit.Actor); + Assert.Equal(AuditOutcome.Success, audit.Outcome); + } + + /// + /// Verifies that a successful kill writes a canonical dashboard-kill-worker + /// to the audit store (SEC-12). + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task KillWorkerAsync_AdminKill_WritesCanonicalAuditEvent() + { + FakeSessionManager sessionManager = new(); + RecordingAuditWriter auditWriter = new(); + DashboardSessionAdminService service = CreateService(sessionManager, auditWriter); + + DashboardSessionAdminResult result = await service.KillWorkerAsync( + CreateUser(DashboardRoles.Admin), + "session-1", + CancellationToken.None); + + Assert.True(result.Succeeded); + AuditEvent audit = Assert.Single(auditWriter.Events); + Assert.Equal(DashboardSessionAdminService.KillWorkerAction, audit.Action); + Assert.Equal("session-1", audit.Target); + Assert.Equal(AuditOutcome.Success, audit.Outcome); + } + + /// + /// Verifies that an unauthorized (viewer) close attempt still writes a Denied + /// audit row, so rejected destructive attempts are durably recorded (SEC-12). + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task CloseSessionAsync_ViewerDenied_WritesDeniedAuditEvent() + { + FakeSessionManager sessionManager = new(); + RecordingAuditWriter auditWriter = new(); + DashboardSessionAdminService service = CreateService(sessionManager, auditWriter); + + DashboardSessionAdminResult result = await service.CloseSessionAsync( + CreateUser(DashboardRoles.Viewer), + "session-1", + CancellationToken.None); + + Assert.False(result.Succeeded); + AuditEvent audit = Assert.Single(auditWriter.Events); + Assert.Equal(DashboardSessionAdminService.CloseSessionAction, audit.Action); + Assert.Equal(AuditOutcome.Denied, audit.Outcome); + } + + private static DashboardSessionAdminService CreateService( + ISessionManager sessionManager, + IAuditWriter? auditWriter = null) { DefaultHttpContext httpContext = new(); httpContext.Connection.RemoteIpAddress = System.Net.IPAddress.Loopback; return new DashboardSessionAdminService( sessionManager, - new HttpContextAccessor { HttpContext = httpContext }); + new HttpContextAccessor { HttpContext = httpContext }, + auditWriter ?? new RecordingAuditWriter()); } private static ClaimsPrincipal CreateUser(string role) @@ -219,6 +298,21 @@ public sealed class DashboardSessionAdminServiceTests return new ClaimsPrincipal(identity); } + private sealed class RecordingAuditWriter : IAuditWriter + { + private readonly ConcurrentQueue _events = new(); + + /// Gets the audit events written through this writer, in order. + public IReadOnlyList Events => _events.ToArray(); + + /// + public Task WriteAsync(AuditEvent evt, CancellationToken ct = default) + { + _events.Enqueue(evt); + return Task.CompletedTask; + } + } + private sealed class FakeSessionManager : ISessionManager { /// Gets the number of times CloseSessionAsync was invoked. diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Metrics/GatewayMetricsTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Metrics/GatewayMetricsTests.cs index 0181f5e..2dec8bd 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Metrics/GatewayMetricsTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Metrics/GatewayMetricsTests.cs @@ -157,6 +157,56 @@ public sealed class GatewayMetricsTests Assert.Equal(2, capturedMode); } + /// + /// Verifies that increments + /// mxgateway.heartbeats.failed without emitting a session_id tag: the tag is + /// unbounded cardinality (every session mints a new exporter time series), so per-session + /// attribution is deliberately kept out of the exported counter (SEC-20). + /// + [Fact] + public void HeartbeatFailed_IncrementsCounterWithoutSessionIdTag() + { + using GatewayMetrics metrics = new(); + using MeterListener listener = new(); + + long capturedValue = 0; + bool sawSessionIdTag = false; + + listener.InstrumentPublished = (instrument, meterListener) => + { + if (ReferenceEquals(instrument.Meter, metrics.Meter) + && instrument.Name == "mxgateway.heartbeats.failed") + { + meterListener.EnableMeasurementEvents(instrument); + } + }; + listener.SetMeasurementEventCallback( + (instrument, measurement, tags, _) => + { + if (!ReferenceEquals(instrument.Meter, metrics.Meter) + || instrument.Name != "mxgateway.heartbeats.failed") + { + return; + } + + capturedValue += measurement; + foreach (KeyValuePair tag in tags) + { + if (tag.Key == "session_id") + { + sawSessionIdTag = true; + } + } + }); + listener.Start(); + + metrics.HeartbeatFailed("session-1"); + + Assert.Equal(1, capturedValue); + Assert.False(sawSessionIdTag); + Assert.Equal(1, metrics.GetSnapshot().HeartbeatFailures); + } + /// Verifies that removing session events only affects that session. [Fact] public void RemoveSessionEvents_RemovesOnlyThatSession() From 8c06635ee58755c2d5b3d3b52e683b6097fe43b3 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 08:02:33 -0400 Subject: [PATCH 10/19] docs(archreview-p1): TST-08 orphaned-testhost leak does not reproduce Empirically verified the full ZB.MOM.WW.MxGateway.Tests suite exits cleanly: - macOS: 0 surviving testhost after a full run. - windev (origin/main worktree): 0 new testhost and 0 new worker processes after a full run, isolated by process StartTime. Static audit confirms the prime-suspect fixtures already dispose deterministically (WorkerClient cancels _stopCts + awaits its read/write/heartbeat tasks with a 5s timeout; GatewaySession disposes the client; harness/E2E fixtures use await using). Drop the stale 'leaves orphaned testhost processes' rationale from CLAUDE.md's Source Update Workflow; keep filtered-run guidance as a speed guideline only. Separately discovered (tracked in 00-tracking.md change log, NOT part of TST-08): Windows-only temp-file-lock flakiness in ~4 full-host test classes (Sqlite connection-pool retaining a temp .db handle; SelfSignedCertificateProvider's fixed-name gw.pfx.tmp racing teardown) that macOS never surfaces. --- CLAUDE.md | 2 +- archreview/remediation/00-tracking.md | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3f617f7..a95b328 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -88,7 +88,7 @@ powershell -ExecutionPolicy Bypass -File scripts/run-client-e2e-tests.ps1 When source code changes, build and test the affected component before reporting work done. If the change crosses component boundaries, build each affected component — don't rely on a single top-level build: -**Run targeted tests per task, never the full suite each time.** When executing a plan task-by-task, run only the tests that exercise the code that task touched (`dotnet test --filter "FullyQualifiedName~"`, or the per-task test named in the plan). The full gateway suite is slow and leaves orphaned testhost processes — run it at most once per phase (after a related batch of tasks lands), not after every task. +**Run targeted tests per task, never the full suite each time.** When executing a plan task-by-task, run only the tests that exercise the code that task touched (`dotnet test --filter "FullyQualifiedName~"`, or the per-task test named in the plan). The full gateway suite is slow — run it at most once per phase (after a related batch of tasks lands), not after every task. This is a speed guideline, not a correctness one: the suite exits cleanly (verified on macOS and the Windows dev box — 0 surviving `testhost`/worker processes after a full run), so filtered runs are about turnaround, not about avoiding a process leak. | Changed area | Required verification | |---|---| diff --git a/archreview/remediation/00-tracking.md b/archreview/remediation/00-tracking.md index 004223d..e0d8a76 100644 --- a/archreview/remediation/00-tracking.md +++ b/archreview/remediation/00-tracking.md @@ -220,7 +220,7 @@ Full design + implementation for each row lives in the linked domain doc under i | TST-05 | Medium | P1 | S | TST-03 | Not started | Real-worker control/COM paths verified opt-in only | | TST-06 | Medium | — | M | — | Not started | Dashboard live-data path untested | | TST-07 | Medium | — | S | — | Not started | Real-clock sleeps with negative assertions are latent flakes | -| TST-08 | Medium | P1 | M | — | Not started | Full-suite orphaned testhost processes | +| TST-08 | Medium | P1 | M | — | Done | Full-suite orphaned testhost processes (does not reproduce; doc de-stale) | | TST-09 | Medium | — | M | — | Not started | Health checks cover only the auth store | | TST-10 | Medium | — | M | — | Not started | Deployment/upgrade undocumented and hand-rolled | | TST-11 | Medium | P2 | M | — | Not started | No version discipline on server; client versions drift | @@ -255,6 +255,7 @@ Findings the review flagged as one coordinated design pass — sequence them tog |---|---| | 2026-07-09 | Initial tracking doc generated from the six domain remediation designs. All 153 findings `Not started`. | | 2026-07-09 | P0 tier executed via parallel agents. GWC-01/02/03, TST-02, TST-12, CLI-01, CLI-03 → `Done` (NonWindows build clean; gateway 135/135 targeted tests pass, Go `go test ./...` + `-race` pass, Rust `test`/`clippy -D warnings` pass). | +| 2026-07-09 | P1 TST-08 (orphaned testhost) → `Done` by evidence: the claimed leak does **not** reproduce. Full `ZB.MOM.WW.MxGateway.Tests` suite exits cleanly on macOS (0 survivors) and on the Windows dev box (windev, `origin/main` worktree: 0 new `testhost` and 0 new `ZB.MOM.WW.MxGateway.Worker` processes after a full run, isolated by StartTime). Static audit confirms the prime-suspect fixtures already dispose deterministically (WorkerClient cancels its `_stopCts` + awaits read/write/heartbeat tasks with a 5 s timeout; GatewaySession disposes the client; E2E/harness fixtures use `await using`). CLAUDE.md "Source Update Workflow" updated to drop the stale "leaves orphaned testhost processes" rationale (filtered runs kept as a speed guideline). **Discovered separately (NOT TST-08, still open):** Windows-only temp-file-lock flakiness in ~4 full-host test classes — `AuthStoreHealthCheckTests` (Microsoft.Data.Sqlite connection-pool holds the temp `.db` handle after `await using`, so the finally `File.Delete` throws "used by another process"), `GatewayTlsBootstrapTests`/`DashboardCookieOptionsTests`/`DashboardHubsRegistrationTests` (a `gw.pfx.tmp` handle from `SelfSignedCertificateProvider`'s fixed-name `path+".tmp"` atomic-write racing teardown). Count varies run-to-run (2–4 fails); macOS never sees it (Unix allows deleting open files). Fix = `SqliteConnection.ClearAllPools()` before temp delete + a unique/guarded `.tmp` name; verify on windev. | | 2026-07-09 | P1 S-misc (dashboard/observability hardening). SEC-02/12/20 → `Done`. SEC-02: `DashboardAuthorizationHandler` restricts the loopback + `Authentication:Mode=Disabled` bypasses to read-only (they satisfy a Viewer-bearing requirement but never `AdminOnly`), closing the policy-layer gap where anonymous localhost was authorized for Admin surfaces. SEC-12: `DashboardSessionAdminService` now emits canonical `AuditEvent`s (`dashboard-close-session`/`dashboard-kill-worker`, category `SessionAdmin`) through `IAuditWriter` on Success/Failure/Denied, so Close/Kill land durable audit rows. SEC-20: dropped the unbounded `session_id` tag from the exported `mxgateway.heartbeats.failed` counter. Docs updated same-change (CLAUDE.md, GatewayDashboardDesign.md, Metrics.md). Server build clean (0 warnings); targeted classes 30/30 pass; broader Dashboard+Security+GatewayApplication+Metrics sweep 295/295 pass. | | 2026-07-09 | P1 Wave 2b (security authz+hub). SEC-05/07/08/11 → `Done` (hub-token lifetime; QueryActiveAlarms scope arm; gateway-side verification cache + last-used coalescing; login rate limit + per-peer gRPC failure limiter). Full-suite checkpoint caught + fixed regressions the earlier narrow SEC-01/04/06 filter missed: cross-platform path-rooting, an `IHostEnvironment` fallback for minimal DI containers, a test-assembly `ASPNETCORE_ENVIRONMENT=Development` default, and a platform-correct default-path assertion. Suite: 747 passed / 42 failed, all 42 pre-existing macOS named-pipe-harness env failures. | | 2026-07-09 | P1 Wave 2a (security). SEC-01/04/06 → `Done` (config path-rooting + production validator guards; Server build clean, validator+hygiene tests 53/53). SEC-10 → `Done`: the shared `ZB.MOM.WW.Auth.ApiKeys` gained optional `ExpiresUtc` (expired keys rejected, auth DB auto-migrates to schema v3) via a concurrent HistorianGateway-remediation session's "G-2"; this repo consumes it by bumping the four `Auth.*` refs 0.1.2→0.1.4 (commit 197731a). Remaining SEC-10 polish (`apikey create --expires` + dashboard staleness badge) tracked as a small follow-up. | From 11a716a07fc8a2adf3ae328559b4262d97f450db Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 08:14:55 -0400 Subject: [PATCH 11/19] test+fix: eliminate Windows full-suite temp-file-lock flakiness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Full-host tests passed in isolation but failed 2-4/run under parallel execution on Windows with 'the process cannot access the file ... because it is being used by another process' (macOS never sees it — Unix deletes open files). Two shared-state parallel collisions, discovered while verifying TST-08: 1. Self-signed cert generation. GatewayTlsBootstrapTests sets process-global Kestrel/TLS env vars that GatewayApplication.Build reads; a parallel host-building test inherits them mid-run and both generate a cert at the same path, racing on the fixed-name '.tmp'. Fixes: - SelfSignedCertificateProvider: stage the PFX in a unique '..tmp' instead of a fixed name, so concurrent/interrupted writers never collide on the temp file (real robustness: two instances or a restart-during-write no longer clash). The final atomic Move (last-writer-wins) still yields an equivalent cert. - TestHostEnvironmentInitializer: default MxGateway__Tls__SelfSignedCertPath to a per-process temp path so the suite never writes the shared ProgramData default or fights the deployed service; first host-building test generates, the rest load it. - GatewayTlsBootstrapTests: [Collection] with DisableParallelization so its global env-var mutation cannot bleed into parallel tests (new GlobalEnvironmentCollection). 2. AuthStoreHealthCheckTests opened a real SQLite file; Microsoft.Data.Sqlite's connection pool keeps the .db handle open after 'await using', so the finally File.Delete threw. Reuse the existing TempDatabaseDirectory helper, which calls SqliteConnection.ClearAllPools() before deleting. Server build clean; affected classes 17/17 on macOS. Windows full-suite verified separately. --- .../Tls/SelfSignedCertificateProvider.cs | 10 +++++++- .../Diagnostics/AuthStoreHealthCheckTests.cs | 17 ++++++------- .../Gateway/GatewayTlsBootstrapTests.cs | 4 ++++ .../GlobalEnvironmentCollection.cs | 18 ++++++++++++++ .../TestHostEnvironmentInitializer.cs | 24 ++++++++++++++++++- 5 files changed, 63 insertions(+), 10 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/TestSupport/GlobalEnvironmentCollection.cs diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Tls/SelfSignedCertificateProvider.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Tls/SelfSignedCertificateProvider.cs index 950fdfe..0d41576 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Tls/SelfSignedCertificateProvider.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Tls/SelfSignedCertificateProvider.cs @@ -163,7 +163,15 @@ public sealed class SelfSignedCertificateProvider // temp file empty, harden its permissions, and only then write the PFX into // the already-protected file. The temp path is in the same directory as the // target so the Move is atomic and preserves the hardened DACL/mode. - string temp = path + ".tmp"; + // + // The temp name carries a unique suffix rather than a fixed ".tmp": two + // processes (or two parallel callers) generating to the same target must not + // collide on one temp file. On Windows a fixed name makes the second writer's + // File.Create/Move fail with "the process cannot access the file ... because it + // is being used by another process"; a unique name lets each generation stage + // independently, and the final atomic Move (last-writer-wins) still yields a + // valid, equivalent certificate at the shared path. + string temp = $"{path}.{Guid.NewGuid():N}.tmp"; using (File.Create(temp)) { } HardenPermissions(temp); diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/AuthStoreHealthCheckTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/AuthStoreHealthCheckTests.cs index f2ed7ff..138a800 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/AuthStoreHealthCheckTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Diagnostics/AuthStoreHealthCheckTests.cs @@ -1,6 +1,7 @@ using Microsoft.Extensions.Diagnostics.HealthChecks; using ZB.MOM.WW.Auth.ApiKeys.Sqlite; using ZB.MOM.WW.MxGateway.Server.Diagnostics; +using ZB.MOM.WW.MxGateway.Tests.Security.Authentication; namespace ZB.MOM.WW.MxGateway.Tests.Diagnostics; @@ -17,14 +18,14 @@ public sealed class AuthStoreHealthCheckTests [Fact] public async Task Healthy_WhenStoreReachable() { - var path = Path.Combine(Path.GetTempPath(), $"authcheck-{Guid.NewGuid():N}.db"); - try - { - var check = new AuthStoreHealthCheck(FactoryFor(path)); - var result = await check.CheckHealthAsync(new HealthCheckContext()); - Assert.Equal(HealthStatus.Healthy, result.Status); - } - finally { if (File.Exists(path)) File.Delete(path); } + // Open a real SQLite file via the health check. TempDatabaseDirectory clears the + // Microsoft.Data.Sqlite connection pool on dispose before deleting the file; without + // that the pool keeps the .db handle open and the delete throws "used by another + // process" on Windows (a latent full-suite flake — the file opens fine on macOS). + using TempDatabaseDirectory directory = TempDatabaseDirectory.Create("authcheck"); + var check = new AuthStoreHealthCheck(FactoryFor(directory.DatabasePath())); + var result = await check.CheckHealthAsync(new HealthCheckContext()); + Assert.Equal(HealthStatus.Healthy, result.Status); } /// The health check reports unhealthy when the database path cannot be opened. diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/GatewayTlsBootstrapTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/GatewayTlsBootstrapTests.cs index 38cc3c9..810b9cd 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/GatewayTlsBootstrapTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/GatewayTlsBootstrapTests.cs @@ -9,6 +9,10 @@ using ZB.MOM.WW.MxGateway.Server; namespace ZB.MOM.WW.MxGateway.Tests.Gateway; +// Sets process-global Kestrel/TLS environment variables that GatewayApplication.Build reads; +// serialized against all other collections so a parallel host-building test cannot inherit them +// mid-run and race on the generated-certificate file. See GlobalEnvironmentCollection. +[Collection(TestSupport.GlobalEnvironmentCollection.Name)] public sealed class GatewayTlsBootstrapTests { /// diff --git a/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/GlobalEnvironmentCollection.cs b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/GlobalEnvironmentCollection.cs new file mode 100644 index 0000000..eca51ca --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/GlobalEnvironmentCollection.cs @@ -0,0 +1,18 @@ +namespace ZB.MOM.WW.MxGateway.Tests.TestSupport; + +/// +/// xUnit collection for tests that mutate process-global state (environment variables +/// read by GatewayApplication.Build, e.g. Kestrel__Endpoints__… and +/// MxGateway__Tls__SelfSignedCertPath). DisableParallelization keeps such a test from +/// running concurrently with any other collection: otherwise a parallel host-building test inherits +/// the mutated variables mid-run and the two race on the same generated-certificate file (on Windows, +/// "the process cannot access the file … because it is being used by another process"). Membership is +/// deliberately narrow — only add classes that set/clear real environment variables, not ones that +/// pass configuration through Build([...]) command-line args. +/// +[CollectionDefinition(Name, DisableParallelization = true)] +public sealed class GlobalEnvironmentCollection +{ + /// The collection name applied via [Collection(GlobalEnvironmentCollection.Name)]. + public const string Name = "GlobalEnvironmentMutation"; +} diff --git a/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TestHostEnvironmentInitializer.cs b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TestHostEnvironmentInitializer.cs index 580de0e..5c00e3c 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TestHostEnvironmentInitializer.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/TestSupport/TestHostEnvironmentInitializer.cs @@ -3,7 +3,8 @@ using System.Runtime.CompilerServices; namespace ZB.MOM.WW.MxGateway.Tests.TestSupport; /// -/// Defaults the host environment to Development for the whole test assembly. +/// Defaults the host environment to Development and isolates the test process's on-disk +/// gateway paths, for the whole test assembly. /// /// /// Many tests build the full gateway host through GatewayApplication.Build against the dev @@ -15,6 +16,15 @@ namespace ZB.MOM.WW.MxGateway.Tests.TestSupport; /// GatewayOptionsValidator with its isProduction constructor (no host environment) and /// are unaffected; a test that needs Production can still pass --environment=Production, which /// overrides this default. +/// +/// The self-signed-cert path is also defaulted to a per-process temp file. Otherwise every +/// full-host test that triggers HTTPS-cert generation writes the shared +/// CommonApplicationData/MxGateway/certs/gateway-selfsigned.pfx default — parallel xUnit +/// test classes then collide on that path's temp file (Windows: "the process cannot access the +/// file ... because it is being used by another process"), and on a shared CI/dev box the suite +/// also fights the deployed gateway service for the same file. Pointing at a per-process path +/// isolates the suite: the first host-building test generates the cert, the rest load it. +/// /// internal static class TestHostEnvironmentInitializer { @@ -25,5 +35,17 @@ internal static class TestHostEnvironmentInitializer { Environment.SetEnvironmentVariable("ASPNETCORE_ENVIRONMENT", "Development"); } + + if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("MxGateway__Tls__SelfSignedCertPath"))) + { + // ProcessId keeps the path stable within this test process (so a valid cert + // generated by the first host-building test is reused by the rest) yet unique + // across processes (so concurrent test runs / the deployed service never share it). + string certPath = Path.Combine( + Path.GetTempPath(), + $"mxgw-tests-{Environment.ProcessId}", + "gateway-selfsigned.pfx"); + Environment.SetEnvironmentVariable("MxGateway__Tls__SelfSignedCertPath", certPath); + } } } From e1a505d662592022eec4465cc8b1d443e318b16d Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 08:20:43 -0400 Subject: [PATCH 12/19] docs(archreview-p1): record Windows temp-file-lock flakiness fix + windev verification 3 consecutive windev full-suite runs: 793 passed / 2 failed, 0 orphans (was flaky 2-4 fails). The 2 stable failures are pre-existing windev-environmental (cert SAN machine-name assertion; event-stream concurrency timing) and pass on macOS. --- archreview/remediation/00-tracking.md | 1 + 1 file changed, 1 insertion(+) diff --git a/archreview/remediation/00-tracking.md b/archreview/remediation/00-tracking.md index e0d8a76..5c0d61a 100644 --- a/archreview/remediation/00-tracking.md +++ b/archreview/remediation/00-tracking.md @@ -255,6 +255,7 @@ Findings the review flagged as one coordinated design pass — sequence them tog |---|---| | 2026-07-09 | Initial tracking doc generated from the six domain remediation designs. All 153 findings `Not started`. | | 2026-07-09 | P0 tier executed via parallel agents. GWC-01/02/03, TST-02, TST-12, CLI-01, CLI-03 → `Done` (NonWindows build clean; gateway 135/135 targeted tests pass, Go `go test ./...` + `-race` pass, Rust `test`/`clippy -D warnings` pass). | +| 2026-07-09 | Windows full-suite temp-file-lock flakiness (discovered via TST-08) → **fixed** (commit `11a716a`). Two shared-state parallel collisions that macOS never surfaces: (1) self-signed cert generation — `SelfSignedCertificateProvider` now stages the PFX in a unique `..tmp` (was a fixed name that concurrent/interrupted writers collided on); `TestHostEnvironmentInitializer` defaults the cert path to a per-process temp dir (so the suite never writes shared `ProgramData` or fights the deployed service); `GatewayTlsBootstrapTests` moved to a `DisableParallelization` collection so its process-global env-var mutation can't bleed into parallel tests. (2) `AuthStoreHealthCheckTests` — reuse the existing `TempDatabaseDirectory` helper (`SqliteConnection.ClearAllPools()` before delete) so the Microsoft.Data.Sqlite pool releases the temp `.db` handle. **windev-verified:** 3 consecutive full-suite runs all **793 passed / 2 failed, 0 new orphans** (was flaky 2–4 fails, count-varying). The 2 stable remaining failures are pre-existing windev-environmental (`SelfSignedCertificateProviderTests.GenerateCertificate_HasExpectedSansEkuAndValidity` machine-name/FQDN SAN assertion; `EventStreamServiceTests…TracksAggregateQueueDepth` timing) — both pass on macOS, unrelated to this change. | | 2026-07-09 | P1 TST-08 (orphaned testhost) → `Done` by evidence: the claimed leak does **not** reproduce. Full `ZB.MOM.WW.MxGateway.Tests` suite exits cleanly on macOS (0 survivors) and on the Windows dev box (windev, `origin/main` worktree: 0 new `testhost` and 0 new `ZB.MOM.WW.MxGateway.Worker` processes after a full run, isolated by StartTime). Static audit confirms the prime-suspect fixtures already dispose deterministically (WorkerClient cancels its `_stopCts` + awaits read/write/heartbeat tasks with a 5 s timeout; GatewaySession disposes the client; E2E/harness fixtures use `await using`). CLAUDE.md "Source Update Workflow" updated to drop the stale "leaves orphaned testhost processes" rationale (filtered runs kept as a speed guideline). **Discovered separately (NOT TST-08, still open):** Windows-only temp-file-lock flakiness in ~4 full-host test classes — `AuthStoreHealthCheckTests` (Microsoft.Data.Sqlite connection-pool holds the temp `.db` handle after `await using`, so the finally `File.Delete` throws "used by another process"), `GatewayTlsBootstrapTests`/`DashboardCookieOptionsTests`/`DashboardHubsRegistrationTests` (a `gw.pfx.tmp` handle from `SelfSignedCertificateProvider`'s fixed-name `path+".tmp"` atomic-write racing teardown). Count varies run-to-run (2–4 fails); macOS never sees it (Unix allows deleting open files). Fix = `SqliteConnection.ClearAllPools()` before temp delete + a unique/guarded `.tmp` name; verify on windev. | | 2026-07-09 | P1 S-misc (dashboard/observability hardening). SEC-02/12/20 → `Done`. SEC-02: `DashboardAuthorizationHandler` restricts the loopback + `Authentication:Mode=Disabled` bypasses to read-only (they satisfy a Viewer-bearing requirement but never `AdminOnly`), closing the policy-layer gap where anonymous localhost was authorized for Admin surfaces. SEC-12: `DashboardSessionAdminService` now emits canonical `AuditEvent`s (`dashboard-close-session`/`dashboard-kill-worker`, category `SessionAdmin`) through `IAuditWriter` on Success/Failure/Denied, so Close/Kill land durable audit rows. SEC-20: dropped the unbounded `session_id` tag from the exported `mxgateway.heartbeats.failed` counter. Docs updated same-change (CLAUDE.md, GatewayDashboardDesign.md, Metrics.md). Server build clean (0 warnings); targeted classes 30/30 pass; broader Dashboard+Security+GatewayApplication+Metrics sweep 295/295 pass. | | 2026-07-09 | P1 Wave 2b (security authz+hub). SEC-05/07/08/11 → `Done` (hub-token lifetime; QueryActiveAlarms scope arm; gateway-side verification cache + last-used coalescing; login rate limit + per-peer gRPC failure limiter). Full-suite checkpoint caught + fixed regressions the earlier narrow SEC-01/04/06 filter missed: cross-platform path-rooting, an `IHostEnvironment` fallback for minimal DI containers, a test-assembly `ASPNETCORE_ENVIRONMENT=Development` default, and a platform-correct default-path assertion. Suite: 747 passed / 42 failed, all 42 pre-existing macOS named-pipe-harness env failures. | From c8b3a2281a67f617aed5c37e9599831a8a3e5713 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 08:49:59 -0400 Subject: [PATCH 13/19] feat(ipc): negotiate worker frame max, add gRPC headroom, bound DrainEvents (IPC-02/03/04 gateway half) Proto foundation + gateway-side of the size/backpressure pass: - IPC-02: add GatewayHello.max_frame_bytes (regen Generated/); gateway sends its negotiated worker-frame max in the handshake so the worker can adopt it instead of a hard-coded default. Worker read-half lands separately. - IPC-03: give the pipe frame max envelope-overhead headroom above the public gRPC cap (WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes = 64 KiB; default Worker.MaxMessageBytes bumped to 16 MiB + reserve), cross-validate the headroom at startup, and pre-check command envelope size in WorkerClient so an oversized command fails only that correlation (ResourceExhausted) instead of faulting the whole session. - IPC-04: reject DrainEvents max_events above a public ceiling in the request validator (worker per-reply cap lands with the worker half). Docs: GatewayConfiguration, WorkerFrameProtocol, gateway.md. Tests: headroom validation, DrainEvents bound, oversized-command per-command failure (pipe-harness, verified on windev). --- docs/GatewayConfiguration.md | 12 +- docs/WorkerFrameProtocol.md | 13 +- gateway.md | 5 +- .../Generated/MxaccessWorker.cs | 149 ++++++++++++------ .../Protos/mxaccess_worker.proto | 6 + .../Configuration/GatewayOptionsValidator.cs | 31 ++++ .../Configuration/WorkerOptions.cs | 12 +- .../Grpc/MxAccessGatewayService.cs | 1 + .../Grpc/MxAccessGrpcRequestValidator.cs | 15 ++ .../Workers/WorkerClient.cs | 22 ++- .../Workers/WorkerClientErrorCode.cs | 5 + .../Workers/WorkerFrameProtocolOptions.cs | 9 ++ .../appsettings.json | 2 +- .../GatewayOptionsValidatorTests.cs | 46 ++++++ .../Grpc/MxAccessGrpcRequestValidatorTests.cs | 45 ++++++ .../Gateway/Workers/WorkerClientTests.cs | 54 ++++++- 16 files changed, 362 insertions(+), 65 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/MxAccessGrpcRequestValidatorTests.cs diff --git a/docs/GatewayConfiguration.md b/docs/GatewayConfiguration.md index 3b1eaec..7dca44b 100644 --- a/docs/GatewayConfiguration.md +++ b/docs/GatewayConfiguration.md @@ -29,7 +29,7 @@ paths, timeouts, queue sizes, enum values, or protocol values are invalid. "ShutdownTimeoutSeconds": 10, "HeartbeatIntervalSeconds": 5, "HeartbeatGraceSeconds": 15, - "MaxMessageBytes": 16777216 + "MaxMessageBytes": 16842752 }, "Sessions": { "DefaultCommandTimeoutSeconds": 30, @@ -117,12 +117,16 @@ CWD (SEC-01). | `MxGateway:Worker:ShutdownTimeoutSeconds` | `10` | Grace period for worker shutdown before the gateway treats shutdown as failed and may kill the worker process tree. | | `MxGateway:Worker:HeartbeatIntervalSeconds` | `5` | Worker heartbeat send interval and gateway heartbeat check cadence input. | | `MxGateway:Worker:HeartbeatGraceSeconds` | `15` | Maximum age of the last worker heartbeat before the gateway faults the worker. This must be greater than or equal to `HeartbeatIntervalSeconds`. | -| `MxGateway:Worker:MaxMessageBytes` | `16777216` | Maximum worker IPC frame payload size in bytes. The validator allows values from `1024` through `268435456`. | +| `MxGateway:Worker:MaxMessageBytes` | `16842752` | Maximum worker IPC frame payload size in bytes. The validator allows values from `1024` through `268435456`, and additionally requires this to be at least `MxGateway:Protocol:MaxGrpcMessageBytes` plus a 65536-byte (64 KiB) envelope-overhead reserve. The default is the 16 MiB public gRPC cap plus that reserve. The gateway conveys the negotiated value to the worker in the handshake (`GatewayHello.max_frame_bytes`), so the worker frames to the same limit rather than a hard-coded default. | `StartupProbeRetryAttempts`, `StartupProbeRetryDelayMilliseconds`, `PipeConnectAttemptTimeoutMilliseconds`, timeout values, heartbeat values, and `MaxMessageBytes` must be positive. `MaxMessageBytes` is intentionally bounded -to avoid accidental large allocations from malformed or oversized frames. +to avoid accidental large allocations from malformed or oversized frames. It +must also stay above `MaxGrpcMessageBytes` by the envelope-overhead reserve so a +maximally-sized accepted gRPC payload always fits one worker frame once wrapped +in a `WorkerEnvelope`; otherwise the oversized outbound write would fault the +whole session. Startup validation fails fast when the headroom is violated. ## Session Options @@ -256,7 +260,7 @@ dev→production hardening posture. | Option | Default | Description | |--------|---------|-------------| | `MxGateway:Protocol:WorkerProtocolVersion` | `1` | Worker IPC protocol version expected by the gateway and worker. This must match `GatewayContractInfo.WorkerProtocolVersion`. | -| `MxGateway:Protocol:MaxGrpcMessageBytes` | `16777216` | Public gRPC max send and receive message size in bytes. The same default is used by official clients. The validator allows values from `1024` through `268435456`. | +| `MxGateway:Protocol:MaxGrpcMessageBytes` | `16777216` | Public gRPC max send and receive message size in bytes. The same default is used by official clients. The validator allows values from `1024` through `268435456`, and requires `MxGateway:Worker:MaxMessageBytes` to exceed this by at least the 64 KiB worker-frame envelope reserve (see the Worker options). | The protocol option is exposed for diagnostics and explicit deployment configuration, not for compatibility negotiation. A mismatch fails validation diff --git a/docs/WorkerFrameProtocol.md b/docs/WorkerFrameProtocol.md index 8a43fbe..b74ff6e 100644 --- a/docs/WorkerFrameProtocol.md +++ b/docs/WorkerFrameProtocol.md @@ -17,8 +17,17 @@ payload_length bytes protobuf WorkerEnvelope ``` The reader rejects zero-length payloads and payloads larger than the configured -maximum before allocating the payload buffer. The default maximum is 16 MiB, -matching the gateway process design. +maximum before allocating the payload buffer. The default maximum is the 16 MiB +public gRPC cap plus a 64 KiB envelope-overhead reserve (16842752 bytes) so a +maximally-sized accepted gRPC payload always fits one worker frame once wrapped +in a `WorkerEnvelope`. + +The gateway is the source of truth for this maximum: it conveys the negotiated +value in the handshake as `GatewayHello.max_frame_bytes`, and the worker adopts +it as its `WorkerFrameProtocolOptions.MaxMessageBytes` instead of a hard-coded +default. A `max_frame_bytes` of 0 (an older gateway that never set the field) +means "use the worker's built-in default". This keeps both ends framing to the +same limit rather than depending on matched compile-time constants. ## Envelope Validation diff --git a/gateway.md b/gateway.md index 30267df..0c5fdf5 100644 --- a/gateway.md +++ b/gateway.md @@ -424,7 +424,10 @@ Optional diagnostics: - `Ping` - `GetSessionState` - `GetWorkerInfo` -- `DrainEvents` +- `DrainEvents` — diagnostic; `max_events` is bounded (the gateway rejects requests + above a public ceiling, and the worker caps each reply at its own per-reply limit, + treating `max_events = 0` as "the default cap") so one drain cannot pack an + unbounded, session-killing reply frame. - `ShutdownWorker` Do not compress MXAccess semantics into generic verbs too early. A command enum diff --git a/src/ZB.MOM.WW.MxGateway.Contracts/Generated/MxaccessWorker.cs b/src/ZB.MOM.WW.MxGateway.Contracts/Generated/MxaccessWorker.cs index 268af42..d8e1444 100644 --- a/src/ZB.MOM.WW.MxGateway.Contracts/Generated/MxaccessWorker.cs +++ b/src/ZB.MOM.WW.MxGateway.Contracts/Generated/MxaccessWorker.cs @@ -44,63 +44,64 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { "Y2Nlc3Nfd29ya2VyLnYxLldvcmtlckV2ZW50SAASPwoQd29ya2VyX2hlYXJ0", "YmVhdBgTIAEoCzIjLm14YWNjZXNzX3dvcmtlci52MS5Xb3JrZXJIZWFydGJl", "YXRIABI3Cgx3b3JrZXJfZmF1bHQYFCABKAsyHy5teGFjY2Vzc193b3JrZXIu", - "djEuV29ya2VyRmF1bHRIAEIGCgRib2R5IloKDEdhdGV3YXlIZWxsbxIiChpz", + "djEuV29ya2VyRmF1bHRIAEIGCgRib2R5InMKDEdhdGV3YXlIZWxsbxIiChpz", "dXBwb3J0ZWRfcHJvdG9jb2xfdmVyc2lvbhgBIAEoDRINCgVub25jZRgCIAEo", - "CRIXCg9nYXRld2F5X3ZlcnNpb24YAyABKAkiaQoLV29ya2VySGVsbG8SGAoQ", - "cHJvdG9jb2xfdmVyc2lvbhgBIAEoDRINCgVub25jZRgCIAEoCRIZChF3b3Jr", - "ZXJfcHJvY2Vzc19pZBgDIAEoBRIWCg53b3JrZXJfdmVyc2lvbhgEIAEoCSKO", - "AQoLV29ya2VyUmVhZHkSGQoRd29ya2VyX3Byb2Nlc3NfaWQYASABKAUSFwoP", - "bXhhY2Nlc3NfcHJvZ2lkGAIgASgJEhYKDm14YWNjZXNzX2Nsc2lkGAMgASgJ", - "EjMKD3JlYWR5X3RpbWVzdGFtcBgEIAEoCzIaLmdvb2dsZS5wcm90b2J1Zi5U", - "aW1lc3RhbXAidwoNV29ya2VyQ29tbWFuZBIvCgdjb21tYW5kGAEgASgLMh4u", - "bXhhY2Nlc3NfZ2F0ZXdheS52MS5NeENvbW1hbmQSNQoRZW5xdWV1ZV90aW1l", - "c3RhbXAYAiABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wIoEBChJX", - "b3JrZXJDb21tYW5kUmVwbHkSMgoFcmVwbHkYASABKAsyIy5teGFjY2Vzc19n", - "YXRld2F5LnYxLk14Q29tbWFuZFJlcGx5EjcKE2NvbXBsZXRlZF90aW1lc3Rh", - "bXAYAiABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wIh4KDFdvcmtl", - "ckNhbmNlbBIOCgZyZWFzb24YASABKAkiUQoOV29ya2VyU2h1dGRvd24SLwoM", - "Z3JhY2VfcGVyaW9kGAEgASgLMhkuZ29vZ2xlLnByb3RvYnVmLkR1cmF0aW9u", - "Eg4KBnJlYXNvbhgCIAEoCSJIChFXb3JrZXJTaHV0ZG93bkFjaxIzCgZzdGF0", - "dXMYASABKAsyIy5teGFjY2Vzc19nYXRld2F5LnYxLlByb3RvY29sU3RhdHVz", - "IjoKC1dvcmtlckV2ZW50EisKBWV2ZW50GAEgASgLMhwubXhhY2Nlc3NfZ2F0", - "ZXdheS52MS5NeEV2ZW50IqUCCg9Xb3JrZXJIZWFydGJlYXQSGQoRd29ya2Vy", - "X3Byb2Nlc3NfaWQYASABKAUSLgoFc3RhdGUYAiABKA4yHy5teGFjY2Vzc193", - "b3JrZXIudjEuV29ya2VyU3RhdGUSPwobbGFzdF9zdGFfYWN0aXZpdHlfdGlt", - "ZXN0YW1wGAMgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRpbWVzdGFtcBIdChVw", - "ZW5kaW5nX2NvbW1hbmRfY291bnQYBCABKA0SIgoab3V0Ym91bmRfZXZlbnRf", - "cXVldWVfZGVwdGgYBSABKA0SGwoTbGFzdF9ldmVudF9zZXF1ZW5jZRgGIAEo", - "BBImCh5jdXJyZW50X2NvbW1hbmRfY29ycmVsYXRpb25faWQYByABKAki9AEK", - "C1dvcmtlckZhdWx0EjkKCGNhdGVnb3J5GAEgASgOMicubXhhY2Nlc3Nfd29y", - "a2VyLnYxLldvcmtlckZhdWx0Q2F0ZWdvcnkSFgoOY29tbWFuZF9tZXRob2QY", - "AiABKAkSFAoHaHJlc3VsdBgDIAEoBUgAiAEBEhYKDmV4Y2VwdGlvbl90eXBl", - "GAQgASgJEhoKEmRpYWdub3N0aWNfbWVzc2FnZRgFIAEoCRI8Cg9wcm90b2Nv", - "bF9zdGF0dXMYBiABKAsyIy5teGFjY2Vzc19nYXRld2F5LnYxLlByb3RvY29s", - "U3RhdHVzQgoKCF9ocmVzdWx0KpcCCgtXb3JrZXJTdGF0ZRIcChhXT1JLRVJf", - "U1RBVEVfVU5TUEVDSUZJRUQQABIZChVXT1JLRVJfU1RBVEVfU1RBUlRJTkcQ", - "ARIcChhXT1JLRVJfU1RBVEVfSEFORFNIQUtJTkcQAhIhCh1XT1JLRVJfU1RB", - "VEVfSU5JVElBTElaSU5HX1NUQRADEhYKEldPUktFUl9TVEFURV9SRUFEWRAE", - "EiIKHldPUktFUl9TVEFURV9FWEVDVVRJTkdfQ09NTUFORBAFEh4KGldPUktF", - "Ul9TVEFURV9TSFVUVElOR19ET1dOEAYSGAoUV09SS0VSX1NUQVRFX1NUT1BQ", - "RUQQBxIYChRXT1JLRVJfU1RBVEVfRkFVTFRFRBAIKscEChNXb3JrZXJGYXVs", - "dENhdGVnb3J5EiUKIVdPUktFUl9GQVVMVF9DQVRFR09SWV9VTlNQRUNJRklF", - "RBAAEisKJ1dPUktFUl9GQVVMVF9DQVRFR09SWV9JTlZBTElEX0FSR1VNRU5U", - "UxABEjcKM1dPUktFUl9GQVVMVF9DQVRFR09SWV9HQVRFV0FZX0FVVEhFTlRJ", - "Q0FUSU9OX0ZBSUxFRBACEisKJ1dPUktFUl9GQVVMVF9DQVRFR09SWV9QUk9U", - "T0NPTF9NSVNNQVRDSBADEiwKKFdPUktFUl9GQVVMVF9DQVRFR09SWV9QUk9U", - "T0NPTF9WSU9MQVRJT04QBBIrCidXT1JLRVJfRkFVTFRfQ0FURUdPUllfUElQ", - "RV9ESVNDT05ORUNURUQQBRIyCi5XT1JLRVJfRkFVTFRfQ0FURUdPUllfTVhB", - "Q0NFU1NfQ1JFQVRJT05fRkFJTEVEEAYSMQotV09SS0VSX0ZBVUxUX0NBVEVH", - "T1JZX01YQUNDRVNTX0NPTU1BTkRfRkFJTEVEEAcSOgo2V09SS0VSX0ZBVUxU", - "X0NBVEVHT1JZX01YQUNDRVNTX0VWRU5UX0NPTlZFUlNJT05fRkFJTEVEEAgS", - "IgoeV09SS0VSX0ZBVUxUX0NBVEVHT1JZX1NUQV9IVU5HEAkSKAokV09SS0VS", - "X0ZBVUxUX0NBVEVHT1JZX1FVRVVFX09WRVJGTE9XEAoSKgomV09SS0VSX0ZB", - "VUxUX0NBVEVHT1JZX1NIVVRET1dOX1RJTUVPVVQQC0ImqgIjWkIuTU9NLldX", - "Lk14R2F0ZXdheS5Db250cmFjdHMuUHJvdG9iBnByb3RvMw==")); + "CRIXCg9nYXRld2F5X3ZlcnNpb24YAyABKAkSFwoPbWF4X2ZyYW1lX2J5dGVz", + "GAQgASgNImkKC1dvcmtlckhlbGxvEhgKEHByb3RvY29sX3ZlcnNpb24YASAB", + "KA0SDQoFbm9uY2UYAiABKAkSGQoRd29ya2VyX3Byb2Nlc3NfaWQYAyABKAUS", + "FgoOd29ya2VyX3ZlcnNpb24YBCABKAkijgEKC1dvcmtlclJlYWR5EhkKEXdv", + "cmtlcl9wcm9jZXNzX2lkGAEgASgFEhcKD214YWNjZXNzX3Byb2dpZBgCIAEo", + "CRIWCg5teGFjY2Vzc19jbHNpZBgDIAEoCRIzCg9yZWFkeV90aW1lc3RhbXAY", + "BCABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wIncKDVdvcmtlckNv", + "bW1hbmQSLwoHY29tbWFuZBgBIAEoCzIeLm14YWNjZXNzX2dhdGV3YXkudjEu", + "TXhDb21tYW5kEjUKEWVucXVldWVfdGltZXN0YW1wGAIgASgLMhouZ29vZ2xl", + "LnByb3RvYnVmLlRpbWVzdGFtcCKBAQoSV29ya2VyQ29tbWFuZFJlcGx5EjIK", + "BXJlcGx5GAEgASgLMiMubXhhY2Nlc3NfZ2F0ZXdheS52MS5NeENvbW1hbmRS", + "ZXBseRI3ChNjb21wbGV0ZWRfdGltZXN0YW1wGAIgASgLMhouZ29vZ2xlLnBy", + "b3RvYnVmLlRpbWVzdGFtcCIeCgxXb3JrZXJDYW5jZWwSDgoGcmVhc29uGAEg", + "ASgJIlEKDldvcmtlclNodXRkb3duEi8KDGdyYWNlX3BlcmlvZBgBIAEoCzIZ", + "Lmdvb2dsZS5wcm90b2J1Zi5EdXJhdGlvbhIOCgZyZWFzb24YAiABKAkiSAoR", + "V29ya2VyU2h1dGRvd25BY2sSMwoGc3RhdHVzGAEgASgLMiMubXhhY2Nlc3Nf", + "Z2F0ZXdheS52MS5Qcm90b2NvbFN0YXR1cyI6CgtXb3JrZXJFdmVudBIrCgVl", + "dmVudBgBIAEoCzIcLm14YWNjZXNzX2dhdGV3YXkudjEuTXhFdmVudCKlAgoP", + "V29ya2VySGVhcnRiZWF0EhkKEXdvcmtlcl9wcm9jZXNzX2lkGAEgASgFEi4K", + "BXN0YXRlGAIgASgOMh8ubXhhY2Nlc3Nfd29ya2VyLnYxLldvcmtlclN0YXRl", + "Ej8KG2xhc3Rfc3RhX2FjdGl2aXR5X3RpbWVzdGFtcBgDIAEoCzIaLmdvb2ds", + "ZS5wcm90b2J1Zi5UaW1lc3RhbXASHQoVcGVuZGluZ19jb21tYW5kX2NvdW50", + "GAQgASgNEiIKGm91dGJvdW5kX2V2ZW50X3F1ZXVlX2RlcHRoGAUgASgNEhsK", + "E2xhc3RfZXZlbnRfc2VxdWVuY2UYBiABKAQSJgoeY3VycmVudF9jb21tYW5k", + "X2NvcnJlbGF0aW9uX2lkGAcgASgJIvQBCgtXb3JrZXJGYXVsdBI5CghjYXRl", + "Z29yeRgBIAEoDjInLm14YWNjZXNzX3dvcmtlci52MS5Xb3JrZXJGYXVsdENh", + "dGVnb3J5EhYKDmNvbW1hbmRfbWV0aG9kGAIgASgJEhQKB2hyZXN1bHQYAyAB", + "KAVIAIgBARIWCg5leGNlcHRpb25fdHlwZRgEIAEoCRIaChJkaWFnbm9zdGlj", + "X21lc3NhZ2UYBSABKAkSPAoPcHJvdG9jb2xfc3RhdHVzGAYgASgLMiMubXhh", + "Y2Nlc3NfZ2F0ZXdheS52MS5Qcm90b2NvbFN0YXR1c0IKCghfaHJlc3VsdCqX", + "AgoLV29ya2VyU3RhdGUSHAoYV09SS0VSX1NUQVRFX1VOU1BFQ0lGSUVEEAAS", + "GQoVV09SS0VSX1NUQVRFX1NUQVJUSU5HEAESHAoYV09SS0VSX1NUQVRFX0hB", + "TkRTSEFLSU5HEAISIQodV09SS0VSX1NUQVRFX0lOSVRJQUxJWklOR19TVEEQ", + "AxIWChJXT1JLRVJfU1RBVEVfUkVBRFkQBBIiCh5XT1JLRVJfU1RBVEVfRVhF", + "Q1VUSU5HX0NPTU1BTkQQBRIeChpXT1JLRVJfU1RBVEVfU0hVVFRJTkdfRE9X", + "ThAGEhgKFFdPUktFUl9TVEFURV9TVE9QUEVEEAcSGAoUV09SS0VSX1NUQVRF", + "X0ZBVUxURUQQCCrHBAoTV29ya2VyRmF1bHRDYXRlZ29yeRIlCiFXT1JLRVJf", + "RkFVTFRfQ0FURUdPUllfVU5TUEVDSUZJRUQQABIrCidXT1JLRVJfRkFVTFRf", + "Q0FURUdPUllfSU5WQUxJRF9BUkdVTUVOVFMQARI3CjNXT1JLRVJfRkFVTFRf", + "Q0FURUdPUllfR0FURVdBWV9BVVRIRU5USUNBVElPTl9GQUlMRUQQAhIrCidX", + "T1JLRVJfRkFVTFRfQ0FURUdPUllfUFJPVE9DT0xfTUlTTUFUQ0gQAxIsCihX", + "T1JLRVJfRkFVTFRfQ0FURUdPUllfUFJPVE9DT0xfVklPTEFUSU9OEAQSKwon", + "V09SS0VSX0ZBVUxUX0NBVEVHT1JZX1BJUEVfRElTQ09OTkVDVEVEEAUSMgou", + "V09SS0VSX0ZBVUxUX0NBVEVHT1JZX01YQUNDRVNTX0NSRUFUSU9OX0ZBSUxF", + "RBAGEjEKLVdPUktFUl9GQVVMVF9DQVRFR09SWV9NWEFDQ0VTU19DT01NQU5E", + "X0ZBSUxFRBAHEjoKNldPUktFUl9GQVVMVF9DQVRFR09SWV9NWEFDQ0VTU19F", + "VkVOVF9DT05WRVJTSU9OX0ZBSUxFRBAIEiIKHldPUktFUl9GQVVMVF9DQVRF", + "R09SWV9TVEFfSFVORxAJEigKJFdPUktFUl9GQVVMVF9DQVRFR09SWV9RVUVV", + "RV9PVkVSRkxPVxAKEioKJldPUktFUl9GQVVMVF9DQVRFR09SWV9TSFVURE9X", + "Tl9USU1FT1VUEAtCJqoCI1pCLk1PTS5XVy5NeEdhdGV3YXkuQ29udHJhY3Rz", + "LlByb3RvYgZwcm90bzM=")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { global::Google.Protobuf.WellKnownTypes.DurationReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.TimestampReflection.Descriptor, global::ZB.MOM.WW.MxGateway.Contracts.Proto.MxaccessGatewayReflection.Descriptor, }, new pbr::GeneratedClrTypeInfo(new[] {typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerState), typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerFaultCategory), }, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerEnvelope), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerEnvelope.Parser, new[]{ "ProtocolVersion", "SessionId", "Sequence", "CorrelationId", "GatewayHello", "WorkerHello", "WorkerReady", "WorkerCommand", "WorkerCommandReply", "WorkerCancel", "WorkerShutdown", "WorkerShutdownAck", "WorkerEvent", "WorkerHeartbeat", "WorkerFault" }, new[]{ "Body" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.GatewayHello), global::ZB.MOM.WW.MxGateway.Contracts.Proto.GatewayHello.Parser, new[]{ "SupportedProtocolVersion", "Nonce", "GatewayVersion" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.GatewayHello), global::ZB.MOM.WW.MxGateway.Contracts.Proto.GatewayHello.Parser, new[]{ "SupportedProtocolVersion", "Nonce", "GatewayVersion", "MaxFrameBytes" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerHello), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerHello.Parser, new[]{ "ProtocolVersion", "Nonce", "WorkerProcessId", "WorkerVersion" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerReady), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerReady.Parser, new[]{ "WorkerProcessId", "MxaccessProgid", "MxaccessClsid", "ReadyTimestamp" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerCommand), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerCommand.Parser, new[]{ "Command", "EnqueueTimestamp" }, null, null, null, null), @@ -1108,6 +1109,7 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { supportedProtocolVersion_ = other.supportedProtocolVersion_; nonce_ = other.nonce_; gatewayVersion_ = other.gatewayVersion_; + maxFrameBytes_ = other.maxFrameBytes_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -1153,6 +1155,25 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { } } + /// Field number for the "max_frame_bytes" field. + public const int MaxFrameBytesFieldNumber = 4; + private uint maxFrameBytes_; + /// + /// Maximum worker-frame payload size, in bytes, negotiated by the gateway from its + /// configured pipe limit. The worker adopts this as its frame-protocol MaxMessageBytes + /// instead of a hard-coded default; 0 (an older gateway that never set the field) means + /// "use the worker's built-in default". Sits above the public gRPC cap by an + /// envelope-overhead margin so an accepted gRPC payload always fits one worker frame. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public uint MaxFrameBytes { + get { return maxFrameBytes_; } + set { + maxFrameBytes_ = value; + } + } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public override bool Equals(object other) { @@ -1171,6 +1192,7 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { if (SupportedProtocolVersion != other.SupportedProtocolVersion) return false; if (Nonce != other.Nonce) return false; if (GatewayVersion != other.GatewayVersion) return false; + if (MaxFrameBytes != other.MaxFrameBytes) return false; return Equals(_unknownFields, other._unknownFields); } @@ -1181,6 +1203,7 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { if (SupportedProtocolVersion != 0) hash ^= SupportedProtocolVersion.GetHashCode(); if (Nonce.Length != 0) hash ^= Nonce.GetHashCode(); if (GatewayVersion.Length != 0) hash ^= GatewayVersion.GetHashCode(); + if (MaxFrameBytes != 0) hash ^= MaxFrameBytes.GetHashCode(); if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); } @@ -1211,6 +1234,10 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { output.WriteRawTag(26); output.WriteString(GatewayVersion); } + if (MaxFrameBytes != 0) { + output.WriteRawTag(32); + output.WriteUInt32(MaxFrameBytes); + } if (_unknownFields != null) { _unknownFields.WriteTo(output); } @@ -1233,6 +1260,10 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { output.WriteRawTag(26); output.WriteString(GatewayVersion); } + if (MaxFrameBytes != 0) { + output.WriteRawTag(32); + output.WriteUInt32(MaxFrameBytes); + } if (_unknownFields != null) { _unknownFields.WriteTo(ref output); } @@ -1252,6 +1283,9 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { if (GatewayVersion.Length != 0) { size += 1 + pb::CodedOutputStream.ComputeStringSize(GatewayVersion); } + if (MaxFrameBytes != 0) { + size += 1 + pb::CodedOutputStream.ComputeUInt32Size(MaxFrameBytes); + } if (_unknownFields != null) { size += _unknownFields.CalculateSize(); } @@ -1273,6 +1307,9 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { if (other.GatewayVersion.Length != 0) { GatewayVersion = other.GatewayVersion; } + if (other.MaxFrameBytes != 0) { + MaxFrameBytes = other.MaxFrameBytes; + } _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -1304,6 +1341,10 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { GatewayVersion = input.ReadString(); break; } + case 32: { + MaxFrameBytes = input.ReadUInt32(); + break; + } } } #endif @@ -1335,6 +1376,10 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { GatewayVersion = input.ReadString(); break; } + case 32: { + MaxFrameBytes = input.ReadUInt32(); + break; + } } } } diff --git a/src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_worker.proto b/src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_worker.proto index 06a22de..e50c4ff 100644 --- a/src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_worker.proto +++ b/src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_worker.proto @@ -42,6 +42,12 @@ message GatewayHello { uint32 supported_protocol_version = 1; string nonce = 2; string gateway_version = 3; + // Maximum worker-frame payload size, in bytes, negotiated by the gateway from its + // configured pipe limit. The worker adopts this as its frame-protocol MaxMessageBytes + // instead of a hard-coded default; 0 (an older gateway that never set the field) means + // "use the worker's built-in default". Sits above the public gRPC cap by an + // envelope-overhead margin so an accepted gRPC payload always fits one worker frame. + uint32 max_frame_bytes = 4; } message WorkerHello { diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs index 34c50e7..d271063 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/GatewayOptionsValidator.cs @@ -1,6 +1,7 @@ using ZB.MOM.WW.Auth.Abstractions.Ldap; using ZB.MOM.WW.Configuration; using ZB.MOM.WW.MxGateway.Contracts; +using ZB.MOM.WW.MxGateway.Server.Workers; namespace ZB.MOM.WW.MxGateway.Server.Configuration; @@ -46,6 +47,7 @@ public sealed class GatewayOptionsValidator : OptionsValidatorBase= MinimumMaxMessageBytes and <= MaximumMaxMessageBytes; + bool grpcInRange = protocol.MaxGrpcMessageBytes is >= MinimumMaxMessageBytes and <= MaximumMaxMessageBytes; + if (!workerInRange || !grpcInRange) + { + return; + } + + long requiredWorkerMax = + (long)protocol.MaxGrpcMessageBytes + WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes; + if (worker.MaxMessageBytes < requiredWorkerMax) + { + builder.Add( + $"MxGateway:Worker:MaxMessageBytes ({worker.MaxMessageBytes}) must be at least " + + $"MxGateway:Protocol:MaxGrpcMessageBytes ({protocol.MaxGrpcMessageBytes}) plus the " + + $"{WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes}-byte worker-frame envelope reserve " + + $"(>= {requiredWorkerMax})."); + } + } + private static void AddIfBlank(string? value, string message, ValidationBuilder builder) { builder.RequireThat(!string.IsNullOrWhiteSpace(value), message); diff --git a/src/ZB.MOM.WW.MxGateway.Server/Configuration/WorkerOptions.cs b/src/ZB.MOM.WW.MxGateway.Server/Configuration/WorkerOptions.cs index 28d3386..baf68ea 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Configuration/WorkerOptions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Configuration/WorkerOptions.cs @@ -33,6 +33,14 @@ public sealed class WorkerOptions /// The grace period in seconds after a heartbeat before considering the worker unresponsive. public int HeartbeatGraceSeconds { get; init; } = 15; - /// The maximum message size in bytes for IPC communication. - public int MaxMessageBytes { get; init; } = 16 * 1024 * 1024; + /// + /// The maximum worker-frame (pipe) message size in bytes. Must stay at least + /// above + /// so a maximally-sized accepted gRPC payload + /// still fits one worker frame (IPC-03); the gateway conveys this value to the worker in the + /// handshake (GatewayHello.max_frame_bytes, IPC-02). Default is the 16 MB public gRPC cap + /// plus that reserve. + /// + public int MaxMessageBytes { get; init; } = + (16 * 1024 * 1024) + Workers.WorkerFrameProtocolOptions.EnvelopeOverheadReserveBytes; } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs b/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs index 6d6d8d2..22675f1 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGatewayService.cs @@ -946,6 +946,7 @@ public sealed class MxAccessGatewayService( WorkerClientErrorCode.GatewayShutdown => StatusCode.Cancelled, WorkerClientErrorCode.InvalidState => StatusCode.FailedPrecondition, WorkerClientErrorCode.ProtocolViolation => StatusCode.Internal, + WorkerClientErrorCode.CommandTooLarge => StatusCode.ResourceExhausted, _ => StatusCode.Unavailable, }; diff --git a/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGrpcRequestValidator.cs b/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGrpcRequestValidator.cs index 5de7766..c6bc0e6 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGrpcRequestValidator.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Grpc/MxAccessGrpcRequestValidator.cs @@ -5,6 +5,13 @@ namespace ZB.MOM.WW.MxGateway.Server.Grpc; public sealed class MxAccessGrpcRequestValidator { + // Upper bound on a single DrainEvents request. DrainEvents is a diagnostics RPC that returns + // buffered events in one non-streaming reply, so an unbounded max_events could pack the whole + // queue into a session-killing frame (IPC-04). The worker independently caps each reply at its + // own MaxDrainEventsPerReply; this public bound rejects an obviously-abusive request loudly at + // the boundary. max_events = 0 is allowed and means "the worker's default batch cap". + private const uint MaxDrainEventsPerRequest = 10_000; + /// Validates an open session request. /// The request to validate. public void ValidateOpenSession(OpenSessionRequest request) @@ -69,6 +76,14 @@ public sealed class MxAccessGrpcRequestValidator throw InvalidArgument( $"Command kind {command.Kind} requires payload {expectedPayload} but received {command.PayloadCase}."); } + + // The payload case now matches the kind, so command.DrainEvents is non-null here. + if (command.Kind is MxCommandKind.DrainEvents && command.DrainEvents.MaxEvents > MaxDrainEventsPerRequest) + { + throw InvalidArgument( + $"DrainEvents max_events ({command.DrainEvents.MaxEvents}) must not exceed {MaxDrainEventsPerRequest}; " + + "use 0 to request the worker default batch cap."); + } } private static MxCommand.PayloadOneofCase ExpectedPayload(MxCommandKind kind) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs index 64b4e42..b8e603a 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs @@ -187,7 +187,23 @@ public sealed class WorkerClient : IWorkerClient try { - await EnqueueAsync(CreateCommandEnvelope(correlationId, command), cancellationToken).ConfigureAwait(false); + WorkerEnvelope commandEnvelope = CreateCommandEnvelope(correlationId, command); + + // Reject an oversized command at the enqueue boundary so only this correlation fails + // (ResourceExhausted) rather than the frame reaching the write loop and faulting the whole + // session (IPC-03). Command envelopes are the only gateway-authored outbound payload whose + // size the caller controls; checking here keeps a MessageTooLarge in the write loop a + // genuine desync signal. + int envelopeSize = commandEnvelope.CalculateSize(); + if (envelopeSize > _connection.FrameOptions.MaxMessageBytes) + { + throw new WorkerClientException( + WorkerClientErrorCode.CommandTooLarge, + $"Worker command {method} serializes to {envelopeSize} bytes, exceeding the negotiated " + + $"worker-frame maximum of {_connection.FrameOptions.MaxMessageBytes} bytes."); + } + + await EnqueueAsync(commandEnvelope, cancellationToken).ConfigureAwait(false); using CancellationTokenSource timeoutCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); Task timeoutTask = Task.Delay(timeout, timeoutCts.Token); Task replyTask = pendingCommand.Task; @@ -910,6 +926,10 @@ public sealed class WorkerClient : IWorkerClient SupportedProtocolVersion = _connection.FrameOptions.ProtocolVersion, Nonce = _connection.Nonce, GatewayVersion = typeof(GatewayContractInfo).Assembly.GetName().Version?.ToString() ?? GatewayVersionFallback, + + // Convey the negotiated worker-frame maximum so the worker adopts it instead of a + // hard-coded default (IPC-02). Sits above the public gRPC cap by the envelope reserve. + MaxFrameBytes = (uint)_connection.FrameOptions.MaxMessageBytes, }); } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClientErrorCode.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClientErrorCode.cs index a6a9542..8f32111 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClientErrorCode.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClientErrorCode.cs @@ -12,4 +12,9 @@ public enum WorkerClientErrorCode GatewayShutdown, WriteFailed, PendingCommandLimitExceeded, + + // The serialized command envelope exceeds the negotiated worker-frame maximum. Rejected at the + // enqueue boundary so only the offending command fails (mapped to ResourceExhausted) instead of + // the oversized frame reaching the write loop and faulting the whole session (IPC-03). + CommandTooLarge, } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerFrameProtocolOptions.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerFrameProtocolOptions.cs index ad9d083..1b664d2 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerFrameProtocolOptions.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerFrameProtocolOptions.cs @@ -10,6 +10,15 @@ public sealed class WorkerFrameProtocolOptions /// Default maximum message size in bytes (16 MB). public const int DefaultMaxMessageBytes = 16 * 1024 * 1024; + /// + /// Byte margin the worker-frame (pipe) maximum must keep above the public gRPC message cap so a + /// gRPC payload accepted at the public boundary always fits inside one worker frame once wrapped + /// in a WorkerEnvelope (correlation id, timestamps, oneof framing). Without this headroom + /// the pipe max equals the gRPC max and a maximally-sized accepted request faults the whole + /// session on the outbound write (IPC-03). 64 KiB is far larger than the fixed envelope overhead. + /// + public const int EnvelopeOverheadReserveBytes = 64 * 1024; + /// /// Initializes worker frame protocol options with a session ID. /// diff --git a/src/ZB.MOM.WW.MxGateway.Server/appsettings.json b/src/ZB.MOM.WW.MxGateway.Server/appsettings.json index cfe89ea..7623e47 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/appsettings.json +++ b/src/ZB.MOM.WW.MxGateway.Server/appsettings.json @@ -38,7 +38,7 @@ "ShutdownTimeoutSeconds": 10, "HeartbeatIntervalSeconds": 5, "HeartbeatGraceSeconds": 15, - "MaxMessageBytes": 16777216 + "MaxMessageBytes": 16842752 }, "Sessions": { "DefaultCommandTimeoutSeconds": 30, diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs index d2614a3..65c60f1 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsValidatorTests.cs @@ -771,4 +771,50 @@ public sealed class GatewayOptionsValidatorTests Assert.True(result.Failed); Assert.Contains(result.Failures!, f => f.Contains("ApiKeyFailureTrackedPeers")); } + + private static GatewayOptions WithWorkerAndProtocol(WorkerOptions worker, ProtocolOptions protocol) + { + GatewayOptions source = ValidOptions(); + return new GatewayOptions + { + Authentication = source.Authentication, + Ldap = source.Ldap, + Worker = worker, + Sessions = source.Sessions, + Events = source.Events, + Dashboard = source.Dashboard, + Protocol = protocol, + Alarms = source.Alarms, + Tls = source.Tls, + }; + } + + /// + /// Verifies the default worker-frame maximum keeps the required envelope-overhead reserve above + /// the default public gRPC cap, so a stock configuration passes the IPC-03 headroom check. + /// + [Fact] + public void Validate_Succeeds_WhenWorkerFrameMaxHasEnvelopeHeadroom() + { + ValidateOptionsResult result = new GatewayOptionsValidator().Validate(null, ValidOptions()); + Assert.True(result.Succeeded); + } + + /// + /// Verifies that a worker-frame maximum equal to the gRPC cap (zero headroom) fails validation: + /// a maximally-sized accepted gRPC payload would not fit one worker frame once wrapped in a + /// WorkerEnvelope, faulting the whole session on the outbound write (IPC-03). + /// + [Fact] + public void Validate_Fails_WhenWorkerFrameMaxEqualsGrpcMaxWithoutHeadroom() + { + const int grpcMax = 16 * 1024 * 1024; + ValidateOptionsResult result = new GatewayOptionsValidator().Validate( + null, + WithWorkerAndProtocol( + new WorkerOptions { MaxMessageBytes = grpcMax }, + new ProtocolOptions { MaxGrpcMessageBytes = grpcMax })); + Assert.True(result.Failed); + Assert.Contains(result.Failures!, f => f.Contains("MaxMessageBytes") && f.Contains("reserve")); + } } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/MxAccessGrpcRequestValidatorTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/MxAccessGrpcRequestValidatorTests.cs new file mode 100644 index 0000000..52f9908 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Grpc/MxAccessGrpcRequestValidatorTests.cs @@ -0,0 +1,45 @@ +using Grpc.Core; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.MxGateway.Server.Grpc; + +namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Grpc; + +public sealed class MxAccessGrpcRequestValidatorTests +{ + private static MxCommandRequest DrainRequest(uint maxEvents) => new() + { + SessionId = "session-1", + Command = new MxCommand + { + Kind = MxCommandKind.DrainEvents, + DrainEvents = new DrainEventsCommand { MaxEvents = maxEvents }, + }, + }; + + /// + /// Verifies a DrainEvents request within the per-request ceiling passes validation, including the + /// max_events = 0 "worker default cap" sentinel (IPC-04). + /// + [Theory] + [InlineData(0u)] + [InlineData(1u)] + [InlineData(10_000u)] + public void ValidateInvoke_AllowsDrainEvents_WithinCeiling(uint maxEvents) + { + MxAccessGrpcRequestValidator validator = new(); + validator.ValidateInvoke(DrainRequest(maxEvents)); + } + + /// + /// Verifies a DrainEvents request above the per-request ceiling is rejected with InvalidArgument + /// so one accepted request cannot pack an unbounded reply frame (IPC-04). + /// + [Fact] + public void ValidateInvoke_RejectsDrainEvents_AboveCeiling() + { + MxAccessGrpcRequestValidator validator = new(); + RpcException exception = Assert.Throws(() => validator.ValidateInvoke(DrainRequest(10_001))); + Assert.Equal(StatusCode.InvalidArgument, exception.StatusCode); + Assert.Contains("max_events", exception.Status.Detail, StringComparison.Ordinal); + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs index 53411bb..b3d38a5 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs @@ -56,6 +56,52 @@ public sealed class WorkerClientTests Assert.Equal(MxCommandKind.Ping, reply.Reply.Kind); } + /// + /// Verifies that a command whose serialized envelope exceeds the negotiated worker-frame maximum + /// fails only that command with at the enqueue + /// boundary, leaving the client ready for subsequent commands (IPC-03). Without the pre-check the + /// oversized frame would reach the write loop and fault the whole session. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task InvokeAsync_WhenCommandExceedsFrameMax_FailsOnlyThatCommandAndStaysReady() + { + await using PipePair pipePair = await PipePair.CreateAsync(); + await using WorkerClient client = CreateClient(pipePair, maxMessageBytes: 4096); + await CompleteHandshakeAsync(client, pipePair); + + WorkerCommand oversized = new() + { + Command = new MxCommand + { + Kind = MxCommandKind.Write, + Write = new WriteCommand + { + ServerHandle = 1, + ItemHandle = 2, + Value = new MxValue { StringValue = new string('x', 8192) }, + }, + }, + }; + + WorkerClientException exception = await Assert.ThrowsAsync( + async () => await client.InvokeAsync(oversized, TestTimeout, CancellationToken.None)); + Assert.Equal(WorkerClientErrorCode.CommandTooLarge, exception.ErrorCode); + Assert.Equal(WorkerClientState.Ready, client.State); + + // A subsequent normally-sized command still round-trips: the session was not faulted. + Task nextInvoke = client.InvokeAsync( + CreateCommand(MxCommandKind.Ping), + TestTimeout, + CancellationToken.None); + WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout); + await pipePair.WorkerWriter.WriteAsync( + CreateCommandReplyEnvelope(commandEnvelope.CorrelationId, MxCommandKind.Ping)); + WorkerCommandReply reply = await nextInvoke.WaitAsync(TestTimeout); + Assert.Equal(MxCommandKind.Ping, reply.Reply.Kind); + Assert.Equal(WorkerClientState.Ready, client.State); + } + /// Verifies that InvokeAsync ignores late replies and keeps the client ready. /// A task that represents the asynchronous operation. [Fact] @@ -564,9 +610,13 @@ public sealed class WorkerClientTests WorkerClientOptions? options = null, GatewayMetrics? metrics = null, WorkerProcessHandle? processHandle = null, - TimeProvider? timeProvider = null) + TimeProvider? timeProvider = null, + int maxMessageBytes = WorkerFrameProtocolOptions.DefaultMaxMessageBytes) { - WorkerFrameProtocolOptions frameOptions = new(SessionId); + WorkerFrameProtocolOptions frameOptions = new( + SessionId, + GatewayContractInfo.WorkerProtocolVersion, + maxMessageBytes); WorkerClientConnection connection = new( SessionId, Nonce, From ebe6aeac988d97a5c12f410e0a2d1f91910c4618 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 09:09:14 -0400 Subject: [PATCH 14/19] feat(worker): adopt negotiated frame max, bound drain, priority write scheduler (IPC-02/04 + WRK-04/07 worker half) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Worker half of the Wave 3 size/backpressure + write-ordering pass: - IPC-02: the worker adopts GatewayHello.max_frame_bytes during the handshake (WorkerFrameProtocolOptions.AdoptNegotiatedMaxMessageBytes) instead of a hard-coded default; 0 keeps the default, a value above a 256 MiB ceiling is rejected. Reader and writer share the options instance, applied before the message loop. - IPC-04: DrainEvents caps each reply at MaxDrainEventsPerReply (10_000) and treats max_events = 0 as that cap rather than 'drain the entire queue', so one diagnostic drain cannot pack a session-killing reply frame. - WRK-04: WorkerFrameWriter stamps the envelope Sequence at the actual point of writing (under the write lock) instead of at envelope creation, so the on-wire order and the stamped sequence always agree under concurrent producers. - WRK-07: the writer is now a cooperative priority scheduler — callers enqueue at Control or Event priority and the draining lock-holder writes all control frames before any event frame, so replies/faults/heartbeats jump ahead of an event backlog. Per-frame validation/size rejections fail only that frame; a stream write failure fails all queued frames. Tests: monotonic gap-free sequence under concurrency, control-before-event priority (gated stream), negotiated-max adoption, DrainEvents zero-bound. Worker builds x86 only — verified on windev. --- .../Ipc/WorkerFrameProtocolTests.cs | 156 +++++++++++++++ .../Ipc/WorkerPipeSessionTests.cs | 38 ++++ .../TestSupport/FakeRuntimeSession.cs | 10 + .../Ipc/WorkerFrameProtocolOptions.cs | 42 +++- .../Ipc/WorkerFrameWritePriority.cs | 17 ++ .../Ipc/WorkerFrameWriter.cs | 182 ++++++++++++++++-- .../Ipc/WorkerPipeSession.cs | 35 +++- 7 files changed, 450 insertions(+), 30 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWritePriority.cs diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerFrameProtocolTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerFrameProtocolTests.cs index 468a81b..8921762 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerFrameProtocolTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerFrameProtocolTests.cs @@ -1,5 +1,7 @@ +using System; using System.IO; using System.Linq; +using System.Threading; using System.Threading.Tasks; using ZB.MOM.WW.MxGateway.Contracts; using ZB.MOM.WW.MxGateway.Contracts.Proto; @@ -305,6 +307,101 @@ public sealed class WorkerFrameProtocolTests Nonce); } + /// + /// Verifies that under concurrent writers every frame receives a distinct, gap-free sequence in + /// strictly increasing on-wire order — the sequence is stamped by the writer at write time, so the + /// wire order and the stamped sequence always agree (WRK-04). + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task WriteAsync_UnderConcurrentCalls_StampsGapFreeMonotonicSequence() + { + const int frameCount = 50; + WorkerFrameProtocolOptions options = CreateOptions(); + using MemoryStream stream = new(); + WorkerFrameWriter writer = new(stream, options); + + await Task.WhenAll( + Enumerable.Range(0, frameCount).Select(_ => writer.WriteAsync(CreateEventEnvelope()))); + + stream.Position = 0; + WorkerFrameReader reader = new(stream, options); + ulong[] sequences = new ulong[frameCount]; + for (int index = 0; index < frameCount; index++) + { + sequences[index] = (await reader.ReadAsync()).Sequence; + } + + // On-wire order is strictly increasing 1..frameCount with no gaps or duplicates. + Assert.Equal(Enumerable.Range(1, frameCount).Select(value => (ulong)value), sequences); + } + + /// + /// Verifies that when a control frame and an event frame are both queued behind an in-progress + /// write, the draining lock-holder writes the control frame first even though the event was queued + /// earlier (WRK-07). + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task WriteAsync_WhenControlAndEventQueuedTogether_WritesControlFirst() + { + WorkerFrameProtocolOptions options = CreateOptions(); + using GatedWriteStream stream = new(); + WorkerFrameWriter writer = new(stream, options); + + // First write occupies the writer and blocks inside the stream, holding the write lock. + Task firstWrite = writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control); + await AwaitWithTimeoutAsync(stream.FirstWriteStarted); + + // Queue an event first, then a control frame, while the writer is blocked. Both wait for the lock. + Task eventWrite = writer.WriteAsync(CreateEventEnvelope(), WorkerFrameWritePriority.Event); + Task controlWrite = writer.WriteAsync(CreateGatewayHelloEnvelope(), WorkerFrameWritePriority.Control); + await Task.Delay(50); + + stream.ReleaseFirstWrite(); + await AwaitWithTimeoutAsync(Task.WhenAll(firstWrite, eventWrite, controlWrite)); + + stream.Position = 0; + WorkerFrameReader reader = new(stream, options); + WorkerEnvelope frame1 = await reader.ReadAsync(); + WorkerEnvelope frame2 = await reader.ReadAsync(); + WorkerEnvelope frame3 = await reader.ReadAsync(); + + Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, frame1.BodyCase); + // The control frame jumped ahead of the earlier-queued event. + Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, frame2.BodyCase); + Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerEvent, frame3.BodyCase); + } + + /// Verifies a zero negotiated frame maximum keeps the constructor default (IPC-02). + [Fact] + public void AdoptNegotiatedMaxMessageBytes_WithZero_KeepsDefault() + { + WorkerFrameProtocolOptions options = CreateOptions(); + int original = options.MaxMessageBytes; + options.AdoptNegotiatedMaxMessageBytes(0); + Assert.Equal(original, options.MaxMessageBytes); + } + + /// Verifies an in-range negotiated frame maximum is adopted (IPC-02). + [Fact] + public void AdoptNegotiatedMaxMessageBytes_WithInRangeValue_Adopts() + { + WorkerFrameProtocolOptions options = CreateOptions(); + options.AdoptNegotiatedMaxMessageBytes(4 * 1024 * 1024); + Assert.Equal(4 * 1024 * 1024, options.MaxMessageBytes); + } + + /// Verifies a negotiated frame maximum above the worker ceiling is rejected (IPC-02). + [Fact] + public void AdoptNegotiatedMaxMessageBytes_AboveCeiling_Throws() + { + WorkerFrameProtocolOptions options = CreateOptions(); + WorkerFrameProtocolException exception = Assert.Throws( + () => options.AdoptNegotiatedMaxMessageBytes((uint)WorkerFrameProtocolOptions.MaxNegotiableFrameBytes + 1)); + Assert.Equal(WorkerFrameProtocolErrorCode.InvalidConfiguration, exception.ErrorCode); + } + private static WorkerEnvelope CreateGatewayHelloEnvelope(ulong sequence = 1) { return new WorkerEnvelope @@ -321,4 +418,63 @@ public sealed class WorkerFrameProtocolTests }; } + // net48 has no Task.WaitAsync(TimeSpan); fail the test rather than hang if the writer misbehaves. + private static async Task AwaitWithTimeoutAsync(Task task) + { + Task completed = await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(5))); + if (completed != task) + { + throw new TimeoutException("Timed out waiting for the frame writer."); + } + + await task; + } + + private static WorkerEnvelope CreateEventEnvelope() + { + return new WorkerEnvelope + { + ProtocolVersion = GatewayContractInfo.WorkerProtocolVersion, + SessionId = SessionId, + WorkerEvent = new WorkerEvent + { + Event = new MxEvent { SessionId = SessionId }, + }, + }; + } + + // A MemoryStream whose first WriteAsync blocks until released, so a test can queue additional frames + // behind an in-progress write and observe the writer's priority ordering. + private sealed class GatedWriteStream : MemoryStream + { + private readonly SemaphoreSlim _release = new SemaphoreSlim(0); + private readonly TaskCompletionSource _firstWriteStarted = + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + private int _writeCount; + + public Task FirstWriteStarted => _firstWriteStarted.Task; + + public void ReleaseFirstWrite() => _release.Release(); + + public override async Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + if (Interlocked.Increment(ref _writeCount) == 1) + { + _firstWriteStarted.TrySetResult(true); + await _release.WaitAsync(cancellationToken); + } + + await base.WriteAsync(buffer, offset, count, cancellationToken); + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + _release.Dispose(); + } + + base.Dispose(disposing); + } + } } diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs index 117770a..3533efe 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs @@ -449,6 +449,44 @@ public sealed class WorkerPipeSessionTests await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } + /// + /// Verifies that a DrainEvents control command with max_events = 0 is bounded by the + /// worker rather than draining the entire queue into one reply frame: the session passes a + /// capped, non-zero maximum to the runtime session (IPC-04). + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task RunAsync_DrainEventsWithZeroMaxEvents_BoundsTheDrain() + { + using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(5)); + using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); + FakeRuntimeSession runtime = new() { SuppressDrainForBatchSize = 128 }; + WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime); + runtime.EnqueueEvent(CreateWorkerEvent(sequence: 11)); + Task runTask = session.RunAsync(cancellation.Token); + await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); + + await pipePair.GatewayWriter + .WriteAsync( + CreateControlCommandEnvelope( + "drain-cap-1", + MxCommandKind.DrainEvents, + command => command.DrainEvents = new DrainEventsCommand { MaxEvents = 0 }), + cancellation.Token); + + await ReadUntilAsync( + pipePair.GatewayReader, + WorkerEnvelope.BodyOneofCase.WorkerCommandReply, + cancellation.Token); + + // The client asked for "all" (0) but the worker must pass a bounded, non-zero cap so the reply + // frame cannot grow without limit. + Assert.NotNull(runtime.LastDrainMaxEvents); + Assert.NotEqual(0u, runtime.LastDrainMaxEvents!.Value); + + await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); + } + /// /// Verifies that ShutdownWorker returns its OK reply BEFORE the graceful /// shutdown runs and disposes the runtime session, and that the message diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs index c1002a9..776c708 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs @@ -125,6 +125,14 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession /// public uint? SuppressDrainForBatchSize { get; set; } + /// + /// Records the maxEvents argument of the most recent non-suppressed + /// call — i.e. the effective cap the session passed for an explicit + /// DrainEvents control command. Lets a test assert the worker bounds the drain (IPC-04) rather + /// than forwarding the client's raw max_events = 0. + /// + public uint? LastDrainMaxEvents { get; private set; } + /// public IReadOnlyList DrainEvents(uint maxEvents) { @@ -133,6 +141,8 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession return Array.Empty(); } + LastDrainMaxEvents = maxEvents; + lock (gate) { int drainCount = maxEvents == 0 diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameProtocolOptions.cs b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameProtocolOptions.cs index d8332f5..b9e016e 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameProtocolOptions.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameProtocolOptions.cs @@ -10,6 +10,14 @@ public sealed class WorkerFrameProtocolOptions /// Default maximum message size in bytes (16 MB). public const int DefaultMaxMessageBytes = 16 * 1024 * 1024; + /// + /// Upper ceiling the worker will accept for a gateway-negotiated frame maximum + /// (GatewayHello.max_frame_bytes, IPC-02). Matches the gateway's own configuration ceiling + /// so a nonsensical negotiated value is rejected at the handshake rather than driving an absurd + /// per-frame allocation. 256 MiB. + /// + public const int MaxNegotiableFrameBytes = 256 * 1024 * 1024; + /// Initializes a new instance of the WorkerFrameProtocolOptions class from WorkerOptions. /// Worker initialization options. public WorkerFrameProtocolOptions(WorkerOptions options) @@ -98,6 +106,36 @@ public sealed class WorkerFrameProtocolOptions /// Gets the nonce for startup validation. public string Nonce { get; } - /// Gets the maximum message size in bytes. - public int MaxMessageBytes { get; } + /// + /// Gets the maximum worker-frame message size in bytes. Initialized from the constructor and + /// then adopted once from the gateway-negotiated value during the startup handshake + /// (GatewayHello.max_frame_bytes, IPC-02) via , + /// before the message loop starts. Not mutated afterwards, so the single-threaded handshake write + /// is safe for the reader/writer that share this instance. + /// + public int MaxMessageBytes { get; private set; } + + /// + /// Adopts the gateway-negotiated frame maximum conveyed in GatewayHello.max_frame_bytes + /// (IPC-02). A value of 0 (an older gateway that never set the field) is ignored and the + /// constructor default is kept. A value above is rejected. + /// + /// The gateway-negotiated maximum, or 0 for "keep default". + internal void AdoptNegotiatedMaxMessageBytes(uint negotiatedMaxFrameBytes) + { + if (negotiatedMaxFrameBytes == 0) + { + return; + } + + if (negotiatedMaxFrameBytes > MaxNegotiableFrameBytes) + { + throw new WorkerFrameProtocolException( + WorkerFrameProtocolErrorCode.InvalidConfiguration, + $"GatewayHello negotiated frame maximum {negotiatedMaxFrameBytes} exceeds the worker ceiling " + + $"of {MaxNegotiableFrameBytes} bytes."); + } + + MaxMessageBytes = (int)negotiatedMaxFrameBytes; + } } diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWritePriority.cs b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWritePriority.cs new file mode 100644 index 0000000..46c60f8 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWritePriority.cs @@ -0,0 +1,17 @@ +namespace ZB.MOM.WW.MxGateway.Worker.Ipc; + +/// +/// Relative scheduling priority for an outbound worker frame. The single writer task drains all +/// pending frames before any frame, so a command reply, +/// fault, heartbeat, or shutdown acknowledgement is not delayed behind a backlog of queued events +/// (WRK-07). Priority only reorders the write; the frame sequence is stamped at actual write time, +/// so the on-wire order and the envelope Sequence always agree (WRK-04). +/// +public enum WorkerFrameWritePriority +{ + /// Control-plane frame (hello, ready, command reply, heartbeat, fault, shutdown ack). Written ahead of events. + Control = 0, + + /// Event frame. Written only when no control frame is pending. + Event = 1, +} diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWriter.cs b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWriter.cs index a80d548..5ae2e58 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWriter.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerFrameWriter.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.IO; using System.Threading; using System.Threading.Tasks; @@ -7,12 +8,40 @@ using ZB.MOM.WW.MxGateway.Contracts.Proto; namespace ZB.MOM.WW.MxGateway.Worker.Ipc; -/// Writes worker frames to a stream with length-prefixed protobuf serialization. +/// +/// Writes worker frames to a stream with length-prefixed protobuf serialization. Callers enqueue a +/// frame at a and then contend for a single write lock; whoever +/// holds the lock drains every queued frame, control frames first, so a reply, fault, or heartbeat is +/// never delayed behind an event backlog (WRK-07). The envelope Sequence is stamped by the +/// draining lock-holder at the moment of writing, so the on-wire order and the stamped sequence always +/// agree even under concurrent callers and priority reordering (WRK-04). +/// public sealed class WorkerFrameWriter { + private sealed class PendingFrame + { + public PendingFrame(WorkerEnvelope envelope) + { + Envelope = envelope; + Completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + } + + public WorkerEnvelope Envelope { get; } + + public TaskCompletionSource Completion { get; } + } + private readonly WorkerFrameProtocolOptions _options; - private readonly SemaphoreSlim _writeLock = new(1, 1); + private readonly SemaphoreSlim _writeLock = new SemaphoreSlim(1, 1); private readonly Stream _stream; + private readonly object _gate = new object(); + private readonly Queue _controlFrames = new Queue(); + private readonly Queue _eventFrames = new Queue(); + + // Only ever read/written by the current write-lock holder while draining, so no interlock is + // needed. Starts at 0 and is pre-incremented, so the first written frame carries sequence 1 + // (matching the previous behaviour). + private ulong _nextSequence; /// Initializes a new instance of the WorkerFrameWriter class. /// Stream to write frames to. @@ -25,12 +54,25 @@ public sealed class WorkerFrameWriter _options = options ?? throw new ArgumentNullException(nameof(options)); } - /// Writes a worker envelope frame to the stream with length prefix. + /// Writes a control-priority worker envelope frame to the stream with length prefix. /// Worker envelope to write. /// Token to cancel the asynchronous operation. - /// A task that represents the asynchronous operation. + /// A task that completes when the frame has been written and flushed. + public Task WriteAsync( + WorkerEnvelope envelope, + CancellationToken cancellationToken = default) + { + return WriteAsync(envelope, WorkerFrameWritePriority.Control, cancellationToken); + } + + /// Queues a worker envelope frame for writing at the given priority and drains the queue. + /// Worker envelope to write. + /// Scheduling priority; control frames are written ahead of event frames. + /// Token to cancel waiting for the write lock. + /// A task that completes when the frame has been written and flushed. public async Task WriteAsync( WorkerEnvelope envelope, + WorkerFrameWritePriority priority, CancellationToken cancellationToken = default) { if (envelope is null) @@ -38,8 +80,120 @@ public sealed class WorkerFrameWriter throw new ArgumentNullException(nameof(envelope)); } + PendingFrame frame = new PendingFrame(envelope); + lock (_gate) + { + if (priority == WorkerFrameWritePriority.Event) + { + _eventFrames.Enqueue(frame); + } + else + { + _controlFrames.Enqueue(frame); + } + } + + // Contend for the single writer: whoever wins drains every currently-queued frame in priority + // order, so this frame is written by this call or by a concurrent caller that got the lock + // first. Either way it completes via its own TaskCompletionSource. + await _writeLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + await DrainQueuedFramesAsync().ConfigureAwait(false); + } + finally + { + _writeLock.Release(); + } + + await frame.Completion.Task.ConfigureAwait(false); + } + + // Runs only under _writeLock. Drains control frames before event frames, stamping and writing each. + // The stream write itself is not cancellable: a frame is written atomically or fails, never left + // half-written on the pipe because a caller gave up waiting. + private async Task DrainQueuedFramesAsync() + { + while (true) + { + PendingFrame? frame = DequeueNext(); + if (frame is null) + { + return; + } + + try + { + await WriteFrameAsync(frame.Envelope).ConfigureAwait(false); + frame.Completion.TrySetResult(true); + } + catch (WorkerFrameProtocolException exception) when (IsPerFrameRejection(exception)) + { + // Validation, empty-payload, and oversized-frame errors are specific to this frame and + // do not damage the stream; fail only this frame and keep draining the rest. + frame.Completion.TrySetException(exception); + } + catch (Exception exception) + { + // A stream write/flush failure means the pipe is broken; fail this frame and every frame + // still queued so no caller awaits forever, then stop draining. + frame.Completion.TrySetException(exception); + FailAllQueued(exception); + return; + } + } + } + + private static bool IsPerFrameRejection(WorkerFrameProtocolException exception) + { + return exception.ErrorCode is WorkerFrameProtocolErrorCode.InvalidEnvelope + or WorkerFrameProtocolErrorCode.MessageTooLarge + or WorkerFrameProtocolErrorCode.ProtocolVersionMismatch + or WorkerFrameProtocolErrorCode.SessionMismatch; + } + + private PendingFrame? DequeueNext() + { + lock (_gate) + { + if (_controlFrames.Count > 0) + { + return _controlFrames.Dequeue(); + } + + if (_eventFrames.Count > 0) + { + return _eventFrames.Dequeue(); + } + + return null; + } + } + + private void FailAllQueued(Exception exception) + { + lock (_gate) + { + while (_controlFrames.Count > 0) + { + _controlFrames.Dequeue().Completion.TrySetException(exception); + } + + while (_eventFrames.Count > 0) + { + _eventFrames.Dequeue().Completion.TrySetException(exception); + } + } + } + + private async Task WriteFrameAsync(WorkerEnvelope envelope) + { WorkerEnvelopeValidator.Validate(envelope, _options); + // Stamp the sequence at the actual point of writing, under the write lock, so the wire order + // and the stamped sequence agree regardless of caller concurrency or priority (WRK-04). + envelope.Sequence = unchecked(++_nextSequence); + int payloadLength = envelope.CalculateSize(); if (payloadLength == 0) { @@ -55,26 +209,16 @@ public sealed class WorkerFrameWriter $"Worker envelope payload length {payloadLength} exceeds the configured maximum of {_options.MaxMessageBytes} bytes."); } - // Serialize once into a single buffer that carries the 4-byte - // length prefix followed by the payload, then issue one stream write. - // This avoids a second serialization pass (envelope.ToByteArray() - // would re-run CalculateSize internally), a separate prefix array, - // and a separate prefix write. + // Serialize once into a single buffer that carries the 4-byte length prefix followed by the + // payload, then issue one stream write. This avoids a second serialization pass, a separate + // prefix array, and a separate prefix write. int frameLength = sizeof(uint) + payloadLength; byte[] frame = new byte[frameLength]; WriteUInt32LittleEndian(frame, (uint)payloadLength); envelope.WriteTo(new Span(frame, sizeof(uint), payloadLength)); - await _writeLock.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - await _stream.WriteAsync(frame, 0, frameLength, cancellationToken).ConfigureAwait(false); - await _stream.FlushAsync(cancellationToken).ConfigureAwait(false); - } - finally - { - _writeLock.Release(); - } + await _stream.WriteAsync(frame, 0, frameLength, CancellationToken.None).ConfigureAwait(false); + await _stream.FlushAsync(CancellationToken.None).ConfigureAwait(false); } private static void WriteUInt32LittleEndian( diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs index b13b601..fea8594 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs @@ -18,6 +18,13 @@ public sealed class WorkerPipeSession private static readonly TimeSpan BackgroundTaskStopTimeout = TimeSpan.FromSeconds(1); private const uint EventDrainBatchSize = 128; + // Hard cap on how many events a single DrainEvents diagnostic reply may carry. DrainEvents is a + // non-streaming control command, so an unbounded drain (including the max_events = 0 "as many as + // available" request) could pack the whole queue into one session-killing reply frame (IPC-04). + // The gateway request validator rejects requests above its public ceiling; this worker-side cap is + // the backstop and defines the effective per-reply maximum. Kept in step with that public ceiling. + private const uint MaxDrainEventsPerReply = 10_000; + private readonly WorkerFrameProtocolOptions _options; private readonly Func _processIdProvider; private readonly Func _runtimeSessionFactory; @@ -28,7 +35,6 @@ public sealed class WorkerPipeSession private readonly object _commandTaskGate = new(); private readonly HashSet _activeCommandTasks = new(); private IWorkerRuntimeSession? _runtimeSession; - private long _nextSequence; // Mutated from the message loop, command tasks, the heartbeat loop and the // shutdown path; volatile so cross-thread reads observe the latest state @@ -225,6 +231,12 @@ public sealed class WorkerPipeSession WorkerFrameProtocolErrorCode.NonceMismatch, "GatewayHello nonce does not match the worker launch nonce."); } + + // Adopt the gateway-negotiated frame maximum so both ends frame to the same limit instead of + // matched compile-time defaults (IPC-02). Applied here, before the message loop, so every + // post-handshake frame is validated against the negotiated value; the reader and writer share + // this options instance. The hello frame itself was small and already read under the default. + _options.AdoptNegotiatedMaxMessageBytes(gatewayHello.MaxFrameBytes); } private Task WriteWorkerHelloAsync(CancellationToken cancellationToken) @@ -361,8 +373,11 @@ public sealed class WorkerPipeSession foreach (WorkerEvent workerEvent in events) { + // Events are the low-priority frame class: the writer holds them behind any pending + // control frame (reply, fault, heartbeat, shutdown ack) so those are not delayed + // behind an event backlog (WRK-07). await _writer - .WriteAsync(CreateEnvelope(workerEvent), cancellationToken) + .WriteAsync(CreateEnvelope(workerEvent), WorkerFrameWritePriority.Event, cancellationToken) .ConfigureAwait(false); } } @@ -527,7 +542,12 @@ public sealed class WorkerPipeSession IWorkerRuntimeSession? runtimeSession = _runtimeSession; if (runtimeSession is not null) { - uint maxEvents = command.DrainEvents?.MaxEvents ?? 0; + // Bound the diagnostic drain so max_events = 0 ("as many as available") or an over-large + // request cannot pack the whole queue into one session-killing reply frame (IPC-04). + uint requested = command.DrainEvents?.MaxEvents ?? 0; + uint maxEvents = requested == 0 || requested > MaxDrainEventsPerReply + ? MaxDrainEventsPerReply + : requested; foreach (WorkerEvent workerEvent in runtimeSession.DrainEvents(maxEvents)) { if (workerEvent.Event is not null) @@ -1004,19 +1024,16 @@ public sealed class WorkerPipeSession private WorkerEnvelope CreateBaseEnvelope() { + // Sequence is deliberately left unset here: the frame writer stamps it at the actual point of + // writing, under its single drain task, so the on-wire order and the stamped sequence agree + // even under concurrent producers and priority reordering (WRK-04). return new WorkerEnvelope { ProtocolVersion = _options.ProtocolVersion, SessionId = _options.SessionId, - Sequence = NextSequence(), }; } - private ulong NextSequence() - { - return unchecked((ulong)Interlocked.Increment(ref _nextSequence)); - } - private async Task InitializeMxAccessAsync(CancellationToken cancellationToken) { // RunAsync constructs the runtime session via _runtimeSessionFactory() From 309296f467858f08af58755ed6e1601015977167 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 09:16:31 -0400 Subject: [PATCH 15/19] test(ipc): refresh proto descriptor set + MaxMessageBytes default expectation Follow-up to the IPC-02 proto change and IPC-03 headroom default: - regenerate clients/proto/descriptors/mxaccessgw-client-v1.protoset with the pinned protoc 34.1 so it carries GatewayHello.max_frame_bytes (IPC-01 descriptor refresh; ClientProtoInputTests.Descriptor_ContainsEveryContractMessageAndField). - update GatewayOptionsTests design-default assertion to 16 MiB + 64 KiB reserve. --- .../descriptors/mxaccessgw-client-v1.protoset | Bin 116804 -> 117332 bytes .../Configuration/GatewayOptionsTests.cs | 3 ++- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/clients/proto/descriptors/mxaccessgw-client-v1.protoset b/clients/proto/descriptors/mxaccessgw-client-v1.protoset index bdb64fb1824570a7ec8501f9dd82fef6ba09d79f..b42b7921536a2bf6fc6e2aca15b3972122211cbb 100644 GIT binary patch delta 3442 zcmYLLTT>KA6rS#xVRvW6%c!V;+M)<3i+Dj$P*dUsQj!vkq$DLUyE`lkyF2UdEEh{1 zP2#O8RmnrDe8^KOl?R_q6_xk=g#3m4j^sPtJ>whSp6`6;JLmLt^Y_oGe||~5{FC)A z^1trv^ewV&i+%d=Tb}*c@1Az{RK2aGm4;XKm-5@8-#lyy*55wYj-T()uQ%z(om_z} zI+@@5oRn)D#`_QA4|Z_pW0n1DvkM=7YO+t!N7Jz^#!c5e&yRNgYKj?et6XhW#b(f0 z_ZvA`Le#zOO5hbmv;5Q_6y=(bH3vn_F9l)Q3;iN~5#g#YN&uVQwpeKdRZ$L`jwl4R zm2#=o0H>G>H=ABb zHRixWGxXsoSP`CB^%})oAt<6m(O>aem2g}Pi+-;rf=Utkv8iy?3*mVKKQ;Z3{6=D> z>{p5dqUw7!nC)pbec6X8x#CXD91=BUjU#Q8rzDCv zCu5`OxE!6t(WD%}81L43)b&_W&P3W+=1?3XW@ne|+H|dP;|tX_vBq`RCUrfbZ2&-S zBGNtp!JN=GfS@;_ZJ6Zn8b2ya0U+<1wgE&jJ$xXfUBmEc$RG-yk#`;TRUW_%KCH``ZG^gFguENwXJbUnp>wip%e8JAw^i2`SU1xLrJO}wPx3*T z0sy&5emq7In3JL}hR~Zlb5`debcp*gXqsQrA+j;8ZP?Prba$*t!=Juz zQQBy8t$W5-%0?R$?xhb%xi+#f!w*O~fCyWe(OUrob0)JV4nl9n-7oc0u65sdp!8B; z-A{K&y%g!q>gfW&-YoBw0ze?ncB$b4L2gzLH%0rOgAKWVAlR7GHh?IoZ2%!}PTNSk z)TZiJ}7PiQMPU?It)Z>Nw^S09+jD;yU(5B(JNk#l$((uzqMo%Lcy9XZ61 zR>YB`w;=0UC8MTxlLf0JR~=bamX@=sy9R*zDnBSq0fD*Np}Gcw-m308OS>uS>kR-l z%Gw4H#q{M2gtW50-LiBmuIaOs19|?iqodnlj5|g^`tneRk+al*i;0l-q9?lRaG+TrUgP?<=AmK@+|qP@+&&e#)zu= z^Fjz|)n0W$0U@nAIz}V64cDq0PvyvM8f4wxOLDeB6MrHv;m9V0*b{YY5W;|-96hF) z2sKon2+>#@T)ugPKy0YafB>56UMB!zGf|YWL=grs2+=$?)Qky%xN-87CSq`#>cAsJXEoYPnrj1dQ=M@H zQ?9wCzF7ibZt)IXAO+&q5&5hjmUhkUgh+Sdc0x>pxGjlf%6?|NXKEQV$7eRVk1U(f zHlC}`mjGhV)z?4>63^8K3xv=MJ|t}bNHh>55MK=DWS@)%9`UO>L}?Kpi4n!d=qOR? z2>-0SWYBEK?C_5rlW-u0_y_JKgQ_CqiGRiLgeoo;Lhwus!F|Q>YAP41}Cyo5h6yvK@*%@B+|VKsnwgB2Ig)oaGD zX)cySa4d%4zK+;9UDG6-ajiEDr$`CoH6kct2=W_-bELRHwg8iCnXF-0qb30=HP5lp)j&;TW delta 2909 zcmX|DO;Z(D5S`mI^WMz7XmnKk;E5>Wx9~uPhXN@fC6<_ylvpC-O9==mMf@NoBI@|D zOcq(B%C)Oh)=3qMZ1V?FmA{bRk(_gH-&vmO)90S5psM%Eeh#{{D`K73qiG0@%Xtma#<<#u zt&Q=zmo!Kl?>}d4RPrLa5?pods6gUMvfJuaXk$WlTRi{_WkQDn1mZ;crH?^xBHv~8 zs(ExR_{Paq!Mc{TS-C3ZCbhW$us10OC{RRzU{1EX?gBw?QoCE_{I9DG0NA*$Hh>sY z8$d|At~Qc9x)FTqY$Raaz=h?VNoZqA?+gHJ6jRz=KrpAe-K7D7-jrUNG>>iux13%I z)=m9VQtD013EK$(=uPVf00iQ6N0*O5Zu;19+oddzZU^5vxgrDYcCy#{%cwV_UmgI; zozX832*erv@_?W>qhDUeF5MCLbpgP}9oc3N24ZZx+eJV~yVK@R!-rSQ%CIs8d9!+H zGNI;debC2{H+%7tx&*i@uCh$HZ|>?)fRJ`KbsH0i%)>`CEy}|=Z5H+q(&i*7^Z5GPNjmiaj%umQ8L`dXGDO^eZ@ zW&OGkLfUe-TTnnqTOMq3|4AI=(Zk>e+wz#5d>HSbUL3IHkK7U_K+irB)4KwNQ!0RcQN6)~Fv&@Ljz7%U~>aMig-h|O9p^Fk0;k96tT=+zUk z4>O4D&J(w=h(LU@cb^i`ou|^x?jXc|Je5v+{|Ui-+I3W!=-e}R;}POzeO4A%0r8o; z7?i(l WzZJ%!BR-+z8pt2EZoN1EUHu=0M$8!i diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsTests.cs index 26e80e7..5a114af 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Configuration/GatewayOptionsTests.cs @@ -35,7 +35,8 @@ public sealed class GatewayOptionsTests Assert.Equal(10, options.Worker.ShutdownTimeoutSeconds); Assert.Equal(5, options.Worker.HeartbeatIntervalSeconds); Assert.Equal(15, options.Worker.HeartbeatGraceSeconds); - Assert.Equal(16 * 1024 * 1024, options.Worker.MaxMessageBytes); + // 16 MiB public gRPC cap plus the 64 KiB worker-frame envelope reserve (IPC-03 headroom). + Assert.Equal((16 * 1024 * 1024) + (64 * 1024), options.Worker.MaxMessageBytes); Assert.Equal(30, options.Sessions.DefaultCommandTimeoutSeconds); Assert.Equal(64, options.Sessions.MaxSessions); From bfbfcdd1125cdf9c210a03b4422b1fdb82b112da Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 09:21:48 -0400 Subject: [PATCH 16/19] docs(archreview-p1): mark IPC-02/03/04 + WRK-04/07 Done (Wave 3, windev-verified) --- archreview/remediation/00-tracking.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/archreview/remediation/00-tracking.md b/archreview/remediation/00-tracking.md index 5c0d61a..3ad4355 100644 --- a/archreview/remediation/00-tracking.md +++ b/archreview/remediation/00-tracking.md @@ -90,10 +90,10 @@ Full design + implementation for each row lives in the linked domain doc under i | WRK-01 | High | P0 | M | — | Done | Long `ReadBulk` self-faults as `StaHung`; all replies then dropped | | WRK-02 | Medium | — | M | — | Not started | STA thread death after startup is silent; future work hangs forever | | WRK-03 | Medium | — | S | WRK-02 | Not started | Commands after shutdown starts are dropped with no reply | -| WRK-04 | Medium | — | M | — | Not started | Envelope `sequence` can appear out of order on the wire | +| WRK-04 | Medium | — | M | — | Done | Envelope `sequence` can appear out of order on the wire | | WRK-05 | Medium | — | M | — | Not started | One transient alarm-poll failure kills the whole session | | WRK-06 | Medium | P2 | S | — | Not started | `MXSTATUS_PROXY` conversion reflects per field, per event | -| WRK-07 | Medium | P1 | M | WRK-04 | Not started | Documented outbound write priority not implemented | +| WRK-07 | Medium | P1 | M | WRK-04 | Done | Documented outbound write priority not implemented | | WRK-08 | Low | — | S | — | Not started | Residual event queue discarded at graceful shutdown, undocumented | | WRK-09 | Low | — | S | — | Not started | No `AppDomain.UnhandledException` hook | | WRK-10 | Low | — | S | — | Not started | Top-level catch logs exception type but never message | @@ -113,9 +113,9 @@ Full design + implementation for each row lives in the linked domain doc under i | ID | Sev | Tier | Eff | Dep | Status | Title | |---|---|:-:|:-:|---|---|---| | IPC-01 | High | P1 | M | — | Done | Published client descriptor set 7 weeks stale; nothing enforces freshness | -| IPC-02 | Medium | P1 | M | IPC-03 | Not started | Worker max frame size hard-coded, cannot follow gateway config | -| IPC-03 | Medium | P1 | M | IPC-02 | Not started | gRPC max == pipe max with zero headroom; oversized write faults whole session | -| IPC-04 | Medium | P1 | S | IPC-03 | Not started | `DrainEvents max_events=0` packs entire queue into one reply frame | +| IPC-02 | Medium | P1 | M | IPC-03 | Done | Worker max frame size hard-coded, cannot follow gateway config | +| IPC-03 | Medium | P1 | M | IPC-02 | Done | gRPC max == pipe max with zero headroom; oversized write faults whole session | +| IPC-04 | Medium | P1 | S | IPC-03 | Done | `DrainEvents max_events=0` packs entire queue into one reply frame | | IPC-05 | Medium | P2 | S | — | Not started | Redundant deep copies on command and event hot paths | | IPC-06 | Medium | P2 | S | — | Not started | `gateway.md` Worker Envelope sketch no longer matches the contract | | IPC-07 | Medium | P2 | S | — | Not started | `docs/Grpc.md` says six RPCs; there are seven | @@ -257,6 +257,7 @@ Findings the review flagged as one coordinated design pass — sequence them tog | 2026-07-09 | P0 tier executed via parallel agents. GWC-01/02/03, TST-02, TST-12, CLI-01, CLI-03 → `Done` (NonWindows build clean; gateway 135/135 targeted tests pass, Go `go test ./...` + `-race` pass, Rust `test`/`clippy -D warnings` pass). | | 2026-07-09 | Windows full-suite temp-file-lock flakiness (discovered via TST-08) → **fixed** (commit `11a716a`). Two shared-state parallel collisions that macOS never surfaces: (1) self-signed cert generation — `SelfSignedCertificateProvider` now stages the PFX in a unique `..tmp` (was a fixed name that concurrent/interrupted writers collided on); `TestHostEnvironmentInitializer` defaults the cert path to a per-process temp dir (so the suite never writes shared `ProgramData` or fights the deployed service); `GatewayTlsBootstrapTests` moved to a `DisableParallelization` collection so its process-global env-var mutation can't bleed into parallel tests. (2) `AuthStoreHealthCheckTests` — reuse the existing `TempDatabaseDirectory` helper (`SqliteConnection.ClearAllPools()` before delete) so the Microsoft.Data.Sqlite pool releases the temp `.db` handle. **windev-verified:** 3 consecutive full-suite runs all **793 passed / 2 failed, 0 new orphans** (was flaky 2–4 fails, count-varying). The 2 stable remaining failures are pre-existing windev-environmental (`SelfSignedCertificateProviderTests.GenerateCertificate_HasExpectedSansEkuAndValidity` machine-name/FQDN SAN assertion; `EventStreamServiceTests…TracksAggregateQueueDepth` timing) — both pass on macOS, unrelated to this change. | | 2026-07-09 | P1 TST-08 (orphaned testhost) → `Done` by evidence: the claimed leak does **not** reproduce. Full `ZB.MOM.WW.MxGateway.Tests` suite exits cleanly on macOS (0 survivors) and on the Windows dev box (windev, `origin/main` worktree: 0 new `testhost` and 0 new `ZB.MOM.WW.MxGateway.Worker` processes after a full run, isolated by StartTime). Static audit confirms the prime-suspect fixtures already dispose deterministically (WorkerClient cancels its `_stopCts` + awaits read/write/heartbeat tasks with a 5 s timeout; GatewaySession disposes the client; E2E/harness fixtures use `await using`). CLAUDE.md "Source Update Workflow" updated to drop the stale "leaves orphaned testhost processes" rationale (filtered runs kept as a speed guideline). **Discovered separately (NOT TST-08, still open):** Windows-only temp-file-lock flakiness in ~4 full-host test classes — `AuthStoreHealthCheckTests` (Microsoft.Data.Sqlite connection-pool holds the temp `.db` handle after `await using`, so the finally `File.Delete` throws "used by another process"), `GatewayTlsBootstrapTests`/`DashboardCookieOptionsTests`/`DashboardHubsRegistrationTests` (a `gw.pfx.tmp` handle from `SelfSignedCertificateProvider`'s fixed-name `path+".tmp"` atomic-write racing teardown). Count varies run-to-run (2–4 fails); macOS never sees it (Unix allows deleting open files). Fix = `SqliteConnection.ClearAllPools()` before temp delete + a unique/guarded `.tmp` name; verify on windev. | +| 2026-07-09 | P1 Wave 3 (size/backpressure topology + write ordering). IPC-02/03/04 + WRK-04/07 → `Done`. **Size negotiation (IPC-02):** added `GatewayHello.max_frame_bytes` (regen `Generated/` + `clients/proto` descriptor refresh with pinned protoc 34.1); the gateway sends its negotiated worker-frame max and the worker adopts it (`WorkerFrameProtocolOptions.AdoptNegotiatedMaxMessageBytes`, 0 = keep default, >256 MiB rejected) instead of a hard-coded default. **Headroom (IPC-03):** the pipe frame max now sits `EnvelopeOverheadReserveBytes` (64 KiB) above the public gRPC cap (default `Worker.MaxMessageBytes` 16 MiB→16 MiB+64 KiB), cross-validated at startup; `WorkerClient` pre-checks command envelope size and fails only the offending correlation (`ResourceExhausted`) instead of `SetFaulted`ing the session. **Drain bound (IPC-04):** gateway request validator rejects `DrainEvents max_events` above 10 000; the worker caps each reply at `MaxDrainEventsPerReply` (10 000) and treats `max_events = 0` as that cap, not "drain all". **Sequence (WRK-04):** `WorkerFrameWriter` stamps the envelope `Sequence` at the point of writing under the write lock, so wire order and stamped sequence always agree under concurrent producers. **Priority (WRK-07):** the worker writer is now a cooperative priority scheduler — control frames (reply/fault/heartbeat/shutdown-ack) drain ahead of event frames; per-frame validation/size rejections fail only that frame, a stream failure fails all queued. Docs same-change (GatewayConfiguration, WorkerFrameProtocol, gateway.md). **Verified:** macOS NonWindows build clean + validator/grpc tests green; **windev** x86 worker builds clean, `Worker.Tests` 352 passed / 0 failed / 11 skipped (incl. new monotonic-sequence, control-before-event priority, negotiated-max, drain-bound tests), gateway `Tests` 799 passed / 3 failed — all 3 pre-existing windev-environmental (SelfSigned SAN + 2 `EventStreamServiceTests` timing, both pass in isolation). Commits `c8b3a22` (gateway half), `ebe6aea` (worker half), `309296f` (descriptor + default-expectation refresh). GWC-04 (event-channel decoupling) is the remaining Wave 3 item. | | 2026-07-09 | P1 S-misc (dashboard/observability hardening). SEC-02/12/20 → `Done`. SEC-02: `DashboardAuthorizationHandler` restricts the loopback + `Authentication:Mode=Disabled` bypasses to read-only (they satisfy a Viewer-bearing requirement but never `AdminOnly`), closing the policy-layer gap where anonymous localhost was authorized for Admin surfaces. SEC-12: `DashboardSessionAdminService` now emits canonical `AuditEvent`s (`dashboard-close-session`/`dashboard-kill-worker`, category `SessionAdmin`) through `IAuditWriter` on Success/Failure/Denied, so Close/Kill land durable audit rows. SEC-20: dropped the unbounded `session_id` tag from the exported `mxgateway.heartbeats.failed` counter. Docs updated same-change (CLAUDE.md, GatewayDashboardDesign.md, Metrics.md). Server build clean (0 warnings); targeted classes 30/30 pass; broader Dashboard+Security+GatewayApplication+Metrics sweep 295/295 pass. | | 2026-07-09 | P1 Wave 2b (security authz+hub). SEC-05/07/08/11 → `Done` (hub-token lifetime; QueryActiveAlarms scope arm; gateway-side verification cache + last-used coalescing; login rate limit + per-peer gRPC failure limiter). Full-suite checkpoint caught + fixed regressions the earlier narrow SEC-01/04/06 filter missed: cross-platform path-rooting, an `IHostEnvironment` fallback for minimal DI containers, a test-assembly `ASPNETCORE_ENVIRONMENT=Development` default, and a platform-correct default-path assertion. Suite: 747 passed / 42 failed, all 42 pre-existing macOS named-pipe-harness env failures. | | 2026-07-09 | P1 Wave 2a (security). SEC-01/04/06 → `Done` (config path-rooting + production validator guards; Server build clean, validator+hygiene tests 53/53). SEC-10 → `Done`: the shared `ZB.MOM.WW.Auth.ApiKeys` gained optional `ExpiresUtc` (expired keys rejected, auth DB auto-migrates to schema v3) via a concurrent HistorianGateway-remediation session's "G-2"; this repo consumes it by bumping the four `Auth.*` refs 0.1.2→0.1.4 (commit 197731a). Remaining SEC-10 polish (`apikey create --expires` + dashboard staleness badge) tracked as a small follow-up. | From 32d6b4891087e21d39dabf833e68082056ef1773 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 09:28:13 -0400 Subject: [PATCH 17/19] feat(gateway): decouple worker event enqueue from the read loop (GWC-04) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The read loop awaited EnqueueWorkerEventAsync inline, which blocks in the bounded event channel's timed WriteAsync (up to EventChannelFullModeTimeout, default 5s) when the channel is full with no/slow consumer. A WorkerCommandReply or heartbeat queued behind an event frame was then stalled, so an in-flight InvokeAsync could hit CommandTimeout even though the worker replied in time. Mirror the existing outbound WriteLoopAsync: add an unbounded event staging channel and a dedicated EventWriteLoopAsync. DispatchEnvelope is now fully synchronous — the WorkerEvent branch hands the event off with a non-blocking TryWrite and the read loop never awaits. The event write loop owns the timed WriteAsync into the bounded channel and the sustained-overflow ProtocolViolation fault (unchanged contract). Registered in WaitForBackgroundTasks + completed on close/fault/dispose. Test: reply arriving after events with a full, consumer-less event channel is dispatched promptly (no CommandTimeout) — pipe-harness, verified on windev. Docs: GatewayProcessDesign read/write/event-loop section. --- docs/GatewayProcessDesign.md | 20 ++++- .../Workers/WorkerClient.cs | 82 ++++++++++++++++--- .../Gateway/Workers/WorkerClientTests.cs | 45 ++++++++++ 3 files changed, 132 insertions(+), 15 deletions(-) diff --git a/docs/GatewayProcessDesign.md b/docs/GatewayProcessDesign.md index ae5b5ca..3437b38 100644 --- a/docs/GatewayProcessDesign.md +++ b/docs/GatewayProcessDesign.md @@ -481,7 +481,9 @@ Internally it owns: - pipe stream, - read loop, - write loop, +- event write loop, - outbound command/control channel serialized by the write loop, +- unbounded event staging channel drained by the event write loop, - bounded inbound event channel, - pending command dictionary keyed by correlation id, - heartbeat monitor, @@ -499,15 +501,29 @@ The read loop: 1. Reads one frame. 2. Parses `WorkerEnvelope`. 3. Validates protocol fields. -4. Dispatches by body type: +4. Dispatches by body type, **synchronously and without blocking**: - `WorkerCommandReply`: completes pending command. - - `WorkerEvent`: enqueues event. + - `WorkerEvent`: hands the event to the event write loop via a non-blocking staging write. - `WorkerHeartbeat`: updates heartbeat timestamp. - `WorkerFault`: faults session. 5. Stops when pipe closes or cancellation is requested. If the pipe closes while the session is not closing, fault the session. +The read loop never awaits event enqueue. Events are staged to the event write +loop with a non-blocking write, so a full inbound event channel (a slow or +absent `StreamEvents` consumer) cannot stall the read loop behind an event and +delay a command reply or heartbeat (GWC-04). The bounded-channel backpressure +window (`EventChannelFullModeTimeout`) and the sustained-overflow fault are +applied by the event write loop, not the read loop. + +### Event write loop + +The event write loop drains the staging channel and performs the timed write +into the bounded inbound event channel. When the inbound channel stays full past +`EventChannelFullModeTimeout` it faults the session (`ProtocolViolation`) — the +same overflow contract as before, moved off the read loop. + ### Write loop The write loop serializes all writes to the pipe. No other code should write to diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs index b8e603a..342ae6d 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs @@ -24,6 +24,13 @@ public sealed class WorkerClient : IWorkerClient private readonly WorkerFrameWriter _writer; private readonly Channel _outboundEnvelopes; private readonly Channel _events; + + // Staging hand-off between the read loop and the dedicated event writer. The read loop writes + // here with a non-blocking TryWrite so a full consumer channel (_events) can never stall the read + // loop behind an event — replies and heartbeats keep flowing (GWC-04). Unbounded, but only fills + // during the bounded EventChannelFullModeTimeout window before EventWriteLoopAsync faults on a + // sustained backlog, after which the read loop stops. + private readonly Channel _eventStaging; private readonly ConcurrentDictionary _pendingCommands = new(StringComparer.Ordinal); private readonly SemaphoreSlim _pendingCommandSlots; private readonly CancellationTokenSource _stopCts = new(); @@ -35,6 +42,7 @@ public sealed class WorkerClient : IWorkerClient private int _eventsReaderClaimed; private Task? _readLoopTask; private Task? _writeLoopTask; + private Task? _eventWriteLoopTask; private Task? _heartbeatLoopTask; private bool _workerStartRecorded; private bool _disposed; @@ -82,6 +90,14 @@ public sealed class WorkerClient : IWorkerClient FullMode = BoundedChannelFullMode.Wait, AllowSynchronousContinuations = false, }); + _eventStaging = Channel.CreateUnbounded( + new UnboundedChannelOptions + { + // The read loop is the only writer; EventWriteLoopAsync is the only reader. + SingleReader = true, + SingleWriter = true, + AllowSynchronousContinuations = false, + }); _lastHeartbeatAt = _timeProvider.GetUtcNow(); } @@ -144,6 +160,7 @@ public sealed class WorkerClient : IWorkerClient MarkReady(readyEnvelope.WorkerReady); _readLoopTask = Task.Run(ReadLoopAsync); + _eventWriteLoopTask = Task.Run(EventWriteLoopAsync); _heartbeatLoopTask = Task.Run(HeartbeatLoopAsync); } @@ -344,6 +361,7 @@ public sealed class WorkerClient : IWorkerClient KillOwnedProcess("Dispose"); _stopCts.Cancel(); _outboundEnvelopes.Writer.TryComplete(); + _eventStaging.Writer.TryComplete(); _events.Writer.TryComplete(); CompletePendingCommands( new WorkerClientException( @@ -398,7 +416,7 @@ public sealed class WorkerClient : IWorkerClient while (!_stopCts.IsCancellationRequested) { WorkerEnvelope envelope = await _reader.ReadAsync(_stopCts.Token).ConfigureAwait(false); - await DispatchEnvelopeAsync(envelope, _stopCts.Token).ConfigureAwait(false); + DispatchEnvelope(envelope); } } catch (OperationCanceledException) when (_stopCts.IsCancellationRequested || IsTerminalState()) @@ -497,12 +515,14 @@ public sealed class WorkerClient : IWorkerClient return true; } - /// Routes received envelope to appropriate handler. + /// + /// Routes a received envelope to its handler. Every branch dispatches synchronously and + /// immediately — the event branch only stages the event for the dedicated writer — so a full + /// event channel can never delay a command reply, heartbeat, fault, or shutdown ack behind an + /// event backlog (GWC-04). + /// /// The envelope to dispatch. - /// Cancellation token. - private async Task DispatchEnvelopeAsync( - WorkerEnvelope envelope, - CancellationToken cancellationToken) + private void DispatchEnvelope(WorkerEnvelope envelope) { switch (envelope.BodyCase) { @@ -510,7 +530,7 @@ public sealed class WorkerClient : IWorkerClient CompleteCommand(envelope); break; case WorkerEnvelope.BodyOneofCase.WorkerEvent: - await EnqueueWorkerEventAsync(envelope.WorkerEvent, cancellationToken).ConfigureAwait(false); + StageWorkerEvent(envelope.WorkerEvent); break; case WorkerEnvelope.BodyOneofCase.WorkerHeartbeat: MarkHeartbeat(envelope.WorkerHeartbeat); @@ -533,6 +553,45 @@ public sealed class WorkerClient : IWorkerClient } } + /// + /// Hands a received worker event to the dedicated event writer without blocking the read loop. + /// The staging channel is unbounded and this is the only writer, so TryWrite always + /// succeeds unless the channel has been completed during shutdown — in which case the event is + /// safely dropped because the client is closing. Backpressure and the sustained-overflow fault + /// are applied by against the bounded consumer channel, + /// off the read loop (GWC-04). + /// + /// The event received from the worker. + private void StageWorkerEvent(WorkerEvent workerEvent) + { + if (workerEvent.Event is not null) + { + _metrics?.EventReceived(SessionId, workerEvent.Event.Family.ToString()); + } + + _eventStaging.Writer.TryWrite(workerEvent); + } + + /// + /// Drains staged worker events and applies the bounded-channel backpressure (and + /// sustained-overflow fault) on a dedicated task, so the timed write + /// never runs on the read loop (GWC-04). Mirrors for events. + /// + private async Task EventWriteLoopAsync() + { + try + { + await foreach (WorkerEvent workerEvent in + _eventStaging.Reader.ReadAllAsync(_stopCts.Token).ConfigureAwait(false)) + { + await EnqueueWorkerEventAsync(workerEvent, _stopCts.Token).ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (_stopCts.IsCancellationRequested || IsTerminalState()) + { + } + } + /// /// Enqueues a worker event for client consumption. The channel is /// configured with @@ -552,11 +611,6 @@ public sealed class WorkerClient : IWorkerClient WorkerEvent workerEvent, CancellationToken cancellationToken) { - if (workerEvent.Event is not null) - { - _metrics?.EventReceived(SessionId, workerEvent.Event.Family.ToString()); - } - if (_events.Writer.TryWrite(workerEvent)) { int queueDepth = Interlocked.Increment(ref _eventQueueDepth); @@ -747,6 +801,7 @@ public sealed class WorkerClient : IWorkerClient _stopCts.Cancel(); _outboundEnvelopes.Writer.TryComplete(); + _eventStaging.Writer.TryComplete(); _events.Writer.TryComplete(); CompletePendingCommands( new WorkerClientException( @@ -780,6 +835,7 @@ public sealed class WorkerClient : IWorkerClient _stopCts.Cancel(); _outboundEnvelopes.Writer.TryComplete(fault); + _eventStaging.Writer.TryComplete(fault); _events.Writer.TryComplete(fault); KillOwnedProcess(errorCode.ToString()); CompletePendingCommands(fault); @@ -1005,7 +1061,7 @@ public sealed class WorkerClient : IWorkerClient /// Cancellation token. private async Task WaitForBackgroundTasksAsync(CancellationToken cancellationToken) { - Task[] tasks = new[] { _readLoopTask, _writeLoopTask, _heartbeatLoopTask } + Task[] tasks = new[] { _readLoopTask, _writeLoopTask, _eventWriteLoopTask, _heartbeatLoopTask } .Where(task => task is not null) .Cast() .ToArray(); diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs index b3d38a5..8300759 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientTests.cs @@ -218,6 +218,51 @@ public sealed class WorkerClientTests Assert.Equal(WorkerClientState.Faulted, client.State); } + /// + /// Verifies that a command reply arriving on the pipe after events is dispatched promptly even + /// when the event channel is full and has no consumer — event enqueue is decoupled from the read + /// loop, so a blocked event writer cannot delay a reply (GWC-04). The event full-mode timeout is + /// set far above the command timeout: without the decoupling the read loop would block behind the + /// full event channel and the in-flight InvokeAsync would hit CommandTimeout. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task ReadLoop_WhenEventChannelFull_DispatchesCommandReplyWithoutBlocking() + { + await using PipePair pipePair = await PipePair.CreateAsync(); + await using WorkerClient client = CreateClient( + pipePair, + new WorkerClientOptions + { + EventChannelCapacity = 1, + // Much larger than TestTimeout: the read loop must not block for this long behind the + // full event channel while a reply waits. + EventChannelFullModeTimeout = TimeSpan.FromSeconds(30), + HeartbeatGrace = TimeSpan.FromSeconds(60), + HeartbeatCheckInterval = TimeSpan.FromSeconds(60), + }); + await CompleteHandshakeAsync(client, pipePair); + + // No StreamEvents consumer is attached, so the capacity-1 event channel fills and the event + // writer blocks on the second event's timed WriteAsync. + await pipePair.WorkerWriter.WriteAsync( + CreateEventEnvelope(sequence: 11, MxEventFamily.OnDataChange)); + await pipePair.WorkerWriter.WriteAsync( + CreateEventEnvelope(sequence: 12, MxEventFamily.OnDataChange)); + + Task invokeTask = client.InvokeAsync( + CreateCommand(MxCommandKind.Ping), + TestTimeout, + CancellationToken.None); + WorkerEnvelope commandEnvelope = await pipePair.WorkerReader.ReadAsync().AsTask().WaitAsync(TestTimeout); + await pipePair.WorkerWriter.WriteAsync( + CreateCommandReplyEnvelope(commandEnvelope.CorrelationId, MxCommandKind.Ping)); + + WorkerCommandReply reply = await invokeTask.WaitAsync(TestTimeout); + Assert.Equal(MxCommandKind.Ping, reply.Reply.Kind); + Assert.Equal(WorkerClientState.Ready, client.State); + } + /// /// Verifies that when the client faults it kills the owned worker process. /// The assertion waits on , which From de57d834b11d7eb83a88c0d1817df808e8dbd9d4 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 09:28:39 -0400 Subject: [PATCH 18/19] =?UTF-8?q?docs(archreview-p1):=20mark=20GWC-04=20Do?= =?UTF-8?q?ne=20=E2=80=94=20Wave=203=20complete?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- archreview/remediation/00-tracking.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/archreview/remediation/00-tracking.md b/archreview/remediation/00-tracking.md index 3ad4355..7f729e3 100644 --- a/archreview/remediation/00-tracking.md +++ b/archreview/remediation/00-tracking.md @@ -62,7 +62,7 @@ Full design + implementation for each row lives in the linked domain doc under i | GWC-01 | Critical | P0 | M | — | Done | Alarm monitor and distributor pump both drain the single worker event channel | | GWC-02 | High | P0 | M | — | Done | Faulted sessions are never swept | | GWC-03 | High | P0 | S | — | Done | Documented sparse-array max-length bound is unimplemented | -| GWC-04 | Medium | P1 | M | — | Not started | Full event channel stalls the worker read loop behind command replies | +| GWC-04 | Medium | P1 | M | — | Done | Full event channel stalls the worker read loop behind command replies | | GWC-05 | Medium | — | S | — | Not started | Worker pipe created with no ACL / no CurrentUserOnly | | GWC-06 | Medium | P2 | S | GWC-07,08 | Not started | Stopwatch allocated per streamed event | | GWC-07 | Medium | P2 | S | GWC-06,08 | Not started | Every mapped event is deep-cloned | @@ -257,6 +257,7 @@ Findings the review flagged as one coordinated design pass — sequence them tog | 2026-07-09 | P0 tier executed via parallel agents. GWC-01/02/03, TST-02, TST-12, CLI-01, CLI-03 → `Done` (NonWindows build clean; gateway 135/135 targeted tests pass, Go `go test ./...` + `-race` pass, Rust `test`/`clippy -D warnings` pass). | | 2026-07-09 | Windows full-suite temp-file-lock flakiness (discovered via TST-08) → **fixed** (commit `11a716a`). Two shared-state parallel collisions that macOS never surfaces: (1) self-signed cert generation — `SelfSignedCertificateProvider` now stages the PFX in a unique `..tmp` (was a fixed name that concurrent/interrupted writers collided on); `TestHostEnvironmentInitializer` defaults the cert path to a per-process temp dir (so the suite never writes shared `ProgramData` or fights the deployed service); `GatewayTlsBootstrapTests` moved to a `DisableParallelization` collection so its process-global env-var mutation can't bleed into parallel tests. (2) `AuthStoreHealthCheckTests` — reuse the existing `TempDatabaseDirectory` helper (`SqliteConnection.ClearAllPools()` before delete) so the Microsoft.Data.Sqlite pool releases the temp `.db` handle. **windev-verified:** 3 consecutive full-suite runs all **793 passed / 2 failed, 0 new orphans** (was flaky 2–4 fails, count-varying). The 2 stable remaining failures are pre-existing windev-environmental (`SelfSignedCertificateProviderTests.GenerateCertificate_HasExpectedSansEkuAndValidity` machine-name/FQDN SAN assertion; `EventStreamServiceTests…TracksAggregateQueueDepth` timing) — both pass on macOS, unrelated to this change. | | 2026-07-09 | P1 TST-08 (orphaned testhost) → `Done` by evidence: the claimed leak does **not** reproduce. Full `ZB.MOM.WW.MxGateway.Tests` suite exits cleanly on macOS (0 survivors) and on the Windows dev box (windev, `origin/main` worktree: 0 new `testhost` and 0 new `ZB.MOM.WW.MxGateway.Worker` processes after a full run, isolated by StartTime). Static audit confirms the prime-suspect fixtures already dispose deterministically (WorkerClient cancels its `_stopCts` + awaits read/write/heartbeat tasks with a 5 s timeout; GatewaySession disposes the client; E2E/harness fixtures use `await using`). CLAUDE.md "Source Update Workflow" updated to drop the stale "leaves orphaned testhost processes" rationale (filtered runs kept as a speed guideline). **Discovered separately (NOT TST-08, still open):** Windows-only temp-file-lock flakiness in ~4 full-host test classes — `AuthStoreHealthCheckTests` (Microsoft.Data.Sqlite connection-pool holds the temp `.db` handle after `await using`, so the finally `File.Delete` throws "used by another process"), `GatewayTlsBootstrapTests`/`DashboardCookieOptionsTests`/`DashboardHubsRegistrationTests` (a `gw.pfx.tmp` handle from `SelfSignedCertificateProvider`'s fixed-name `path+".tmp"` atomic-write racing teardown). Count varies run-to-run (2–4 fails); macOS never sees it (Unix allows deleting open files). Fix = `SqliteConnection.ClearAllPools()` before temp delete + a unique/guarded `.tmp` name; verify on windev. | +| 2026-07-09 | P1 Wave 3 (event-channel decoupling). GWC-04 → `Done`. `WorkerClient`'s read loop awaited `EnqueueWorkerEventAsync` inline, which blocks in the bounded event channel's timed `WriteAsync` (≤ `EventChannelFullModeTimeout`, 5 s) when the channel is full with a slow/absent `StreamEvents` consumer — stalling any `WorkerCommandReply`/heartbeat queued behind an event frame, so an in-flight `InvokeAsync` could hit `CommandTimeout` despite a timely worker reply. Fix mirrors the existing outbound `WriteLoopAsync`: an unbounded event **staging** channel + a dedicated `EventWriteLoopAsync`; `DispatchEnvelope` is now fully synchronous and the `WorkerEvent` branch hands off with a non-blocking `TryWrite`, so the read loop never awaits event enqueue. The event write loop owns the timed write + the sustained-overflow `ProtocolViolation` fault (unchanged contract); registered in `WaitForBackgroundTasks`, completed on close/fault/dispose. Server build clean (0 warnings); non-pipe event tests 37/38 (the 1 is the known parallel-load `EventStreamServiceTests…TracksAggregateQueueDepth` timing flake — passes in isolation). New pipe-harness test (reply after events with a full consumer-less channel dispatches without `CommandTimeout`) verified on windev. Docs: GatewayProcessDesign read/write/event-loop section. Commit `32d6b48`. **Wave 3 complete.** | | 2026-07-09 | P1 Wave 3 (size/backpressure topology + write ordering). IPC-02/03/04 + WRK-04/07 → `Done`. **Size negotiation (IPC-02):** added `GatewayHello.max_frame_bytes` (regen `Generated/` + `clients/proto` descriptor refresh with pinned protoc 34.1); the gateway sends its negotiated worker-frame max and the worker adopts it (`WorkerFrameProtocolOptions.AdoptNegotiatedMaxMessageBytes`, 0 = keep default, >256 MiB rejected) instead of a hard-coded default. **Headroom (IPC-03):** the pipe frame max now sits `EnvelopeOverheadReserveBytes` (64 KiB) above the public gRPC cap (default `Worker.MaxMessageBytes` 16 MiB→16 MiB+64 KiB), cross-validated at startup; `WorkerClient` pre-checks command envelope size and fails only the offending correlation (`ResourceExhausted`) instead of `SetFaulted`ing the session. **Drain bound (IPC-04):** gateway request validator rejects `DrainEvents max_events` above 10 000; the worker caps each reply at `MaxDrainEventsPerReply` (10 000) and treats `max_events = 0` as that cap, not "drain all". **Sequence (WRK-04):** `WorkerFrameWriter` stamps the envelope `Sequence` at the point of writing under the write lock, so wire order and stamped sequence always agree under concurrent producers. **Priority (WRK-07):** the worker writer is now a cooperative priority scheduler — control frames (reply/fault/heartbeat/shutdown-ack) drain ahead of event frames; per-frame validation/size rejections fail only that frame, a stream failure fails all queued. Docs same-change (GatewayConfiguration, WorkerFrameProtocol, gateway.md). **Verified:** macOS NonWindows build clean + validator/grpc tests green; **windev** x86 worker builds clean, `Worker.Tests` 352 passed / 0 failed / 11 skipped (incl. new monotonic-sequence, control-before-event priority, negotiated-max, drain-bound tests), gateway `Tests` 799 passed / 3 failed — all 3 pre-existing windev-environmental (SelfSigned SAN + 2 `EventStreamServiceTests` timing, both pass in isolation). Commits `c8b3a22` (gateway half), `ebe6aea` (worker half), `309296f` (descriptor + default-expectation refresh). GWC-04 (event-channel decoupling) is the remaining Wave 3 item. | | 2026-07-09 | P1 S-misc (dashboard/observability hardening). SEC-02/12/20 → `Done`. SEC-02: `DashboardAuthorizationHandler` restricts the loopback + `Authentication:Mode=Disabled` bypasses to read-only (they satisfy a Viewer-bearing requirement but never `AdminOnly`), closing the policy-layer gap where anonymous localhost was authorized for Admin surfaces. SEC-12: `DashboardSessionAdminService` now emits canonical `AuditEvent`s (`dashboard-close-session`/`dashboard-kill-worker`, category `SessionAdmin`) through `IAuditWriter` on Success/Failure/Denied, so Close/Kill land durable audit rows. SEC-20: dropped the unbounded `session_id` tag from the exported `mxgateway.heartbeats.failed` counter. Docs updated same-change (CLAUDE.md, GatewayDashboardDesign.md, Metrics.md). Server build clean (0 warnings); targeted classes 30/30 pass; broader Dashboard+Security+GatewayApplication+Metrics sweep 295/295 pass. | | 2026-07-09 | P1 Wave 2b (security authz+hub). SEC-05/07/08/11 → `Done` (hub-token lifetime; QueryActiveAlarms scope arm; gateway-side verification cache + last-used coalescing; login rate limit + per-peer gRPC failure limiter). Full-suite checkpoint caught + fixed regressions the earlier narrow SEC-01/04/06 filter missed: cross-platform path-rooting, an `IHostEnvironment` fallback for minimal DI containers, a test-assembly `ASPNETCORE_ENVIRONMENT=Development` default, and a platform-correct default-path assertion. Suite: 747 passed / 42 failed, all 42 pre-existing macOS named-pipe-harness env failures. | From a038363e74faf7792bfbf6506c3b0dba1f0b838e Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Thu, 9 Jul 2026 09:48:09 -0400 Subject: [PATCH 19/19] feat(security): apikey --expires CLI flag + dashboard expiry/staleness badge (SEC-10 polish) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gateway-side follow-up to the shared auth-lib expiry core (delivered via G-2): - apikey create-key gains optional --expires — absolute ISO-8601 UTC or relative d/h from now; omitted means non-expiring (opt-in, unchanged default). Threaded ApiKeyAdminCommand -> parser -> runner into the library's CreateKeyAsync(..., expiresUtc, ...). list-keys shows an expiry column and an active/expired/revoked status. - Dashboard API Keys page surfaces expiry: an Expires column (Never when unset) and a status badge reading Expired (red) / Expiring (<=7d, amber) / Revoked / Active. DashboardApiKeySummary.ExpiresUtc projected in DashboardSnapshotService; StatusBadge maps the new states. Tests: parser (absolute/relative/invalid/none) + end-to-end past-expiry rejection through the live verifier; dashboard summary suite green. Docs: Authentication.md (verification-flow expiry step, CLI table + examples, dashboard badge). Closes the tracked SEC-10 polish (SEC-10 core was already Done). --- archreview/remediation/00-tracking.md | 1 + docs/Authentication.md | 10 +++- .../Components/Pages/ApiKeysPage.razor | 33 ++++++++++- .../Components/Shared/StatusBadge.razor | 4 +- .../Dashboard/DashboardApiKeySummary.cs | 3 +- .../Dashboard/DashboardSnapshotService.cs | 3 +- .../Authentication/ApiKeyAdminCliRunner.cs | 16 +++++- .../Authentication/ApiKeyAdminCommand.cs | 3 +- .../ApiKeyAdminCommandLineParser.cs | 41 +++++++++++++- .../Authentication/ApiKeyAdminListedKey.cs | 3 +- .../ApiKeyAdminCliRunnerTests.cs | 34 ++++++++++++ .../ApiKeyAdminCommandLineParserTests.cs | 55 +++++++++++++++++++ 12 files changed, 192 insertions(+), 14 deletions(-) diff --git a/archreview/remediation/00-tracking.md b/archreview/remediation/00-tracking.md index 7f729e3..c5c3a63 100644 --- a/archreview/remediation/00-tracking.md +++ b/archreview/remediation/00-tracking.md @@ -257,6 +257,7 @@ Findings the review flagged as one coordinated design pass — sequence them tog | 2026-07-09 | P0 tier executed via parallel agents. GWC-01/02/03, TST-02, TST-12, CLI-01, CLI-03 → `Done` (NonWindows build clean; gateway 135/135 targeted tests pass, Go `go test ./...` + `-race` pass, Rust `test`/`clippy -D warnings` pass). | | 2026-07-09 | Windows full-suite temp-file-lock flakiness (discovered via TST-08) → **fixed** (commit `11a716a`). Two shared-state parallel collisions that macOS never surfaces: (1) self-signed cert generation — `SelfSignedCertificateProvider` now stages the PFX in a unique `..tmp` (was a fixed name that concurrent/interrupted writers collided on); `TestHostEnvironmentInitializer` defaults the cert path to a per-process temp dir (so the suite never writes shared `ProgramData` or fights the deployed service); `GatewayTlsBootstrapTests` moved to a `DisableParallelization` collection so its process-global env-var mutation can't bleed into parallel tests. (2) `AuthStoreHealthCheckTests` — reuse the existing `TempDatabaseDirectory` helper (`SqliteConnection.ClearAllPools()` before delete) so the Microsoft.Data.Sqlite pool releases the temp `.db` handle. **windev-verified:** 3 consecutive full-suite runs all **793 passed / 2 failed, 0 new orphans** (was flaky 2–4 fails, count-varying). The 2 stable remaining failures are pre-existing windev-environmental (`SelfSignedCertificateProviderTests.GenerateCertificate_HasExpectedSansEkuAndValidity` machine-name/FQDN SAN assertion; `EventStreamServiceTests…TracksAggregateQueueDepth` timing) — both pass on macOS, unrelated to this change. | | 2026-07-09 | P1 TST-08 (orphaned testhost) → `Done` by evidence: the claimed leak does **not** reproduce. Full `ZB.MOM.WW.MxGateway.Tests` suite exits cleanly on macOS (0 survivors) and on the Windows dev box (windev, `origin/main` worktree: 0 new `testhost` and 0 new `ZB.MOM.WW.MxGateway.Worker` processes after a full run, isolated by StartTime). Static audit confirms the prime-suspect fixtures already dispose deterministically (WorkerClient cancels its `_stopCts` + awaits read/write/heartbeat tasks with a 5 s timeout; GatewaySession disposes the client; E2E/harness fixtures use `await using`). CLAUDE.md "Source Update Workflow" updated to drop the stale "leaves orphaned testhost processes" rationale (filtered runs kept as a speed guideline). **Discovered separately (NOT TST-08, still open):** Windows-only temp-file-lock flakiness in ~4 full-host test classes — `AuthStoreHealthCheckTests` (Microsoft.Data.Sqlite connection-pool holds the temp `.db` handle after `await using`, so the finally `File.Delete` throws "used by another process"), `GatewayTlsBootstrapTests`/`DashboardCookieOptionsTests`/`DashboardHubsRegistrationTests` (a `gw.pfx.tmp` handle from `SelfSignedCertificateProvider`'s fixed-name `path+".tmp"` atomic-write racing teardown). Count varies run-to-run (2–4 fails); macOS never sees it (Unix allows deleting open files). Fix = `SqliteConnection.ClearAllPools()` before temp delete + a unique/guarded `.tmp` name; verify on windev. | +| 2026-07-09 | SEC-10 polish (gateway-side follow-up to the G-2 auth-lib core) → done. `apikey create-key` gains an optional `--expires` (absolute ISO-8601 UTC or relative `d`/`h`; omit = non-expiring, opt-in) threaded through `ApiKeyAdminCommand`/parser/runner into the library's `CreateKeyAsync(..., expiresUtc, ...)`; `list-keys` shows an expiry column + `active`/`expired`/`revoked` status. Dashboard API Keys page surfaces expiry: an `Expires` column (`Never` when unset) and a status badge reading `Expired` (red) / `Expiring` (≤7 days, amber) / `Revoked` / `Active` (`DashboardApiKeySummary.ExpiresUtc` projected in `DashboardSnapshotService`; `StatusBadge` maps the new states). Server build clean; targeted tests 32/32 (parser abs/relative/invalid + end-to-end past-expiry rejection via the live verifier) + dashboard 28/28. Docs: `docs/Authentication.md` (verification flow expiry step, CLI table + examples, dashboard badge). SEC-10 was already `Done` (core); this closes the tracked polish. | | 2026-07-09 | P1 Wave 3 (event-channel decoupling). GWC-04 → `Done`. `WorkerClient`'s read loop awaited `EnqueueWorkerEventAsync` inline, which blocks in the bounded event channel's timed `WriteAsync` (≤ `EventChannelFullModeTimeout`, 5 s) when the channel is full with a slow/absent `StreamEvents` consumer — stalling any `WorkerCommandReply`/heartbeat queued behind an event frame, so an in-flight `InvokeAsync` could hit `CommandTimeout` despite a timely worker reply. Fix mirrors the existing outbound `WriteLoopAsync`: an unbounded event **staging** channel + a dedicated `EventWriteLoopAsync`; `DispatchEnvelope` is now fully synchronous and the `WorkerEvent` branch hands off with a non-blocking `TryWrite`, so the read loop never awaits event enqueue. The event write loop owns the timed write + the sustained-overflow `ProtocolViolation` fault (unchanged contract); registered in `WaitForBackgroundTasks`, completed on close/fault/dispose. Server build clean (0 warnings); non-pipe event tests 37/38 (the 1 is the known parallel-load `EventStreamServiceTests…TracksAggregateQueueDepth` timing flake — passes in isolation). New pipe-harness test (reply after events with a full consumer-less channel dispatches without `CommandTimeout`) verified on windev. Docs: GatewayProcessDesign read/write/event-loop section. Commit `32d6b48`. **Wave 3 complete.** | | 2026-07-09 | P1 Wave 3 (size/backpressure topology + write ordering). IPC-02/03/04 + WRK-04/07 → `Done`. **Size negotiation (IPC-02):** added `GatewayHello.max_frame_bytes` (regen `Generated/` + `clients/proto` descriptor refresh with pinned protoc 34.1); the gateway sends its negotiated worker-frame max and the worker adopts it (`WorkerFrameProtocolOptions.AdoptNegotiatedMaxMessageBytes`, 0 = keep default, >256 MiB rejected) instead of a hard-coded default. **Headroom (IPC-03):** the pipe frame max now sits `EnvelopeOverheadReserveBytes` (64 KiB) above the public gRPC cap (default `Worker.MaxMessageBytes` 16 MiB→16 MiB+64 KiB), cross-validated at startup; `WorkerClient` pre-checks command envelope size and fails only the offending correlation (`ResourceExhausted`) instead of `SetFaulted`ing the session. **Drain bound (IPC-04):** gateway request validator rejects `DrainEvents max_events` above 10 000; the worker caps each reply at `MaxDrainEventsPerReply` (10 000) and treats `max_events = 0` as that cap, not "drain all". **Sequence (WRK-04):** `WorkerFrameWriter` stamps the envelope `Sequence` at the point of writing under the write lock, so wire order and stamped sequence always agree under concurrent producers. **Priority (WRK-07):** the worker writer is now a cooperative priority scheduler — control frames (reply/fault/heartbeat/shutdown-ack) drain ahead of event frames; per-frame validation/size rejections fail only that frame, a stream failure fails all queued. Docs same-change (GatewayConfiguration, WorkerFrameProtocol, gateway.md). **Verified:** macOS NonWindows build clean + validator/grpc tests green; **windev** x86 worker builds clean, `Worker.Tests` 352 passed / 0 failed / 11 skipped (incl. new monotonic-sequence, control-before-event priority, negotiated-max, drain-bound tests), gateway `Tests` 799 passed / 3 failed — all 3 pre-existing windev-environmental (SelfSigned SAN + 2 `EventStreamServiceTests` timing, both pass in isolation). Commits `c8b3a22` (gateway half), `ebe6aea` (worker half), `309296f` (descriptor + default-expectation refresh). GWC-04 (event-channel decoupling) is the remaining Wave 3 item. | | 2026-07-09 | P1 S-misc (dashboard/observability hardening). SEC-02/12/20 → `Done`. SEC-02: `DashboardAuthorizationHandler` restricts the loopback + `Authentication:Mode=Disabled` bypasses to read-only (they satisfy a Viewer-bearing requirement but never `AdminOnly`), closing the policy-layer gap where anonymous localhost was authorized for Admin surfaces. SEC-12: `DashboardSessionAdminService` now emits canonical `AuditEvent`s (`dashboard-close-session`/`dashboard-kill-worker`, category `SessionAdmin`) through `IAuditWriter` on Success/Failure/Denied, so Close/Kill land durable audit rows. SEC-20: dropped the unbounded `session_id` tag from the exported `mxgateway.heartbeats.failed` counter. Docs updated same-change (CLAUDE.md, GatewayDashboardDesign.md, Metrics.md). Server build clean (0 warnings); targeted classes 30/30 pass; broader Dashboard+Security+GatewayApplication+Metrics sweep 295/295 pass. | diff --git a/docs/Authentication.md b/docs/Authentication.md index 29a17a3..ab3f0e6 100644 --- a/docs/Authentication.md +++ b/docs/Authentication.md @@ -73,7 +73,7 @@ The pepper is intentionally not stored alongside the hash: an attacker who exfil 1. Parse the `Authorization` header into a `ParsedApiKey`. 2. Look up the `ApiKeyRecord` by `KeyId` through `IApiKeyStore.FindByKeyIdAsync`. -3. Reject revoked records (`RevokedUtc is not null`). +3. Reject revoked records (`RevokedUtc is not null`) and expired records (`ExpiresUtc` in the past). Expiry is opt-in — keys created without an expiry never expire; an expired key fails opaquely, indistinguishable to the client from any other auth failure. 4. Hash the presented secret with the configured pepper. 5. Compare hashes with `CryptographicOperations.FixedTimeEquals` to avoid timing oracles. 6. Record a `LastUsedUtc` timestamp via `MarkKeyUsedAsync` and return an `ApiKeyIdentity`. @@ -193,6 +193,8 @@ public static ApiKeyRecord Read(SqliteDataReader reader) Because `RotateAsync` clears `revoked_utc`, rotating a previously revoked key reactivates it. The dashboard API Keys page therefore offers the Rotate (and Revoke) actions only for keys whose status is `Active`; revoked keys instead show a Delete action that calls `DeleteAsync`, so an operator can permanently remove a revoked row without ever risking un-revocation as a side effect of a rotation. +The dashboard API Keys page also surfaces expiry: each row shows an `Expires` column (`Never` when unset) and a status badge that reads `Expired` (past expiry, red), `Expiring` (within seven days, amber), `Revoked`, or `Active`. This is display-only staleness surfacing; expiry is set at creation time via the `apikey create-key --expires` CLI, not from the dashboard. + ### Audit trail `SqliteApiKeyAuditStore` (`IApiKeyAuditStore`) appends `ApiKeyAuditEntry` values to the `api_key_audit` table and stamps each row with a UTC timestamp inside the store rather than trusting the caller. `ListRecentAsync` returns the most recent rows ordered by `audit_id` descending and projects them into `ApiKeyAuditRecord`. Rows are kept even after the referenced key is revoked because the audit history is the durable record of administrative action; the `key_id` column is nullable to accommodate non-key-scoped events such as `init-db`. @@ -226,8 +228,8 @@ The supported subcommands match `ApiKeyAdminCommandKind` exactly: | Subcommand | Required options | Behaviour | |------------|------------------|-----------| | `init-db` | none | Runs the migrator and records an audit entry. | -| `create-key` | `--key-id`, `--display-name` | Generates a new secret, stores its peppered hash and optional constraints, and prints the assembled `mxgw__` token. | -| `list-keys` | none | Lists every stored key with its scopes, constraints, and revocation state. | +| `create-key` | `--key-id`, `--display-name` | Generates a new secret, stores its peppered hash and optional constraints, and prints the assembled `mxgw__` token. Optional `--expires` sets an expiry (absolute ISO-8601 UTC, or a relative `d`/`h` from now); omit it for a non-expiring key. | +| `list-keys` | none | Lists every stored key with its scopes, constraints, revocation state, and expiry (`active`/`expired`/`revoked`). | | `revoke-key` | `--key-id` | Sets `revoked_utc` if the key is currently active. | | `rotate-key` | `--key-id` | Replaces the secret hash and prints the new token. | @@ -237,6 +239,8 @@ Examples: mxgateway apikey init-db mxgateway apikey create-key --key-id ops.alice --display-name "Alice (ops)" --scopes read,write mxgateway apikey create-key --key-id area1.reader --display-name "Area 1 reader" --scopes invoke:read,metadata:read --read-subtree "Area1/*" --browse-subtree "Area1/*" +mxgateway apikey create-key --key-id ops.temp --display-name "Temp contractor" --scopes invoke:read --expires 90d +mxgateway apikey create-key --key-id ops.audit --display-name "Audit window" --scopes metadata:read --expires 2027-01-01T00:00:00Z mxgateway apikey list-keys --json mxgateway apikey revoke-key --key-id ops.alice mxgateway apikey rotate-key --key-id ops.alice diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Pages/ApiKeysPage.razor b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Pages/ApiKeysPage.razor index e51d4a0..0a32be6 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Pages/ApiKeysPage.razor +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Pages/ApiKeysPage.razor @@ -164,6 +164,7 @@ else Constraints Created Last Used + Expires @if (CanManageApiKeys) { Actions @@ -175,12 +176,13 @@ else { @key.KeyId - + @DashboardDisplay.Text(key.DisplayName) @DashboardDisplay.Text(string.Join(", ", key.Scopes.Order(StringComparer.Ordinal))) @DashboardDisplay.Text(ConstraintText(key.Constraints)) @DashboardDisplay.DateTime(key.CreatedUtc) @DashboardDisplay.DateTime(key.LastUsedUtc) + @(key.ExpiresUtc is null ? "Never" : DashboardDisplay.DateTime(key.ExpiresUtc)) @if (CanManageApiKeys) { @@ -468,6 +470,35 @@ else } } + // Window before an expiry within which a key is flagged as "Expiring" (warn) rather than "Active". + private static readonly TimeSpan ExpiringSoonWindow = TimeSpan.FromDays(7); + + // Status vocabulary understood by StatusBadge: Revoked wins over expiry; a past expiry is Expired + // (bad), an expiry inside ExpiringSoonWindow is Expiring (warn), otherwise Active (SEC-10). + private static string KeyStatus(DashboardApiKeySummary key) + { + if (key.RevokedUtc is not null) + { + return "Revoked"; + } + + if (key.ExpiresUtc is { } expiresAt) + { + DateTimeOffset now = DateTimeOffset.UtcNow; + if (expiresAt <= now) + { + return "Expired"; + } + + if (expiresAt - now <= ExpiringSoonWindow) + { + return "Expiring"; + } + } + + return "Active"; + } + private static string ConstraintText(ApiKeyConstraints constraints) { if (constraints.IsEmpty) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Shared/StatusBadge.razor b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Shared/StatusBadge.razor index e97f59f..e7e96a2 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Shared/StatusBadge.razor +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Shared/StatusBadge.razor @@ -9,8 +9,8 @@ { "Ready" or "Healthy" or "Active" => StatusState.Ok, "Creating" or "StartingWorker" or "WaitingForPipe" or "InitializingWorker" or "Closing" - or "Stale" or "Degraded" => StatusState.Warn, - "Faulted" or "Unavailable" => StatusState.Bad, + or "Stale" or "Degraded" or "Expiring" => StatusState.Warn, + "Faulted" or "Unavailable" or "Expired" => StatusState.Bad, _ => StatusState.Idle, }; } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardApiKeySummary.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardApiKeySummary.cs index 820342b..41328d4 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardApiKeySummary.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardApiKeySummary.cs @@ -9,4 +9,5 @@ public sealed record DashboardApiKeySummary( ApiKeyConstraints Constraints, DateTimeOffset CreatedUtc, DateTimeOffset? LastUsedUtc, - DateTimeOffset? RevokedUtc); + DateTimeOffset? RevokedUtc, + DateTimeOffset? ExpiresUtc = null); diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs index 9d2675b..7039f47 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs @@ -273,7 +273,8 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService Constraints: ApiKeyConstraintSerializer.Deserialize(key.ConstraintsJson), CreatedUtc: key.CreatedUtc, LastUsedUtc: key.LastUsedUtc, - RevokedUtc: key.RevokedUtc)) + RevokedUtc: key.RevokedUtc, + ExpiresUtc: key.ExpiresUtc)) .ToArray(); Volatile.Write(ref _apiKeySummaries, summaries); diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/ApiKeyAdminCliRunner.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/ApiKeyAdminCliRunner.cs index 643b14e..e852397 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/ApiKeyAdminCliRunner.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/ApiKeyAdminCliRunner.cs @@ -67,6 +67,7 @@ public sealed class ApiKeyAdminCliRunner(ApiKeyAdminCommands commands) Required(command.DisplayName), command.Scopes, ApiKeyConstraintSerializer.Serialize(command.Constraints), + command.ExpiresUtc, remoteAddress: null, cancellationToken) .ConfigureAwait(false); @@ -134,8 +135,16 @@ public sealed class ApiKeyAdminCliRunner(ApiKeyAdminCommands commands) foreach (ApiKeyAdminListedKey key in result.Keys) { - string revoked = key.RevokedUtc is null ? "active" : "revoked"; - await output.WriteLineAsync($"{key.KeyId}\t{key.DisplayName}\t{revoked}\t{string.Join(',', key.Scopes)}") + string status = key.RevokedUtc is not null + ? "revoked" + : key.ExpiresUtc is { } expiresAt && expiresAt <= DateTimeOffset.UtcNow + ? "expired" + : "active"; + string expiry = key.ExpiresUtc is { } expires + ? expires.ToUniversalTime().ToString("u", System.Globalization.CultureInfo.InvariantCulture) + : "-"; + await output.WriteLineAsync( + $"{key.KeyId}\t{key.DisplayName}\t{status}\t{expiry}\t{string.Join(',', key.Scopes)}") .ConfigureAwait(false); } } @@ -150,7 +159,8 @@ public sealed class ApiKeyAdminCliRunner(ApiKeyAdminCommands commands) Constraints: ApiKeyConstraintSerializer.Deserialize(key.ConstraintsJson), CreatedUtc: key.CreatedUtc, LastUsedUtc: key.LastUsedUtc, - RevokedUtc: key.RevokedUtc); + RevokedUtc: key.RevokedUtc, + ExpiresUtc: key.ExpiresUtc); } private static string Required(string? value) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/ApiKeyAdminCommand.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/ApiKeyAdminCommand.cs index b2d5105..46c6f2d 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/ApiKeyAdminCommand.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/ApiKeyAdminCommand.cs @@ -8,4 +8,5 @@ public sealed record ApiKeyAdminCommand( string? KeyId, string? DisplayName, IReadOnlySet Scopes, - ApiKeyConstraints Constraints); + ApiKeyConstraints Constraints, + DateTimeOffset? ExpiresUtc = null); diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/ApiKeyAdminCommandLineParser.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/ApiKeyAdminCommandLineParser.cs index 315c0fc..6ec44da 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/ApiKeyAdminCommandLineParser.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/ApiKeyAdminCommandLineParser.cs @@ -82,9 +82,11 @@ public static class ApiKeyAdminCommandLineParser string? displayName = GetOption(options, "display-name"); IReadOnlySet scopes = ParseScopes(GetOption(options, "scopes")); ApiKeyConstraints constraints; + DateTimeOffset? expiresUtc; try { constraints = ParseConstraints(options); + expiresUtc = ParseExpiry(GetOption(options, "expires")); } catch (FormatException exception) { @@ -111,7 +113,8 @@ public static class ApiKeyAdminCommandLineParser KeyId: keyId, DisplayName: displayName, Scopes: scopes, - Constraints: constraints)); + Constraints: constraints, + ExpiresUtc: expiresUtc)); } private static bool TryParseKind(string value, out ApiKeyAdminCommandKind kind) @@ -233,6 +236,42 @@ public static class ApiKeyAdminCommandLineParser ReadHistorizedOnly: HasFlag(options, "read-historized-only")); } + // Parses the optional --expires value into an absolute UTC expiry (SEC-10). Accepts a relative + // "d"/"h" duration from now (operator-friendly) or an absolute ISO-8601 instant/date + // (assumed UTC). Null/blank means no expiry — expiry stays opt-in, preserving prior behaviour. + private static DateTimeOffset? ParseExpiry(string? value) + { + if (string.IsNullOrWhiteSpace(value)) + { + return null; + } + + string trimmed = value.Trim(); + char unit = char.ToLowerInvariant(trimmed[^1]); + if (unit is 'd' or 'h' + && int.TryParse( + trimmed[..^1], + System.Globalization.NumberStyles.None, + System.Globalization.CultureInfo.InvariantCulture, + out int amount)) + { + TimeSpan offset = unit == 'd' ? TimeSpan.FromDays(amount) : TimeSpan.FromHours(amount); + return DateTimeOffset.UtcNow + offset; + } + + if (DateTimeOffset.TryParse( + trimmed, + System.Globalization.CultureInfo.InvariantCulture, + System.Globalization.DateTimeStyles.AssumeUniversal | System.Globalization.DateTimeStyles.AdjustToUniversal, + out DateTimeOffset parsed)) + { + return parsed; + } + + throw new FormatException( + "--expires must be a relative duration ('d' or 'h') or an absolute ISO-8601 UTC timestamp."); + } + private static int? ParseNullableInt(string? value) { if (string.IsNullOrWhiteSpace(value)) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/ApiKeyAdminListedKey.cs b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/ApiKeyAdminListedKey.cs index f621282..2aec91a 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/ApiKeyAdminListedKey.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Security/Authentication/ApiKeyAdminListedKey.cs @@ -8,4 +8,5 @@ public sealed record ApiKeyAdminListedKey( ApiKeyConstraints Constraints, DateTimeOffset CreatedUtc, DateTimeOffset? LastUsedUtc, - DateTimeOffset? RevokedUtc); + DateTimeOffset? RevokedUtc, + DateTimeOffset? ExpiresUtc = null); diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/ApiKeyAdminCliRunnerTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/ApiKeyAdminCliRunnerTests.cs index e95d5cc..6890a75 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/ApiKeyAdminCliRunnerTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/ApiKeyAdminCliRunnerTests.cs @@ -52,6 +52,40 @@ public sealed class ApiKeyAdminCliRunnerTests : IDisposable Assert.Contains(auditRecords, record => record.EventType == "create-key" && record.KeyId == "operator01"); } + /// + /// Verifies that a key created with an already-past --expires is rejected by the verifier + /// — the CLI expiry wiring reaches the store and the library enforces it end-to-end (SEC-10). + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task CreateKeyAsync_WithPastExpiry_KeyIsRejectedByVerifier() + { + await using ServiceProvider services = BuildServices(CreateTempDatabasePath()); + ApiKeyAdminCliRunner runner = services.GetRequiredService(); + StringWriter output = new(); + + await runner.RunAsync( + new ApiKeyAdminCommand( + Kind: ApiKeyAdminCommandKind.CreateKey, + Json: true, + SqlitePath: null, + Pepper: null, + KeyId: "operator01", + DisplayName: "Operator", + Scopes: new HashSet(StringComparer.Ordinal) { "session:open" }, + Constraints: ApiKeyConstraints.Empty, + ExpiresUtc: DateTimeOffset.UtcNow - TimeSpan.FromMinutes(1)), + output, + CancellationToken.None); + + string apiKey = ReadApiKey(output.ToString()); + + IApiKeyVerifier verifier = services.GetRequiredService(); + ApiKeyVerification verification = await verifier.VerifyAsync($"Bearer {apiKey}", CancellationToken.None); + + Assert.False(verification.Succeeded); + } + /// Verifies that ListKeysAsync does not print the raw secret. /// A task that represents the asynchronous operation. [Fact] diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/ApiKeyAdminCommandLineParserTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/ApiKeyAdminCommandLineParserTests.cs index 3f82499..9fe0eba 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/ApiKeyAdminCommandLineParserTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Security/Authentication/ApiKeyAdminCommandLineParserTests.cs @@ -52,6 +52,61 @@ public sealed class ApiKeyAdminCommandLineParserTests Assert.Contains("events:read", result.Command.Scopes); } + /// A create-key command without --expires leaves the key non-expiring (opt-in, SEC-10). + [Fact] + public void Parse_CreateKeyCommand_WithoutExpires_HasNoExpiry() + { + ApiKeyAdminParseResult result = ApiKeyAdminCommandLineParser.Parse( + ["apikey", "create-key", "--key-id", "operator01", "--display-name", "Operator", "--scopes", "session:open"]); + + Assert.NotNull(result.Command); + Assert.Null(result.Command.ExpiresUtc); + } + + /// An absolute ISO-8601 --expires value is parsed to that exact UTC instant (SEC-10). + [Fact] + public void Parse_CreateKeyCommand_WithAbsoluteExpires_ParsesInstant() + { + ApiKeyAdminParseResult result = ApiKeyAdminCommandLineParser.Parse( + ["apikey", "create-key", "--key-id", "operator01", "--display-name", "Operator", "--scopes", "session:open", + "--expires", "2030-01-02T03:04:05Z"]); + + Assert.NotNull(result.Command); + Assert.Equal( + new DateTimeOffset(2030, 1, 2, 3, 4, 5, TimeSpan.Zero), + result.Command.ExpiresUtc); + } + + /// A relative "<N>d" --expires value resolves to a future UTC instant (SEC-10). + [Fact] + public void Parse_CreateKeyCommand_WithRelativeExpires_ResolvesToFuture() + { + DateTimeOffset before = DateTimeOffset.UtcNow; + ApiKeyAdminParseResult result = ApiKeyAdminCommandLineParser.Parse( + ["apikey", "create-key", "--key-id", "operator01", "--display-name", "Operator", "--scopes", "session:open", + "--expires", "30d"]); + + Assert.NotNull(result.Command); + Assert.NotNull(result.Command.ExpiresUtc); + Assert.InRange( + result.Command.ExpiresUtc!.Value, + before + TimeSpan.FromDays(30) - TimeSpan.FromMinutes(1), + DateTimeOffset.UtcNow + TimeSpan.FromDays(30) + TimeSpan.FromMinutes(1)); + } + + /// An unparseable --expires value fails at parse time (SEC-10). + [Fact] + public void Parse_CreateKeyCommand_WithInvalidExpires_Fails() + { + ApiKeyAdminParseResult result = ApiKeyAdminCommandLineParser.Parse( + ["apikey", "create-key", "--key-id", "operator01", "--display-name", "Operator", "--scopes", "session:open", + "--expires", "not-a-date"]); + + Assert.Null(result.Command); + Assert.NotNull(result.Error); + Assert.Contains("--expires", result.Error, StringComparison.Ordinal); + } + /// /// A create-key command with a non-canonical scope /// string (e.g. CLAUDE.md's stale invoke instead of invoke:read)