From 30ac3f322568e6923740eaa1a2fdaf5050935b02 Mon Sep 17 00:00:00 2001 From: Joseph Doherty Date: Sat, 21 Nov 2026 08:11:31 -0500 Subject: [PATCH] feat(worker,gateway): report record-info pin outcome in WorkerReady; drain-thread completion tests The worker logged its MXSTATUS_PROXY record-info pin outcome only to its own stderr, which the gateway's launcher does not capture, so operators never saw whether the pin took. - Contract (additive, internal IPC): WorkerReady.record_info_pin_status = 5. No protocol version bump; an older peer ignores it or sends it empty. Regenerated Contracts/Generated, the client protoset, the Rust vendored proto, and the Go/Python/Java worker bindings (real delta only). - Worker: the handshake copies MxStatusProxyRecordInfoPin.Status into WorkerReady unless the runtime already set it. - Gateway: WorkerClient records it (IWorkerClient.RecordInfoPinStatus, default null) and logs it at Information when pinned, Warning otherwise; empty is not logged. Shown as "Record-info pin" in the dashboard session details Worker table via DashboardSessionSummary.WorkerRecordInfoPinStatus. - Tests: drain-thread cancellation (task ends Canceled; RunAsync cancels, runtime disposed, thread exits, no EventDrain stop-failure log) and fault (exception on the thread faults RunAsync with the same instance) paths; WorkerReady carries the pin status; gateway logging/recording and the dashboard projection. - Docs: gateway.md STA thread model (drain thread, pin step in COM lifetime), MxAccessWorkerInstanceDesign, GatewayProcessDesign, GatewayDashboardDesign, WorkerFrameProtocol. --- .../internal/generated/mxaccess_worker.pb.go | 23 +- .../mxaccess_worker/v1/MxaccessWorker.java | 334 +++++++++++++++--- .../descriptors/mxaccessgw-client-v1.protoset | Bin 129480 -> 130084 bytes .../generated/mxaccess_worker_pb2.py | 44 +-- clients/rust/protos/mxaccess_worker.proto | 11 + docs/GatewayDashboardDesign.md | 4 +- docs/GatewayProcessDesign.md | 9 + docs/MxAccessWorkerInstanceDesign.md | 14 +- docs/WorkerFrameProtocol.md | 6 + gateway.md | 38 +- .../Generated/MxaccessWorker.cs | 140 +++++--- .../Protos/mxaccess_worker.proto | 7 + .../Components/Pages/SessionDetailsPage.razor | 1 + .../Dashboard/DashboardSessionSummary.cs | 3 +- .../Dashboard/DashboardSnapshotService.cs | 3 +- .../Workers/IWorkerClient.cs | 8 + .../Workers/WorkerClient.cs | 45 +++ .../DashboardSnapshotServiceTests.cs | 28 ++ .../Workers/Fakes/FakeWorkerHarness.cs | 12 +- .../Workers/WorkerClientRecordInfoPinTests.cs | 96 +++++ .../Ipc/WorkerPipeSessionTests.cs | 145 ++++++++ .../TestSupport/FakeRuntimeSession.cs | 25 ++ .../Ipc/WorkerPipeSession.cs | 27 +- 23 files changed, 878 insertions(+), 145 deletions(-) create mode 100644 src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientRecordInfoPinTests.cs diff --git a/clients/go/internal/generated/mxaccess_worker.pb.go b/clients/go/internal/generated/mxaccess_worker.pb.go index 60c0c4bb..1427d6a5 100644 --- a/clients/go/internal/generated/mxaccess_worker.pb.go +++ b/clients/go/internal/generated/mxaccess_worker.pb.go @@ -580,8 +580,15 @@ type WorkerReady struct { MxaccessProgid string `protobuf:"bytes,2,opt,name=mxaccess_progid,json=mxaccessProgid,proto3" json:"mxaccess_progid,omitempty"` MxaccessClsid string `protobuf:"bytes,3,opt,name=mxaccess_clsid,json=mxaccessClsid,proto3" json:"mxaccess_clsid,omitempty"` ReadyTimestamp *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=ready_timestamp,json=readyTimestamp,proto3" json:"ready_timestamp,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Outcome of the worker's process-wide MXSTATUS_PROXY record-info pin + // (MxStatusProxyRecordInfoPin): "pinned {library-guid} vM.m record {record-guid}" on + // success, or the failing call and HRESULT / exception type otherwise. Empty when the + // worker made no attempt (an older worker, or a runtime that never created the + // MXAccess COM object). Diagnostic only: the gateway logs it and shows it on the + // dashboard, and never gates readiness on it, because the pin is a cache warm-up. + RecordInfoPinStatus string `protobuf:"bytes,5,opt,name=record_info_pin_status,json=recordInfoPinStatus,proto3" json:"record_info_pin_status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *WorkerReady) Reset() { @@ -642,6 +649,13 @@ func (x *WorkerReady) GetReadyTimestamp() *timestamppb.Timestamp { return nil } +func (x *WorkerReady) GetRecordInfoPinStatus() string { + if x != nil { + return x.RecordInfoPinStatus + } + return "" +} + type WorkerCommand struct { state protoimpl.MessageState `protogen:"open.v1"` Command *MxCommand `protobuf:"bytes,1,opt,name=command,proto3" json:"command,omitempty"` @@ -1139,12 +1153,13 @@ const file_mxaccess_worker_proto_rawDesc = "" + "\x10protocol_version\x18\x01 \x01(\rR\x0fprotocolVersion\x12\x14\n" + "\x05nonce\x18\x02 \x01(\tR\x05nonce\x12*\n" + "\x11worker_process_id\x18\x03 \x01(\x05R\x0fworkerProcessId\x12%\n" + - "\x0eworker_version\x18\x04 \x01(\tR\rworkerVersion\"\xce\x01\n" + + "\x0eworker_version\x18\x04 \x01(\tR\rworkerVersion\"\x83\x02\n" + "\vWorkerReady\x12*\n" + "\x11worker_process_id\x18\x01 \x01(\x05R\x0fworkerProcessId\x12'\n" + "\x0fmxaccess_progid\x18\x02 \x01(\tR\x0emxaccessProgid\x12%\n" + "\x0emxaccess_clsid\x18\x03 \x01(\tR\rmxaccessClsid\x12C\n" + - "\x0fready_timestamp\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\x0ereadyTimestamp\"\x92\x01\n" + + "\x0fready_timestamp\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\x0ereadyTimestamp\x123\n" + + "\x16record_info_pin_status\x18\x05 \x01(\tR\x13recordInfoPinStatus\"\x92\x01\n" + "\rWorkerCommand\x128\n" + "\acommand\x18\x01 \x01(\v2\x1e.mxaccess_gateway.v1.MxCommandR\acommand\x12G\n" + "\x11enqueue_timestamp\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x10enqueueTimestamp\"\x9c\x01\n" + diff --git a/clients/java/src/main/generated/main/java/mxaccess_worker/v1/MxaccessWorker.java b/clients/java/src/main/generated/main/java/mxaccess_worker/v1/MxaccessWorker.java index 0267bbdf..a0dc1bce 100644 --- a/clients/java/src/main/generated/main/java/mxaccess_worker/v1/MxaccessWorker.java +++ b/clients/java/src/main/generated/main/java/mxaccess_worker/v1/MxaccessWorker.java @@ -5477,6 +5477,36 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile { * .google.protobuf.Timestamp ready_timestamp = 4; */ com.google.protobuf.TimestampOrBuilder getReadyTimestampOrBuilder(); + + /** + *
+     * Outcome of the worker's process-wide MXSTATUS_PROXY record-info pin
+     * (MxStatusProxyRecordInfoPin): "pinned {library-guid} vM.m record {record-guid}" on
+     * success, or the failing call and HRESULT / exception type otherwise. Empty when the
+     * worker made no attempt (an older worker, or a runtime that never created the
+     * MXAccess COM object). Diagnostic only: the gateway logs it and shows it on the
+     * dashboard, and never gates readiness on it, because the pin is a cache warm-up.
+     * 
+ * + * string record_info_pin_status = 5; + * @return The recordInfoPinStatus. + */ + java.lang.String getRecordInfoPinStatus(); + /** + *
+     * Outcome of the worker's process-wide MXSTATUS_PROXY record-info pin
+     * (MxStatusProxyRecordInfoPin): "pinned {library-guid} vM.m record {record-guid}" on
+     * success, or the failing call and HRESULT / exception type otherwise. Empty when the
+     * worker made no attempt (an older worker, or a runtime that never created the
+     * MXAccess COM object). Diagnostic only: the gateway logs it and shows it on the
+     * dashboard, and never gates readiness on it, because the pin is a cache warm-up.
+     * 
+ * + * string record_info_pin_status = 5; + * @return The bytes for recordInfoPinStatus. + */ + com.google.protobuf.ByteString + getRecordInfoPinStatusBytes(); } /** * Protobuf type {@code mxaccess_worker.v1.WorkerReady} @@ -5502,6 +5532,7 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile { private WorkerReady() { mxaccessProgid_ = ""; mxaccessClsid_ = ""; + recordInfoPinStatus_ = ""; } public static final com.google.protobuf.Descriptors.Descriptor @@ -5633,6 +5664,63 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile { return readyTimestamp_ == null ? com.google.protobuf.Timestamp.getDefaultInstance() : readyTimestamp_; } + public static final int RECORD_INFO_PIN_STATUS_FIELD_NUMBER = 5; + @SuppressWarnings("serial") + private volatile java.lang.Object recordInfoPinStatus_ = ""; + /** + *
+     * Outcome of the worker's process-wide MXSTATUS_PROXY record-info pin
+     * (MxStatusProxyRecordInfoPin): "pinned {library-guid} vM.m record {record-guid}" on
+     * success, or the failing call and HRESULT / exception type otherwise. Empty when the
+     * worker made no attempt (an older worker, or a runtime that never created the
+     * MXAccess COM object). Diagnostic only: the gateway logs it and shows it on the
+     * dashboard, and never gates readiness on it, because the pin is a cache warm-up.
+     * 
+ * + * string record_info_pin_status = 5; + * @return The recordInfoPinStatus. + */ + @java.lang.Override + public java.lang.String getRecordInfoPinStatus() { + java.lang.Object ref = recordInfoPinStatus_; + if (ref instanceof java.lang.String) { + return (java.lang.String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + recordInfoPinStatus_ = s; + return s; + } + } + /** + *
+     * Outcome of the worker's process-wide MXSTATUS_PROXY record-info pin
+     * (MxStatusProxyRecordInfoPin): "pinned {library-guid} vM.m record {record-guid}" on
+     * success, or the failing call and HRESULT / exception type otherwise. Empty when the
+     * worker made no attempt (an older worker, or a runtime that never created the
+     * MXAccess COM object). Diagnostic only: the gateway logs it and shows it on the
+     * dashboard, and never gates readiness on it, because the pin is a cache warm-up.
+     * 
+ * + * string record_info_pin_status = 5; + * @return The bytes for recordInfoPinStatus. + */ + @java.lang.Override + public com.google.protobuf.ByteString + getRecordInfoPinStatusBytes() { + java.lang.Object ref = recordInfoPinStatus_; + if (ref instanceof java.lang.String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + recordInfoPinStatus_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + private byte memoizedIsInitialized = -1; @java.lang.Override public final boolean isInitialized() { @@ -5659,6 +5747,9 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile { if (((bitField0_ & 0x00000001) != 0)) { output.writeMessage(4, getReadyTimestamp()); } + if (!com.google.protobuf.GeneratedMessage.isStringEmpty(recordInfoPinStatus_)) { + com.google.protobuf.GeneratedMessage.writeString(output, 5, recordInfoPinStatus_); + } getUnknownFields().writeTo(output); } @@ -5682,6 +5773,9 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile { size += com.google.protobuf.CodedOutputStream .computeMessageSize(4, getReadyTimestamp()); } + if (!com.google.protobuf.GeneratedMessage.isStringEmpty(recordInfoPinStatus_)) { + size += com.google.protobuf.GeneratedMessage.computeStringSize(5, recordInfoPinStatus_); + } size += getUnknownFields().getSerializedSize(); memoizedSize = size; return size; @@ -5708,6 +5802,8 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile { if (!getReadyTimestamp() .equals(other.getReadyTimestamp())) return false; } + if (!getRecordInfoPinStatus() + .equals(other.getRecordInfoPinStatus())) return false; if (!getUnknownFields().equals(other.getUnknownFields())) return false; return true; } @@ -5729,6 +5825,8 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile { hash = (37 * hash) + READY_TIMESTAMP_FIELD_NUMBER; hash = (53 * hash) + getReadyTimestamp().hashCode(); } + hash = (37 * hash) + RECORD_INFO_PIN_STATUS_FIELD_NUMBER; + hash = (53 * hash) + getRecordInfoPinStatus().hashCode(); hash = (29 * hash) + getUnknownFields().hashCode(); memoizedHashCode = hash; return hash; @@ -5874,6 +5972,7 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile { readyTimestampBuilder_.dispose(); readyTimestampBuilder_ = null; } + recordInfoPinStatus_ = ""; return this; } @@ -5923,6 +6022,9 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile { : readyTimestampBuilder_.build(); to_bitField0_ |= 0x00000001; } + if (((from_bitField0_ & 0x00000010) != 0)) { + result.recordInfoPinStatus_ = recordInfoPinStatus_; + } result.bitField0_ |= to_bitField0_; } @@ -5954,6 +6056,11 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile { if (other.hasReadyTimestamp()) { mergeReadyTimestamp(other.getReadyTimestamp()); } + if (!other.getRecordInfoPinStatus().isEmpty()) { + recordInfoPinStatus_ = other.recordInfoPinStatus_; + bitField0_ |= 0x00000010; + onChanged(); + } this.mergeUnknownFields(other.getUnknownFields()); onChanged(); return this; @@ -6002,6 +6109,11 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile { bitField0_ |= 0x00000008; break; } // case 34 + case 42: { + recordInfoPinStatus_ = input.readStringRequireUtf8(); + bitField0_ |= 0x00000010; + break; + } // case 42 default: { if (!super.parseUnknownField(input, extensionRegistry, tag)) { done = true; // was an endgroup tag @@ -6316,6 +6428,123 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile { return readyTimestampBuilder_; } + private java.lang.Object recordInfoPinStatus_ = ""; + /** + *
+       * Outcome of the worker's process-wide MXSTATUS_PROXY record-info pin
+       * (MxStatusProxyRecordInfoPin): "pinned {library-guid} vM.m record {record-guid}" on
+       * success, or the failing call and HRESULT / exception type otherwise. Empty when the
+       * worker made no attempt (an older worker, or a runtime that never created the
+       * MXAccess COM object). Diagnostic only: the gateway logs it and shows it on the
+       * dashboard, and never gates readiness on it, because the pin is a cache warm-up.
+       * 
+ * + * string record_info_pin_status = 5; + * @return The recordInfoPinStatus. + */ + public java.lang.String getRecordInfoPinStatus() { + java.lang.Object ref = recordInfoPinStatus_; + if (!(ref instanceof java.lang.String)) { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + java.lang.String s = bs.toStringUtf8(); + recordInfoPinStatus_ = s; + return s; + } else { + return (java.lang.String) ref; + } + } + /** + *
+       * Outcome of the worker's process-wide MXSTATUS_PROXY record-info pin
+       * (MxStatusProxyRecordInfoPin): "pinned {library-guid} vM.m record {record-guid}" on
+       * success, or the failing call and HRESULT / exception type otherwise. Empty when the
+       * worker made no attempt (an older worker, or a runtime that never created the
+       * MXAccess COM object). Diagnostic only: the gateway logs it and shows it on the
+       * dashboard, and never gates readiness on it, because the pin is a cache warm-up.
+       * 
+ * + * string record_info_pin_status = 5; + * @return The bytes for recordInfoPinStatus. + */ + public com.google.protobuf.ByteString + getRecordInfoPinStatusBytes() { + java.lang.Object ref = recordInfoPinStatus_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (java.lang.String) ref); + recordInfoPinStatus_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + /** + *
+       * Outcome of the worker's process-wide MXSTATUS_PROXY record-info pin
+       * (MxStatusProxyRecordInfoPin): "pinned {library-guid} vM.m record {record-guid}" on
+       * success, or the failing call and HRESULT / exception type otherwise. Empty when the
+       * worker made no attempt (an older worker, or a runtime that never created the
+       * MXAccess COM object). Diagnostic only: the gateway logs it and shows it on the
+       * dashboard, and never gates readiness on it, because the pin is a cache warm-up.
+       * 
+ * + * string record_info_pin_status = 5; + * @param value The recordInfoPinStatus to set. + * @return This builder for chaining. + */ + public Builder setRecordInfoPinStatus( + java.lang.String value) { + if (value == null) { throw new NullPointerException(); } + recordInfoPinStatus_ = value; + bitField0_ |= 0x00000010; + onChanged(); + return this; + } + /** + *
+       * Outcome of the worker's process-wide MXSTATUS_PROXY record-info pin
+       * (MxStatusProxyRecordInfoPin): "pinned {library-guid} vM.m record {record-guid}" on
+       * success, or the failing call and HRESULT / exception type otherwise. Empty when the
+       * worker made no attempt (an older worker, or a runtime that never created the
+       * MXAccess COM object). Diagnostic only: the gateway logs it and shows it on the
+       * dashboard, and never gates readiness on it, because the pin is a cache warm-up.
+       * 
+ * + * string record_info_pin_status = 5; + * @return This builder for chaining. + */ + public Builder clearRecordInfoPinStatus() { + recordInfoPinStatus_ = getDefaultInstance().getRecordInfoPinStatus(); + bitField0_ = (bitField0_ & ~0x00000010); + onChanged(); + return this; + } + /** + *
+       * Outcome of the worker's process-wide MXSTATUS_PROXY record-info pin
+       * (MxStatusProxyRecordInfoPin): "pinned {library-guid} vM.m record {record-guid}" on
+       * success, or the failing call and HRESULT / exception type otherwise. Empty when the
+       * worker made no attempt (an older worker, or a runtime that never created the
+       * MXAccess COM object). Diagnostic only: the gateway logs it and shows it on the
+       * dashboard, and never gates readiness on it, because the pin is a cache warm-up.
+       * 
+ * + * string record_info_pin_status = 5; + * @param value The bytes for recordInfoPinStatus to set. + * @return This builder for chaining. + */ + public Builder setRecordInfoPinStatusBytes( + com.google.protobuf.ByteString value) { + if (value == null) { throw new NullPointerException(); } + checkByteStringIsUtf8(value); + recordInfoPinStatus_ = value; + bitField0_ |= 0x00000010; + onChanged(); + return this; + } + // @@protoc_insertion_point(builder_scope:mxaccess_worker.v1.WorkerReady) } @@ -12698,60 +12927,61 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile { "ay_version\030\003 \001(\t\022\027\n\017max_frame_bytes\030\004 \001(" + "\r\"i\n\013WorkerHello\022\030\n\020protocol_version\030\001 \001" + "(\r\022\r\n\005nonce\030\002 \001(\t\022\031\n\021worker_process_id\030\003" + - " \001(\005\022\026\n\016worker_version\030\004 \001(\t\"\216\001\n\013WorkerR" + + " \001(\005\022\026\n\016worker_version\030\004 \001(\t\"\256\001\n\013WorkerR" + "eady\022\031\n\021worker_process_id\030\001 \001(\005\022\027\n\017mxacc" + "ess_progid\030\002 \001(\t\022\026\n\016mxaccess_clsid\030\003 \001(\t" + "\0223\n\017ready_timestamp\030\004 \001(\0132\032.google.proto" + - "buf.Timestamp\"w\n\rWorkerCommand\022/\n\007comman" + - "d\030\001 \001(\0132\036.mxaccess_gateway.v1.MxCommand\022" + - "5\n\021enqueue_timestamp\030\002 \001(\0132\032.google.prot" + - "obuf.Timestamp\"\201\001\n\022WorkerCommandReply\0222\n" + - "\005reply\030\001 \001(\0132#.mxaccess_gateway.v1.MxCom" + - "mandReply\0227\n\023completed_timestamp\030\002 \001(\0132\032" + - ".google.protobuf.Timestamp\"\036\n\014WorkerCanc" + - "el\022\016\n\006reason\030\001 \001(\t\"Q\n\016WorkerShutdown\022/\n\014" + - "grace_period\030\001 \001(\0132\031.google.protobuf.Dur" + - "ation\022\016\n\006reason\030\002 \001(\t\"H\n\021WorkerShutdownA" + - "ck\0223\n\006status\030\001 \001(\0132#.mxaccess_gateway.v1" + - ".ProtocolStatus\":\n\013WorkerEvent\022+\n\005event\030" + - "\001 \001(\0132\034.mxaccess_gateway.v1.MxEvent\"\245\002\n\017" + - "WorkerHeartbeat\022\031\n\021worker_process_id\030\001 \001" + - "(\005\022.\n\005state\030\002 \001(\0162\037.mxaccess_worker.v1.W" + - "orkerState\022?\n\033last_sta_activity_timestam" + - "p\030\003 \001(\0132\032.google.protobuf.Timestamp\022\035\n\025p" + - "ending_command_count\030\004 \001(\r\022\"\n\032outbound_e" + - "vent_queue_depth\030\005 \001(\r\022\033\n\023last_event_seq" + - "uence\030\006 \001(\004\022&\n\036current_command_correlati" + - "on_id\030\007 \001(\t\"\364\001\n\013WorkerFault\0229\n\010category\030" + - "\001 \001(\0162\'.mxaccess_worker.v1.WorkerFaultCa" + - "tegory\022\026\n\016command_method\030\002 \001(\t\022\024\n\007hresul" + - "t\030\003 \001(\005H\000\210\001\001\022\026\n\016exception_type\030\004 \001(\t\022\032\n\022" + - "diagnostic_message\030\005 \001(\t\022<\n\017protocol_sta" + - "tus\030\006 \001(\0132#.mxaccess_gateway.v1.Protocol" + - "StatusB\n\n\010_hresult*\227\002\n\013WorkerState\022\034\n\030WO" + - "RKER_STATE_UNSPECIFIED\020\000\022\031\n\025WORKER_STATE" + - "_STARTING\020\001\022\034\n\030WORKER_STATE_HANDSHAKING\020" + - "\002\022!\n\035WORKER_STATE_INITIALIZING_STA\020\003\022\026\n\022" + - "WORKER_STATE_READY\020\004\022\"\n\036WORKER_STATE_EXE" + - "CUTING_COMMAND\020\005\022\036\n\032WORKER_STATE_SHUTTIN" + - "G_DOWN\020\006\022\030\n\024WORKER_STATE_STOPPED\020\007\022\030\n\024WO" + - "RKER_STATE_FAULTED\020\010*\307\004\n\023WorkerFaultCate" + - "gory\022%\n!WORKER_FAULT_CATEGORY_UNSPECIFIE" + - "D\020\000\022+\n\'WORKER_FAULT_CATEGORY_INVALID_ARG" + - "UMENTS\020\001\0227\n3WORKER_FAULT_CATEGORY_GATEWA" + - "Y_AUTHENTICATION_FAILED\020\002\022+\n\'WORKER_FAUL" + - "T_CATEGORY_PROTOCOL_MISMATCH\020\003\022,\n(WORKER" + - "_FAULT_CATEGORY_PROTOCOL_VIOLATION\020\004\022+\n\'" + - "WORKER_FAULT_CATEGORY_PIPE_DISCONNECTED\020" + - "\005\0222\n.WORKER_FAULT_CATEGORY_MXACCESS_CREA" + - "TION_FAILED\020\006\0221\n-WORKER_FAULT_CATEGORY_M" + - "XACCESS_COMMAND_FAILED\020\007\022:\n6WORKER_FAULT" + - "_CATEGORY_MXACCESS_EVENT_CONVERSION_FAIL" + - "ED\020\010\022\"\n\036WORKER_FAULT_CATEGORY_STA_HUNG\020\t" + - "\022(\n$WORKER_FAULT_CATEGORY_QUEUE_OVERFLOW" + - "\020\n\022*\n&WORKER_FAULT_CATEGORY_SHUTDOWN_TIM" + - "EOUT\020\013B&\252\002#ZB.MOM.WW.MxGateway.Contracts" + - ".Protob\006proto3" + "buf.Timestamp\022\036\n\026record_info_pin_status\030" + + "\005 \001(\t\"w\n\rWorkerCommand\022/\n\007command\030\001 \001(\0132" + + "\036.mxaccess_gateway.v1.MxCommand\0225\n\021enque" + + "ue_timestamp\030\002 \001(\0132\032.google.protobuf.Tim" + + "estamp\"\201\001\n\022WorkerCommandReply\0222\n\005reply\030\001" + + " \001(\0132#.mxaccess_gateway.v1.MxCommandRepl" + + "y\0227\n\023completed_timestamp\030\002 \001(\0132\032.google." + + "protobuf.Timestamp\"\036\n\014WorkerCancel\022\016\n\006re" + + "ason\030\001 \001(\t\"Q\n\016WorkerShutdown\022/\n\014grace_pe" + + "riod\030\001 \001(\0132\031.google.protobuf.Duration\022\016\n" + + "\006reason\030\002 \001(\t\"H\n\021WorkerShutdownAck\0223\n\006st" + + "atus\030\001 \001(\0132#.mxaccess_gateway.v1.Protoco" + + "lStatus\":\n\013WorkerEvent\022+\n\005event\030\001 \001(\0132\034." + + "mxaccess_gateway.v1.MxEvent\"\245\002\n\017WorkerHe" + + "artbeat\022\031\n\021worker_process_id\030\001 \001(\005\022.\n\005st" + + "ate\030\002 \001(\0162\037.mxaccess_worker.v1.WorkerSta" + + "te\022?\n\033last_sta_activity_timestamp\030\003 \001(\0132" + + "\032.google.protobuf.Timestamp\022\035\n\025pending_c" + + "ommand_count\030\004 \001(\r\022\"\n\032outbound_event_que" + + "ue_depth\030\005 \001(\r\022\033\n\023last_event_sequence\030\006 " + + "\001(\004\022&\n\036current_command_correlation_id\030\007 " + + "\001(\t\"\364\001\n\013WorkerFault\0229\n\010category\030\001 \001(\0162\'." + + "mxaccess_worker.v1.WorkerFaultCategory\022\026" + + "\n\016command_method\030\002 \001(\t\022\024\n\007hresult\030\003 \001(\005H" + + "\000\210\001\001\022\026\n\016exception_type\030\004 \001(\t\022\032\n\022diagnost" + + "ic_message\030\005 \001(\t\022<\n\017protocol_status\030\006 \001(" + + "\0132#.mxaccess_gateway.v1.ProtocolStatusB\n" + + "\n\010_hresult*\227\002\n\013WorkerState\022\034\n\030WORKER_STA" + + "TE_UNSPECIFIED\020\000\022\031\n\025WORKER_STATE_STARTIN" + + "G\020\001\022\034\n\030WORKER_STATE_HANDSHAKING\020\002\022!\n\035WOR" + + "KER_STATE_INITIALIZING_STA\020\003\022\026\n\022WORKER_S" + + "TATE_READY\020\004\022\"\n\036WORKER_STATE_EXECUTING_C" + + "OMMAND\020\005\022\036\n\032WORKER_STATE_SHUTTING_DOWN\020\006" + + "\022\030\n\024WORKER_STATE_STOPPED\020\007\022\030\n\024WORKER_STA" + + "TE_FAULTED\020\010*\307\004\n\023WorkerFaultCategory\022%\n!" + + "WORKER_FAULT_CATEGORY_UNSPECIFIED\020\000\022+\n\'W" + + "ORKER_FAULT_CATEGORY_INVALID_ARGUMENTS\020\001" + + "\0227\n3WORKER_FAULT_CATEGORY_GATEWAY_AUTHEN" + + "TICATION_FAILED\020\002\022+\n\'WORKER_FAULT_CATEGO" + + "RY_PROTOCOL_MISMATCH\020\003\022,\n(WORKER_FAULT_C" + + "ATEGORY_PROTOCOL_VIOLATION\020\004\022+\n\'WORKER_F" + + "AULT_CATEGORY_PIPE_DISCONNECTED\020\005\0222\n.WOR" + + "KER_FAULT_CATEGORY_MXACCESS_CREATION_FAI" + + "LED\020\006\0221\n-WORKER_FAULT_CATEGORY_MXACCESS_" + + "COMMAND_FAILED\020\007\022:\n6WORKER_FAULT_CATEGOR" + + "Y_MXACCESS_EVENT_CONVERSION_FAILED\020\010\022\"\n\036" + + "WORKER_FAULT_CATEGORY_STA_HUNG\020\t\022(\n$WORK" + + "ER_FAULT_CATEGORY_QUEUE_OVERFLOW\020\n\022*\n&WO" + + "RKER_FAULT_CATEGORY_SHUTDOWN_TIMEOUT\020\013B&" + + "\252\002#ZB.MOM.WW.MxGateway.Contracts.Protob\006" + + "proto3" }; descriptor = com.google.protobuf.Descriptors.FileDescriptor .internalBuildGeneratedFileFrom(descriptorData, @@ -12783,7 +13013,7 @@ public final class MxaccessWorker extends com.google.protobuf.GeneratedFile { internal_static_mxaccess_worker_v1_WorkerReady_fieldAccessorTable = new com.google.protobuf.GeneratedMessage.FieldAccessorTable( internal_static_mxaccess_worker_v1_WorkerReady_descriptor, - new java.lang.String[] { "WorkerProcessId", "MxaccessProgid", "MxaccessClsid", "ReadyTimestamp", }); + new java.lang.String[] { "WorkerProcessId", "MxaccessProgid", "MxaccessClsid", "ReadyTimestamp", "RecordInfoPinStatus", }); internal_static_mxaccess_worker_v1_WorkerCommand_descriptor = getDescriptor().getMessageType(4); internal_static_mxaccess_worker_v1_WorkerCommand_fieldAccessorTable = new diff --git a/clients/proto/descriptors/mxaccessgw-client-v1.protoset b/clients/proto/descriptors/mxaccessgw-client-v1.protoset index feabcbe706807fd5790db4c7edcff808e416f140..28e595ed29b783031c4e58c19eaee4e09c9d0017 100644 GIT binary patch delta 3047 zcmYLLU2has7~XScrn@^Uic=`i7C5wgw9pphBOquDQ4&yEpe+;u(@uAX?zG#PGBew4 zDKLB(xB!*!n&_3$TjPaE|ANNojn^h7Ua6Pfcw_K+&pET)?UVO;pYyyQbI$(oPyXwF z^Eba|Ki%hl46XcppZ#j_JAC=8N9>ma&OW;>^2%XU8TW(B;rN^%jK_(a)Z_J*U_Hg* zRuw&g;AuY?k)f5j$L!C7{rPS??^wCqGJEp-muprUU$A#<_Wtt9H|($UZ^JfCgPV?V zf^TjA%@C*RNjaSLM0i;wRZld+XvT|n#9}TA%U&G!H~fkx2G5U-9vwY9GJbma)cFsE zI=Y{X2^h0Q&)|Y`a5@SX7Kf#8?X35(=mID3Dq^YTr<0KzE%r~<{mOMQKR7U}wZxLv zl#wnGLNu(~5XN|fDNO}Ift93MV5a`xn?*e$$;vNxCbVIY#lIn;tm zqJ|%P1LD~1T(T${RS%)6XQ#SAG3%nNAQWzrc%XttcAUBb$k zJm(mLyicY8v=CAPKp+l^9a#)=gFAQWG=zruUL9g7@`iXv1}WItpCRN8?Ri7mupIMD zZdBQ@z&cZCmwFc280NcW3IODW`PK}9IJ|9Vh9EcGKcLeP8sYm@{T64)8__m^DA<{4 zGUScy-6w6dIOe(Bd1a#otaF7mQm=(%#OWFny#k36| zq+N1aQ|Y7NnBzHD*(iWEUf3XQ6v)N|-y;tOfQ<>=T_6xAw(ZJd(3=?8E#-=iSwCpSW|^{QqLy6D&Hhi z08nm~=Q9Li6?M=I5QAK`bE8f}$mi=Rlp;pM_xUQ#q&E4lwq^)C$<)lxw`82xVr3sAd-|;>ohtbd0kH5ETx=7 zCF$tYDggE7;WVmx^KhC}y?HoIs@^;t$IWx051>iSH`8cR^F?*Tl{igme$!@snuY#1 z_?tSy}RC&l(D8H9NtjwkwwkA(M&TS zasv;FGKprTTz;5VF6GppR9#ufGr>&P(Qv37-1;{e-)66pF0s4>MFu&+Zo2G=i>vt- z!`dq@7M5UV2Eo0>u<44Mcg))ipYAH`A5wAi5^mGvJjl1%2D)xZ7?drHQrNW%!rHP^e*{spef~sahCDW8P!pzd&*p<= z{w;fvK4838lQ00UgA22xht%4`5Rl zPyS~NPpWh#2Sj+M&c%cP@s28CjOgJ@VHy!&CTxNG@TD+D^ysBS8WG?gt=j_bBestg zo(y9|j~PACsy{R$U~P)ih#oUM%8HB;Jz@N9twbXNJd{CjpRglzZIdt_r>EIY0f7ss zb_x*Or`b+1j)&)iXAF0uGJ&_p1l*oMke{($dI3ooH{tV_oxoj4Fq?M?+~+Gle#c%M F{~wdLiQoVL delta 2452 zcmYL~%WqXx6voe9XPR}@ca3+-)tp%kP*TR=kvjRONEy)~535^AX7_GxJEZB1MH zG>3_zmX{9fz?qSm0~1a9FPIpaIPy=>7>RM>#OSy7Ui)QWgQ1?kmLui1` zN=PAXfLa}KtaRE@hP;8kGgz8*km^D29a}~K*g(POYZpRwJ!&#`A=`~)(p?bh8=)5C0D^L))pi#IzY*zf zNC=He832?qDrJB;CS`z-HY#OAslMsGYs-kBx)~kfGBO=;9b?pI76zb|bj$n#AbX}j$eWIxr$*j1n#Dh6LResi4ofA0A(zI#&P-f#F{I6OlqC(@nUxcl zPyuP#Y`z?y_1V}SIRue6izCNBW---wybtVbicsB=Sr>UWbFymy+~=s#EDVBhuF38i z1iLxeb&>lxPo~05gHXr3lmX&k+$=6UoB5VjQ+|;!o0=RZ09se0M1iOb4s(QXSkE)(8j^!4pEDPYW0@;=q&7>A$d&Pl}w$R=o z%R#tEuS;5h7A?x(RU}L*^B2494FyEr;`x4VdEljb$@|c>Jm5|)1^d}A@VMp6_7Y}5 zXxUyGOz6*YTe}DmTCw{?h`YK%rmIX4u9%HwYzexXTk>%3=9VUwTcW^KTRpP4SF3rN z50lj+ow7E1wMOQ@gpkkin!T`?AY5xcD8dZwS*I>;4`O_D*KLoPpj_`fDoS*2!=8AA z_^fW^n=3%LVb3^XVd`(%UzP#FO==SV5W>wv=2bx~O8u=YSMsH}l`A7Cw~UgD3Nr7B zvcup;W&xip9ViS#-&6Q~_n~}{GT7;m81c*gtBe@Yj}-N^{He+}s`D$;OXV}=o%(m` zd4|s?BnhK|>J&eaya4E9MHeK9*)YE5&_w(BaIw};^?mqD5aVHVzL*a%_r;U{iNcd= z3eR{%%-xcU83FN*T4OxSpY9o>AB@fzQv&A3cs)Pcs}OTy>=R!M3V4nWJQs|oaa-Ym z&TRWID`qK*Ff1S*Wh=%5_CV2_;)H22_B(|3fx5`oHXGw{dZ=)!tS_D?# convert event args to WorkerEvent - -> enqueue outbound event + -> enqueue outbound event, signal the drain thread -pipe writer thread - -> dequeue replies/events - -> write framed protobuf messages +event drain thread (mxgw-event-drain, dedicated, not thread pool) + -> park on the event queue's wake signal + -> drain a batch of WorkerEvents + -> write them as one framed batch, flushed synchronously + +command replies and heartbeats + -> written through the same frame writer, which orders control frames + ahead of queued events ``` Do not block the STA on pipe writes or gRPC calls. The STA should enqueue results/events and return to pumping messages. +The event drain loop has its own background thread because of the .NET +Framework thread pool. As an async loop, every event woke a pool worker, and +each one spun before parking again; a production trace put that spin at about a +quarter of the worker's CPU at ~240 events/s. The dedicated thread parks on the +signal, and with a synchronous flush the steady-state event path queues no pool +work. The drain loop's task still completes, faults or cancels exactly as the +loop does, so the session's supervision of it is unchanged. See +`docs/MxAccessWorkerInstanceDesign.md`. + ### Message Pump The STA loop must pump Windows messages and service command work. A typical @@ -878,9 +892,19 @@ Worker startup: 1. set apartment state to STA, 2. initialize COM on the STA, -3. instantiate `LMXProxyServerClass` or the installed MXAccess interop class, -4. attach event handlers, -5. send `WorkerReady`. +3. pin the LMXProxy type library and the `MXSTATUS_PROXY` record info for the + process lifetime (once per process, best effort), +4. instantiate `LMXProxyServerClass` or the installed MXAccess interop class, +5. attach event handlers, +6. send `WorkerReady`, carrying the pin outcome in `record_info_pin_status`. + +The pin exists because `lmxproxy.dll` looks up the `MXSTATUS_PROXY` record type +for every `OnDataChange`. With no live reference to it, oleaut32 re-reads the +registry and reloads the type library on each call, which a production trace put +at about 70% of the STA's CPU. Holding one reference makes the lookup a cache +hit. A failed pin is never fatal, since MXAccess behaves the same either way, but +the gateway logs it as a warning and shows it on the dashboard's session details +page, because the worker's own stderr is not captured. Worker shutdown: diff --git a/src/ZB.MOM.WW.MxGateway.Contracts/Generated/MxaccessWorker.cs b/src/ZB.MOM.WW.MxGateway.Contracts/Generated/MxaccessWorker.cs index 28634908..a0cd211a 100644 --- a/src/ZB.MOM.WW.MxGateway.Contracts/Generated/MxaccessWorker.cs +++ b/src/ZB.MOM.WW.MxGateway.Contracts/Generated/MxaccessWorker.cs @@ -49,61 +49,62 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { "CRIXCg9nYXRld2F5X3ZlcnNpb24YAyABKAkSFwoPbWF4X2ZyYW1lX2J5dGVz", "GAQgASgNImkKC1dvcmtlckhlbGxvEhgKEHByb3RvY29sX3ZlcnNpb24YASAB", "KA0SDQoFbm9uY2UYAiABKAkSGQoRd29ya2VyX3Byb2Nlc3NfaWQYAyABKAUS", - "FgoOd29ya2VyX3ZlcnNpb24YBCABKAkijgEKC1dvcmtlclJlYWR5EhkKEXdv", + "FgoOd29ya2VyX3ZlcnNpb24YBCABKAkirgEKC1dvcmtlclJlYWR5EhkKEXdv", "cmtlcl9wcm9jZXNzX2lkGAEgASgFEhcKD214YWNjZXNzX3Byb2dpZBgCIAEo", "CRIWCg5teGFjY2Vzc19jbHNpZBgDIAEoCRIzCg9yZWFkeV90aW1lc3RhbXAY", - "BCABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wIncKDVdvcmtlckNv", - "bW1hbmQSLwoHY29tbWFuZBgBIAEoCzIeLm14YWNjZXNzX2dhdGV3YXkudjEu", - "TXhDb21tYW5kEjUKEWVucXVldWVfdGltZXN0YW1wGAIgASgLMhouZ29vZ2xl", - "LnByb3RvYnVmLlRpbWVzdGFtcCKBAQoSV29ya2VyQ29tbWFuZFJlcGx5EjIK", - "BXJlcGx5GAEgASgLMiMubXhhY2Nlc3NfZ2F0ZXdheS52MS5NeENvbW1hbmRS", - "ZXBseRI3ChNjb21wbGV0ZWRfdGltZXN0YW1wGAIgASgLMhouZ29vZ2xlLnBy", - "b3RvYnVmLlRpbWVzdGFtcCIeCgxXb3JrZXJDYW5jZWwSDgoGcmVhc29uGAEg", - "ASgJIlEKDldvcmtlclNodXRkb3duEi8KDGdyYWNlX3BlcmlvZBgBIAEoCzIZ", - "Lmdvb2dsZS5wcm90b2J1Zi5EdXJhdGlvbhIOCgZyZWFzb24YAiABKAkiSAoR", - "V29ya2VyU2h1dGRvd25BY2sSMwoGc3RhdHVzGAEgASgLMiMubXhhY2Nlc3Nf", - "Z2F0ZXdheS52MS5Qcm90b2NvbFN0YXR1cyI6CgtXb3JrZXJFdmVudBIrCgVl", - "dmVudBgBIAEoCzIcLm14YWNjZXNzX2dhdGV3YXkudjEuTXhFdmVudCKlAgoP", - "V29ya2VySGVhcnRiZWF0EhkKEXdvcmtlcl9wcm9jZXNzX2lkGAEgASgFEi4K", - "BXN0YXRlGAIgASgOMh8ubXhhY2Nlc3Nfd29ya2VyLnYxLldvcmtlclN0YXRl", - "Ej8KG2xhc3Rfc3RhX2FjdGl2aXR5X3RpbWVzdGFtcBgDIAEoCzIaLmdvb2ds", - "ZS5wcm90b2J1Zi5UaW1lc3RhbXASHQoVcGVuZGluZ19jb21tYW5kX2NvdW50", - "GAQgASgNEiIKGm91dGJvdW5kX2V2ZW50X3F1ZXVlX2RlcHRoGAUgASgNEhsK", - "E2xhc3RfZXZlbnRfc2VxdWVuY2UYBiABKAQSJgoeY3VycmVudF9jb21tYW5k", - "X2NvcnJlbGF0aW9uX2lkGAcgASgJIvQBCgtXb3JrZXJGYXVsdBI5CghjYXRl", - "Z29yeRgBIAEoDjInLm14YWNjZXNzX3dvcmtlci52MS5Xb3JrZXJGYXVsdENh", - "dGVnb3J5EhYKDmNvbW1hbmRfbWV0aG9kGAIgASgJEhQKB2hyZXN1bHQYAyAB", - "KAVIAIgBARIWCg5leGNlcHRpb25fdHlwZRgEIAEoCRIaChJkaWFnbm9zdGlj", - "X21lc3NhZ2UYBSABKAkSPAoPcHJvdG9jb2xfc3RhdHVzGAYgASgLMiMubXhh", - "Y2Nlc3NfZ2F0ZXdheS52MS5Qcm90b2NvbFN0YXR1c0IKCghfaHJlc3VsdCqX", - "AgoLV29ya2VyU3RhdGUSHAoYV09SS0VSX1NUQVRFX1VOU1BFQ0lGSUVEEAAS", - "GQoVV09SS0VSX1NUQVRFX1NUQVJUSU5HEAESHAoYV09SS0VSX1NUQVRFX0hB", - "TkRTSEFLSU5HEAISIQodV09SS0VSX1NUQVRFX0lOSVRJQUxJWklOR19TVEEQ", - "AxIWChJXT1JLRVJfU1RBVEVfUkVBRFkQBBIiCh5XT1JLRVJfU1RBVEVfRVhF", - "Q1VUSU5HX0NPTU1BTkQQBRIeChpXT1JLRVJfU1RBVEVfU0hVVFRJTkdfRE9X", - "ThAGEhgKFFdPUktFUl9TVEFURV9TVE9QUEVEEAcSGAoUV09SS0VSX1NUQVRF", - "X0ZBVUxURUQQCCrHBAoTV29ya2VyRmF1bHRDYXRlZ29yeRIlCiFXT1JLRVJf", - "RkFVTFRfQ0FURUdPUllfVU5TUEVDSUZJRUQQABIrCidXT1JLRVJfRkFVTFRf", - "Q0FURUdPUllfSU5WQUxJRF9BUkdVTUVOVFMQARI3CjNXT1JLRVJfRkFVTFRf", - "Q0FURUdPUllfR0FURVdBWV9BVVRIRU5USUNBVElPTl9GQUlMRUQQAhIrCidX", - "T1JLRVJfRkFVTFRfQ0FURUdPUllfUFJPVE9DT0xfTUlTTUFUQ0gQAxIsCihX", - "T1JLRVJfRkFVTFRfQ0FURUdPUllfUFJPVE9DT0xfVklPTEFUSU9OEAQSKwon", - "V09SS0VSX0ZBVUxUX0NBVEVHT1JZX1BJUEVfRElTQ09OTkVDVEVEEAUSMgou", - "V09SS0VSX0ZBVUxUX0NBVEVHT1JZX01YQUNDRVNTX0NSRUFUSU9OX0ZBSUxF", - "RBAGEjEKLVdPUktFUl9GQVVMVF9DQVRFR09SWV9NWEFDQ0VTU19DT01NQU5E", - "X0ZBSUxFRBAHEjoKNldPUktFUl9GQVVMVF9DQVRFR09SWV9NWEFDQ0VTU19F", - "VkVOVF9DT05WRVJTSU9OX0ZBSUxFRBAIEiIKHldPUktFUl9GQVVMVF9DQVRF", - "R09SWV9TVEFfSFVORxAJEigKJFdPUktFUl9GQVVMVF9DQVRFR09SWV9RVUVV", - "RV9PVkVSRkxPVxAKEioKJldPUktFUl9GQVVMVF9DQVRFR09SWV9TSFVURE9X", - "Tl9USU1FT1VUEAtCJqoCI1pCLk1PTS5XVy5NeEdhdGV3YXkuQ29udHJhY3Rz", - "LlByb3RvYgZwcm90bzM=")); + "BCABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEh4KFnJlY29yZF9p", + "bmZvX3Bpbl9zdGF0dXMYBSABKAkidwoNV29ya2VyQ29tbWFuZBIvCgdjb21t", + "YW5kGAEgASgLMh4ubXhhY2Nlc3NfZ2F0ZXdheS52MS5NeENvbW1hbmQSNQoR", + "ZW5xdWV1ZV90aW1lc3RhbXAYAiABKAsyGi5nb29nbGUucHJvdG9idWYuVGlt", + "ZXN0YW1wIoEBChJXb3JrZXJDb21tYW5kUmVwbHkSMgoFcmVwbHkYASABKAsy", + "Iy5teGFjY2Vzc19nYXRld2F5LnYxLk14Q29tbWFuZFJlcGx5EjcKE2NvbXBs", + "ZXRlZF90aW1lc3RhbXAYAiABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0", + "YW1wIh4KDFdvcmtlckNhbmNlbBIOCgZyZWFzb24YASABKAkiUQoOV29ya2Vy", + "U2h1dGRvd24SLwoMZ3JhY2VfcGVyaW9kGAEgASgLMhkuZ29vZ2xlLnByb3Rv", + "YnVmLkR1cmF0aW9uEg4KBnJlYXNvbhgCIAEoCSJIChFXb3JrZXJTaHV0ZG93", + "bkFjaxIzCgZzdGF0dXMYASABKAsyIy5teGFjY2Vzc19nYXRld2F5LnYxLlBy", + "b3RvY29sU3RhdHVzIjoKC1dvcmtlckV2ZW50EisKBWV2ZW50GAEgASgLMhwu", + "bXhhY2Nlc3NfZ2F0ZXdheS52MS5NeEV2ZW50IqUCCg9Xb3JrZXJIZWFydGJl", + "YXQSGQoRd29ya2VyX3Byb2Nlc3NfaWQYASABKAUSLgoFc3RhdGUYAiABKA4y", + "Hy5teGFjY2Vzc193b3JrZXIudjEuV29ya2VyU3RhdGUSPwobbGFzdF9zdGFf", + "YWN0aXZpdHlfdGltZXN0YW1wGAMgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRp", + "bWVzdGFtcBIdChVwZW5kaW5nX2NvbW1hbmRfY291bnQYBCABKA0SIgoab3V0", + "Ym91bmRfZXZlbnRfcXVldWVfZGVwdGgYBSABKA0SGwoTbGFzdF9ldmVudF9z", + "ZXF1ZW5jZRgGIAEoBBImCh5jdXJyZW50X2NvbW1hbmRfY29ycmVsYXRpb25f", + "aWQYByABKAki9AEKC1dvcmtlckZhdWx0EjkKCGNhdGVnb3J5GAEgASgOMicu", + "bXhhY2Nlc3Nfd29ya2VyLnYxLldvcmtlckZhdWx0Q2F0ZWdvcnkSFgoOY29t", + "bWFuZF9tZXRob2QYAiABKAkSFAoHaHJlc3VsdBgDIAEoBUgAiAEBEhYKDmV4", + "Y2VwdGlvbl90eXBlGAQgASgJEhoKEmRpYWdub3N0aWNfbWVzc2FnZRgFIAEo", + "CRI8Cg9wcm90b2NvbF9zdGF0dXMYBiABKAsyIy5teGFjY2Vzc19nYXRld2F5", + "LnYxLlByb3RvY29sU3RhdHVzQgoKCF9ocmVzdWx0KpcCCgtXb3JrZXJTdGF0", + "ZRIcChhXT1JLRVJfU1RBVEVfVU5TUEVDSUZJRUQQABIZChVXT1JLRVJfU1RB", + "VEVfU1RBUlRJTkcQARIcChhXT1JLRVJfU1RBVEVfSEFORFNIQUtJTkcQAhIh", + "Ch1XT1JLRVJfU1RBVEVfSU5JVElBTElaSU5HX1NUQRADEhYKEldPUktFUl9T", + "VEFURV9SRUFEWRAEEiIKHldPUktFUl9TVEFURV9FWEVDVVRJTkdfQ09NTUFO", + "RBAFEh4KGldPUktFUl9TVEFURV9TSFVUVElOR19ET1dOEAYSGAoUV09SS0VS", + "X1NUQVRFX1NUT1BQRUQQBxIYChRXT1JLRVJfU1RBVEVfRkFVTFRFRBAIKscE", + "ChNXb3JrZXJGYXVsdENhdGVnb3J5EiUKIVdPUktFUl9GQVVMVF9DQVRFR09S", + "WV9VTlNQRUNJRklFRBAAEisKJ1dPUktFUl9GQVVMVF9DQVRFR09SWV9JTlZB", + "TElEX0FSR1VNRU5UUxABEjcKM1dPUktFUl9GQVVMVF9DQVRFR09SWV9HQVRF", + "V0FZX0FVVEhFTlRJQ0FUSU9OX0ZBSUxFRBACEisKJ1dPUktFUl9GQVVMVF9D", + "QVRFR09SWV9QUk9UT0NPTF9NSVNNQVRDSBADEiwKKFdPUktFUl9GQVVMVF9D", + "QVRFR09SWV9QUk9UT0NPTF9WSU9MQVRJT04QBBIrCidXT1JLRVJfRkFVTFRf", + "Q0FURUdPUllfUElQRV9ESVNDT05ORUNURUQQBRIyCi5XT1JLRVJfRkFVTFRf", + "Q0FURUdPUllfTVhBQ0NFU1NfQ1JFQVRJT05fRkFJTEVEEAYSMQotV09SS0VS", + "X0ZBVUxUX0NBVEVHT1JZX01YQUNDRVNTX0NPTU1BTkRfRkFJTEVEEAcSOgo2", + "V09SS0VSX0ZBVUxUX0NBVEVHT1JZX01YQUNDRVNTX0VWRU5UX0NPTlZFUlNJ", + "T05fRkFJTEVEEAgSIgoeV09SS0VSX0ZBVUxUX0NBVEVHT1JZX1NUQV9IVU5H", + "EAkSKAokV09SS0VSX0ZBVUxUX0NBVEVHT1JZX1FVRVVFX09WRVJGTE9XEAoS", + "KgomV09SS0VSX0ZBVUxUX0NBVEVHT1JZX1NIVVRET1dOX1RJTUVPVVQQC0Im", + "qgIjWkIuTU9NLldXLk14R2F0ZXdheS5Db250cmFjdHMuUHJvdG9iBnByb3Rv", + "Mw==")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { global::Google.Protobuf.WellKnownTypes.DurationReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.TimestampReflection.Descriptor, global::ZB.MOM.WW.MxGateway.Contracts.Proto.MxaccessGatewayReflection.Descriptor, }, new pbr::GeneratedClrTypeInfo(new[] {typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerState), typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerFaultCategory), }, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerEnvelope), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerEnvelope.Parser, new[]{ "ProtocolVersion", "SessionId", "Sequence", "CorrelationId", "GatewayHello", "WorkerHello", "WorkerReady", "WorkerCommand", "WorkerCommandReply", "WorkerCancel", "WorkerShutdown", "WorkerShutdownAck", "WorkerEvent", "WorkerHeartbeat", "WorkerFault" }, new[]{ "Body" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.GatewayHello), global::ZB.MOM.WW.MxGateway.Contracts.Proto.GatewayHello.Parser, new[]{ "SupportedProtocolVersion", "Nonce", "GatewayVersion", "MaxFrameBytes" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerHello), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerHello.Parser, new[]{ "ProtocolVersion", "Nonce", "WorkerProcessId", "WorkerVersion" }, null, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerReady), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerReady.Parser, new[]{ "WorkerProcessId", "MxaccessProgid", "MxaccessClsid", "ReadyTimestamp" }, null, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerReady), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerReady.Parser, new[]{ "WorkerProcessId", "MxaccessProgid", "MxaccessClsid", "ReadyTimestamp", "RecordInfoPinStatus" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerCommand), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerCommand.Parser, new[]{ "Command", "EnqueueTimestamp" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerCommandReply), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerCommandReply.Parser, new[]{ "Reply", "CompletedTimestamp" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerCancel), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerCancel.Parser, new[]{ "Reason" }, null, null, null, null), @@ -1744,6 +1745,7 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { mxaccessProgid_ = other.mxaccessProgid_; mxaccessClsid_ = other.mxaccessClsid_; readyTimestamp_ = other.readyTimestamp_ != null ? other.readyTimestamp_.Clone() : null; + recordInfoPinStatus_ = other.recordInfoPinStatus_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -1801,6 +1803,26 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { } } + /// Field number for the "record_info_pin_status" field. + public const int RecordInfoPinStatusFieldNumber = 5; + private string recordInfoPinStatus_ = ""; + /// + /// Outcome of the worker's process-wide MXSTATUS_PROXY record-info pin + /// (MxStatusProxyRecordInfoPin): "pinned {library-guid} vM.m record {record-guid}" on + /// success, or the failing call and HRESULT / exception type otherwise. Empty when the + /// worker made no attempt (an older worker, or a runtime that never created the + /// MXAccess COM object). Diagnostic only: the gateway logs it and shows it on the + /// dashboard, and never gates readiness on it, because the pin is a cache warm-up. + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] + public string RecordInfoPinStatus { + get { return recordInfoPinStatus_; } + set { + recordInfoPinStatus_ = pb::ProtoPreconditions.CheckNotNull(value, "value"); + } + } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] public override bool Equals(object other) { @@ -1820,6 +1842,7 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { if (MxaccessProgid != other.MxaccessProgid) return false; if (MxaccessClsid != other.MxaccessClsid) return false; if (!object.Equals(ReadyTimestamp, other.ReadyTimestamp)) return false; + if (RecordInfoPinStatus != other.RecordInfoPinStatus) return false; return Equals(_unknownFields, other._unknownFields); } @@ -1831,6 +1854,7 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { if (MxaccessProgid.Length != 0) hash ^= MxaccessProgid.GetHashCode(); if (MxaccessClsid.Length != 0) hash ^= MxaccessClsid.GetHashCode(); if (readyTimestamp_ != null) hash ^= ReadyTimestamp.GetHashCode(); + if (RecordInfoPinStatus.Length != 0) hash ^= RecordInfoPinStatus.GetHashCode(); if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); } @@ -1865,6 +1889,10 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { output.WriteRawTag(34); output.WriteMessage(ReadyTimestamp); } + if (RecordInfoPinStatus.Length != 0) { + output.WriteRawTag(42); + output.WriteString(RecordInfoPinStatus); + } if (_unknownFields != null) { _unknownFields.WriteTo(output); } @@ -1891,6 +1919,10 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { output.WriteRawTag(34); output.WriteMessage(ReadyTimestamp); } + if (RecordInfoPinStatus.Length != 0) { + output.WriteRawTag(42); + output.WriteString(RecordInfoPinStatus); + } if (_unknownFields != null) { _unknownFields.WriteTo(ref output); } @@ -1913,6 +1945,9 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { if (readyTimestamp_ != null) { size += 1 + pb::CodedOutputStream.ComputeMessageSize(ReadyTimestamp); } + if (RecordInfoPinStatus.Length != 0) { + size += 1 + pb::CodedOutputStream.ComputeStringSize(RecordInfoPinStatus); + } if (_unknownFields != null) { size += _unknownFields.CalculateSize(); } @@ -1940,6 +1975,9 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { } ReadyTimestamp.MergeFrom(other.ReadyTimestamp); } + if (other.RecordInfoPinStatus.Length != 0) { + RecordInfoPinStatus = other.RecordInfoPinStatus; + } _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -1978,6 +2016,10 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { input.ReadMessage(ReadyTimestamp); break; } + case 42: { + RecordInfoPinStatus = input.ReadString(); + break; + } } } #endif @@ -2016,6 +2058,10 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto { input.ReadMessage(ReadyTimestamp); break; } + case 42: { + RecordInfoPinStatus = input.ReadString(); + break; + } } } } diff --git a/src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_worker.proto b/src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_worker.proto index da894a32..13764cb1 100644 --- a/src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_worker.proto +++ b/src/ZB.MOM.WW.MxGateway.Contracts/Protos/mxaccess_worker.proto @@ -65,6 +65,13 @@ message WorkerReady { string mxaccess_progid = 2; string mxaccess_clsid = 3; google.protobuf.Timestamp ready_timestamp = 4; + // Outcome of the worker's process-wide MXSTATUS_PROXY record-info pin + // (MxStatusProxyRecordInfoPin): "pinned {library-guid} vM.m record {record-guid}" on + // success, or the failing call and HRESULT / exception type otherwise. Empty when the + // worker made no attempt (an older worker, or a runtime that never created the + // MXAccess COM object). Diagnostic only: the gateway logs it and shows it on the + // dashboard, and never gates readiness on it, because the pin is a cache warm-up. + string record_info_pin_status = 5; } message WorkerCommand { diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Pages/SessionDetailsPage.razor b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Pages/SessionDetailsPage.razor index 7e3c2d28..480e6331 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Pages/SessionDetailsPage.razor +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/Components/Pages/SessionDetailsPage.razor @@ -102,6 +102,7 @@ else Process id@(CurrentSession.WorkerProcessId?.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? "-") State Last heartbeat@DashboardDisplay.DateTime(CurrentSession.LastWorkerHeartbeatAt) + Record-info pin@DashboardDisplay.Text(CurrentSession.WorkerRecordInfoPinStatus) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSessionSummary.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSessionSummary.cs index c208a52e..316f9ff3 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSessionSummary.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSessionSummary.cs @@ -17,4 +17,5 @@ public sealed record DashboardSessionSummary( WorkerClientState? WorkerState, DateTimeOffset? LastWorkerHeartbeatAt, long EventsReceived, - string? LastFault); + string? LastFault, + string? WorkerRecordInfoPinStatus = null); diff --git a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs index 5ffffcca..1b1be7dc 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Dashboard/DashboardSnapshotService.cs @@ -285,7 +285,8 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService WorkerState: workerClient?.State, LastWorkerHeartbeatAt: workerClient?.LastHeartbeatAt, EventsReceived: eventsReceived, - LastFault: DashboardRedactor.Redact(session.FinalFault)); + LastFault: DashboardRedactor.Redact(session.FinalFault), + WorkerRecordInfoPinStatus: workerClient?.RecordInfoPinStatus); } private static DashboardWorkerSummary CreateWorkerSummary(GatewaySession session) diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/IWorkerClient.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/IWorkerClient.cs index f595d84f..26facf4e 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Workers/IWorkerClient.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/IWorkerClient.cs @@ -27,6 +27,14 @@ public interface IWorkerClient : IAsyncDisposable /// Task? GetStateChangedSignal() => null; + /// + /// Outcome of the worker's MXSTATUS_PROXY record-info pin, as reported in + /// , or before the handshake or when the + /// worker did not report one. Diagnostic only; the default implementation returns + /// . + /// + string? RecordInfoPinStatus => null; + /// UTC timestamp of the most recent heartbeat from the worker. DateTimeOffset LastHeartbeatAt { get; } diff --git a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs index c2aa6978..71d3aebc 100644 --- a/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs +++ b/src/ZB.MOM.WW.MxGateway.Server/Workers/WorkerClient.cs @@ -13,6 +13,8 @@ namespace ZB.MOM.WW.MxGateway.Server.Workers; public sealed class WorkerClient : IWorkerClient { private const string GatewayVersionFallback = "unknown"; + // Prefix of a successful MxStatusProxyRecordInfoPin.Status in the worker; anything else is a failure. + private const string RecordInfoPinnedPrefix = "pinned "; private static readonly TimeSpan DisposeTaskTimeout = TimeSpan.FromSeconds(5); private readonly object _syncRoot = new(); private readonly WorkerClientConnection _connection; @@ -55,6 +57,7 @@ public sealed class WorkerClient : IWorkerClient private TaskCompletionSource _stateChanged = new(TaskCreationOptions.RunContinuationsAsynchronously); private DateTimeOffset _lastHeartbeatAt; private int? _processId; + private string? _recordInfoPinStatus; private int _eventQueueDepth; private int _eventsReaderClaimed; private Task? _readLoopTask; @@ -161,6 +164,9 @@ public sealed class WorkerClient : IWorkerClient } } + /// + public string? RecordInfoPinStatus => Volatile.Read(ref _recordInfoPinStatus); + /// public DateTimeOffset LastHeartbeatAt { @@ -1074,6 +1080,45 @@ public sealed class WorkerClient : IWorkerClient DateTimeOffset readyAt = _timeProvider.GetUtcNow(); DateTimeOffset launchedAt = _connection.ProcessHandle?.LaunchedAt ?? readyAt; _metrics?.WorkerStarted(readyAt - launchedAt); + RecordRecordInfoPinStatus(ready.RecordInfoPinStatus); + } + + /// + /// Records and logs the worker's MXSTATUS_PROXY record-info pin outcome from + /// . + /// + /// + /// The worker logs the outcome too, but to its own stderr, which the gateway does not + /// capture; this is the line operators actually see. A failed pin is a warning rather than a + /// fault: MXAccess still works, but every data change then pays a type-library reload, which + /// is the dominant worker CPU cost at high event rates. An empty status means an older + /// worker, so nothing is logged. + /// + /// The status string the worker reported; empty when absent. + private void RecordRecordInfoPinStatus(string status) + { + if (string.IsNullOrEmpty(status)) + { + return; + } + + Volatile.Write(ref _recordInfoPinStatus, status); + if (status.StartsWith(RecordInfoPinnedPrefix, StringComparison.Ordinal)) + { + _logger.LogInformation( + "Worker {ProcessId} for session {SessionId} pinned the MXSTATUS_PROXY record info: {RecordInfoPinStatus}", + ProcessId, + SessionId, + status); + } + else + { + _logger.LogWarning( + "Worker {ProcessId} for session {SessionId} could not pin the MXSTATUS_PROXY record info; each data change will reload the MXAccess type library: {RecordInfoPinStatus}", + ProcessId, + SessionId, + status); + } } /// Updates the heartbeat timestamp and process ID. diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotServiceTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotServiceTests.cs index acaf569d..23d016e1 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotServiceTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Dashboard/DashboardSnapshotServiceTests.cs @@ -182,6 +182,31 @@ public sealed class DashboardSnapshotServiceTests Assert.Equal("worker pipe disconnected", fault.Message); } + /// + /// The worker's record-info pin outcome, reported in WorkerReady, reaches the session + /// summary the session-details page renders, since the worker's own log line is not captured. + /// + [Fact] + public void GetSnapshot_ProjectsWorkerRecordInfoPinStatus() + { + const string pinStatus = "LoadRegTypeLib failed (HRESULT 0x8002801D)"; + SessionRegistry registry = new(); + GatewaySession session = CreateSession( + "session-pin", + "client-one", + DateTimeOffset.Parse("2026-04-26T10:00:00Z", CultureInfo.InvariantCulture)); + session.AttachWorkerClient( + new FakeWorkerClient("session-pin", 1301, WorkerClientState.Ready) { RecordInfoPinStatus = pinStatus }); + session.MarkReady(); + registry.TryAdd(session); + using GatewayMetrics metrics = new(); + DashboardSnapshotService service = CreateService(registry, metrics); + + DashboardSessionSummary summary = Assert.Single(service.GetSnapshot().Sessions); + + Assert.Equal(pinStatus, summary.WorkerRecordInfoPinStatus); + } + /// /// Verifies snapshot metrics include the cumulative alarm provider switch count. /// @@ -929,6 +954,9 @@ public sealed class DashboardSnapshotServiceTests /// public WorkerClientState State { get; private set; } = state; + /// + public string? RecordInfoPinStatus { get; init; } + /// public DateTimeOffset LastHeartbeatAt { get; } = DateTimeOffset.Parse("2026-04-26T10:02:00Z", CultureInfo.InvariantCulture); diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/Fakes/FakeWorkerHarness.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/Fakes/FakeWorkerHarness.cs index ad6f02c8..11d778a1 100644 --- a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/Fakes/FakeWorkerHarness.cs +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/Fakes/FakeWorkerHarness.cs @@ -1,6 +1,7 @@ using System.Buffers.Binary; using System.IO.Pipes; using Google.Protobuf.WellKnownTypes; +using Microsoft.Extensions.Logging; using ZB.MOM.WW.MxGateway.Contracts; using ZB.MOM.WW.MxGateway.Contracts.Proto; using ZB.MOM.WW.MxGateway.Server.Metrics; @@ -108,11 +109,13 @@ public sealed class FakeWorkerHarness : IAsyncDisposable /// Configuration options for the worker client. /// Gateway metrics collector. /// Time provider for timestamps. + /// Logger for the worker client; null uses a no-op logger. /// A configured worker client connected to this harness. public WorkerClient CreateClient( WorkerClientOptions? options = null, GatewayMetrics? metrics = null, - TimeProvider? timeProvider = null) + TimeProvider? timeProvider = null, + ILogger? logger = null) { if (_gatewayStream is null) { @@ -125,7 +128,7 @@ public sealed class FakeWorkerHarness : IAsyncDisposable _gatewayStream, _frameOptions); - return new WorkerClient(connection, options, metrics, timeProvider); + return new WorkerClient(connection, options, metrics, timeProvider, logger); } /// Completes the worker startup handshake by reading the gateway hello and sending worker hello and ready. @@ -229,12 +232,14 @@ public sealed class FakeWorkerHarness : IAsyncDisposable /// MXAccess COM ProgID. /// MXAccess COM CLSID. /// Token to cancel the asynchronous operation. + /// Record-info pin outcome to report; empty mimics an older worker. /// A task that represents the asynchronous operation. public async Task SendWorkerReadyAsync( int workerProcessId = DefaultWorkerProcessId, string mxaccessProgid = "LMXProxy.LMXProxyServer.1", string mxaccessClsid = "{C30B52F5-2CB5-4760-AF0A-3A344A7EB5DC}", - CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default, + string recordInfoPinStatus = "") { await _writer.WriteAsync( CreateEnvelope( @@ -245,6 +250,7 @@ public sealed class FakeWorkerHarness : IAsyncDisposable MxaccessProgid = mxaccessProgid, MxaccessClsid = mxaccessClsid, ReadyTimestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow), + RecordInfoPinStatus = recordInfoPinStatus, }), cancellationToken).ConfigureAwait(false); } diff --git a/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientRecordInfoPinTests.cs b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientRecordInfoPinTests.cs new file mode 100644 index 00000000..2a6f7592 --- /dev/null +++ b/src/ZB.MOM.WW.MxGateway.Tests/Gateway/Workers/WorkerClientRecordInfoPinTests.cs @@ -0,0 +1,96 @@ +using Microsoft.Extensions.Logging; +using ZB.MOM.WW.MxGateway.Contracts.Proto; +using ZB.MOM.WW.MxGateway.Server.Workers; +using ZB.MOM.WW.MxGateway.Tests.Gateway.Workers.Fakes; +using ZB.MOM.WW.MxGateway.Tests.TestSupport; + +namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Workers; + +/// +/// The worker reports its MXSTATUS_PROXY record-info pin outcome in +/// because the gateway never sees the worker's own stderr log. These tests pin how the gateway +/// surfaces it: kept on the client, logged at Information on success and Warning on failure, +/// and silently absent for an older worker that sends no status. +/// +public sealed class WorkerClientRecordInfoPinTests +{ + private const string PinnedStatus = + "pinned {77e896ec-b3e3-4de4-b96f-f32570164211} v3.2 record {a162d9e4-9019-42f3-a087-79264007e542}"; + + private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5); + + /// A successful pin is recorded on the client and logged at Information. + /// A task that represents the asynchronous operation. + [Fact] + public async Task StartAsync_WorkerReportsPinned_RecordsStatusAndLogsInformation() + { + RecordingLogger logger = new(); + await using FakeWorkerHarness harness = await FakeWorkerHarness.CreateConnectedPairAsync(); + await using WorkerClient client = harness.CreateClient(logger: logger); + + await StartWithReadyAsync(client, harness, PinnedStatus); + + Assert.Equal(PinnedStatus, client.RecordInfoPinStatus); + (LogLevel level, string message, _) = Assert.Single( + logger.Entries, + entry => entry.Message.Contains("record info", StringComparison.Ordinal)); + Assert.Equal(LogLevel.Information, level); + Assert.Contains(PinnedStatus, message, StringComparison.Ordinal); + Assert.Contains(FakeWorkerHarness.DefaultSessionId, message, StringComparison.Ordinal); + } + + /// + /// A failed pin still lets the worker reach Ready (the pin is a cache warm-up, not a + /// prerequisite) but is logged at Warning so the per-event CPU cost it implies is visible. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task StartAsync_WorkerReportsPinFailure_StaysReadyAndLogsWarning() + { + const string failedStatus = "LoadRegTypeLib failed (HRESULT 0x8002801D)"; + RecordingLogger logger = new(); + await using FakeWorkerHarness harness = await FakeWorkerHarness.CreateConnectedPairAsync(); + await using WorkerClient client = harness.CreateClient(logger: logger); + + await StartWithReadyAsync(client, harness, failedStatus); + + Assert.Equal(WorkerClientState.Ready, client.State); + Assert.Equal(failedStatus, client.RecordInfoPinStatus); + (LogLevel level, string message, _) = Assert.Single( + logger.Entries, + entry => entry.Message.Contains("record info", StringComparison.Ordinal)); + Assert.Equal(LogLevel.Warning, level); + Assert.Contains(failedStatus, message, StringComparison.Ordinal); + } + + /// An older worker sends no status: nothing is recorded and nothing is logged about it. + /// A task that represents the asynchronous operation. + [Fact] + public async Task StartAsync_WorkerReportsNoStatus_RecordsNullAndDoesNotLog() + { + RecordingLogger logger = new(); + await using FakeWorkerHarness harness = await FakeWorkerHarness.CreateConnectedPairAsync(); + await using WorkerClient client = harness.CreateClient(logger: logger); + + await StartWithReadyAsync(client, harness, recordInfoPinStatus: string.Empty); + + Assert.Equal(WorkerClientState.Ready, client.State); + Assert.Null(client.RecordInfoPinStatus); + Assert.DoesNotContain( + logger.Entries, + entry => entry.Message.Contains("record info", StringComparison.Ordinal)); + } + + private static async Task StartWithReadyAsync( + WorkerClient client, + FakeWorkerHarness harness, + string recordInfoPinStatus) + { + Task startTask = client.StartAsync(CancellationToken.None); + WorkerEnvelope gatewayHello = await harness.ReadGatewayEnvelopeAsync().WaitAsync(TestTimeout); + Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, gatewayHello.BodyCase); + await harness.SendWorkerHelloAsync().WaitAsync(TestTimeout); + await harness.SendWorkerReadyAsync(recordInfoPinStatus: recordInfoPinStatus).WaitAsync(TestTimeout); + await startTask.WaitAsync(TestTimeout); + } +} diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs index 404c01fc..675e4aff 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/Ipc/WorkerPipeSessionTests.cs @@ -406,6 +406,151 @@ public sealed class WorkerPipeSessionTests await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token); } + /// + /// The drain loop's task mirrors the dedicated thread's outcome: cancelling its token ends + /// the task Canceled (not faulted, not still running), which is what lets the message loop's + /// teardown treat it as an ordinary stop. Exercised directly, before any runtime session is + /// attached, so the loop is in its runtime-less poll and only the cancellation can end it. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task RunEventDrainLoopAsync_WhenCancelled_EndsCanceled() + { + WorkerFrameProtocolOptions options = CreateOptions(); + using MemoryStream inbound = new(); + using MemoryStream outbound = new(); + WorkerPipeSession session = CreateSession(inbound, outbound, options); + using CancellationTokenSource stop = new(); + + Task drainTask = session.RunEventDrainLoopAsync(stop.Token); + Assert.False(drainTask.IsCompleted); + stop.Cancel(); + + Task completedTask = await Task.WhenAny(drainTask, Task.Delay(TimeSpan.FromSeconds(5))); + Assert.Same(drainTask, completedTask); + Assert.True(drainTask.IsCanceled, $"Expected Canceled but the drain task ended {drainTask.Status}."); + } + + /// + /// Cancelling the session while the dedicated drain thread is parked on the event signal + /// ends as cancelled, disposes the runtime, and lets + /// the drain thread exit. Teardown must not log the drain task as faulted or overrunning its + /// stop budget: a cancelled drain is a clean stop. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task RunAsync_WhenCancelledWhileDrainThreadParked_EndsCanceledAndDrainThreadExits() + { + using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10)); + using CancellationTokenSource sessionCancellation = new(); + using PipePair pipePair = await PipePair.CreateAsync(timeout.Token); + RecordingWorkerLogger logger = new(); + FakeRuntimeSession runtime = new() + { + WaitForEventsOnSignalOnly = true, + }; + WorkerPipeSession session = CreatePipeSession( + pipePair.WorkerStream, + runtime, + new WorkerPipeSessionOptions + { + HeartbeatInterval = TimeSpan.FromMilliseconds(100), + HeartbeatGrace = TimeSpan.FromSeconds(5), + }, + logger); + Task runTask = session.RunAsync(sessionCancellation.Token); + await CompleteGatewayHandshakeAsync(pipePair, timeout.Token); + Thread drainThread = await WaitForDrainThreadAsync(runtime, timeout.Token); + + sessionCancellation.Cancel(); + + Task completedTask = await Task.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(5), timeout.Token)); + Assert.Same(runTask, completedTask); + await Assert.ThrowsAnyAsync(async () => await runTask); + Assert.True(runtime.Disposed); + Assert.True(drainThread.Join(TimeSpan.FromSeconds(5)), "The dedicated drain thread did not exit."); + Assert.DoesNotContain( + logger.Events, + entry => entry.Fields.TryGetValue("task", out object? task) && (task as string) == "EventDrain"); + } + + /// + /// An unexpected exception inside the loop on the dedicated drain thread faults the drain + /// task with that same exception, so the message loop's Task.WhenAny supervision ends + /// with it exactly as it did when the loop was an + /// async method: the session stops, the runtime is disposed, and the thread exits. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task RunAsync_WhenDrainLoopThrows_FaultsSessionWithTheSameException() + { + using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10)); + using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token); + FakeRuntimeSession runtime = new() + { + WaitForEventsOnSignalOnly = true, + }; + WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime); + Task runTask = session.RunAsync(cancellation.Token); + await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token); + Thread drainThread = await WaitForDrainThreadAsync(runtime, cancellation.Token); + + InvalidOperationException scripted = new("scripted drain-loop failure"); + runtime.DrainEventsException = scripted; + // Wake the parked thread; its next pass calls DrainEvents, which now throws. + runtime.EnqueueEvent(CreateWorkerEvent(sequence: 4)); + + Task completedTask = await Task.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(5), cancellation.Token)); + Assert.Same(runTask, completedTask); + InvalidOperationException observed = + await Assert.ThrowsAsync(async () => await runTask); + Assert.Same(scripted, observed); + Assert.True(runtime.Disposed); + Assert.True(drainThread.Join(TimeSpan.FromSeconds(5)), "The dedicated drain thread did not exit."); + } + + /// + /// The handshake copies the process-wide record-info pin outcome into WorkerReady, so + /// the gateway can log it: the worker's own log goes to stderr, which the gateway does not + /// capture. Whether the pin succeeds depends on the host, so only the hand-off is asserted. + /// + /// A task that represents the asynchronous operation. + [Fact] + public async Task CompleteStartupHandshakeAsync_ReportsRecordInfoPinStatusInWorkerReady() + { + _ = MxStatusProxyRecordInfoPin.EnsurePinned(); + string? pinStatus = MxStatusProxyRecordInfoPin.Status; + Assert.False(string.IsNullOrEmpty(pinStatus)); + WorkerFrameProtocolOptions options = CreateOptions(); + using MemoryStream inbound = new(); + await new WorkerFrameWriter(inbound, options).WriteAsync(CreateGatewayHelloEnvelope()); + inbound.Position = 0; + using MemoryStream outbound = new(); + WorkerPipeSession session = CreateSession(inbound, outbound, options); + + await session.CompleteStartupHandshakeAsync( + _ => Task.FromResult(new WorkerReady { WorkerProcessId = 1234 })); + + WorkerEnvelope[] written = ReadWrittenFrames(outbound, options); + Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerReady, written[1].BodyCase); + Assert.Equal(pinStatus, written[1].WorkerReady.RecordInfoPinStatus); + } + + // Waits until the drain loop has parked on the fake's event signal at least once and returns the + // thread it parked on: the dedicated drain thread. + private static async Task WaitForDrainThreadAsync( + FakeRuntimeSession runtime, + CancellationToken cancellationToken) + { + Thread? drainThread; + while ((drainThread = runtime.LastWaitForEventsThread) is null) + { + await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationToken).ConfigureAwait(false); + } + + return drainThread; + } + /// /// Verifies that a Ping control command is answered on the worker side /// (not dispatched to the STA) with an OK reply that echoes the ping diff --git a/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs index 83f93cf8..5d5fa18f 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker.Tests/TestSupport/FakeRuntimeSession.cs @@ -42,6 +42,8 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession // disposed SemaphoreSlim would turn that shutdown into an ObjectDisposedException. private readonly SemaphoreSlim eventSignal = new(0, 1); private TimeSpan? lastWaitForEventsTimeout; + private volatile Exception? drainEventsException; + private volatile Thread? lastWaitForEventsThread; private bool refreshStaActivityOnCapture; private WorkerRuntimeHeartbeatSnapshot snapshot = new( DateTimeOffset.UtcNow, @@ -235,9 +237,25 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession /// public int? LastDrainMaxTotalBytes { get; private set; } + /// + /// Gets or sets an exception every call throws, standing in + /// for an unexpected failure inside the event drain loop. Safe to set from the test thread + /// while the drain thread is running. + /// + public Exception? DrainEventsException + { + get => drainEventsException; + set => drainEventsException = value; + } + /// public IReadOnlyList DrainEvents(uint maxEvents) { + if (drainEventsException is { } drainException) + { + throw drainException; + } + if (SuppressDrainForBatchSize is uint suppressed && maxEvents == suppressed) { return Array.Empty(); @@ -393,9 +411,16 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession /// Gets whether the most recent call ran on a thread-pool thread. public bool? LastWaitForEventsOnThreadPool { get; private set; } + /// + /// Gets the thread that made the most recent call, so a test can + /// assert the dedicated drain thread has exited after the session stops. + /// + public Thread? LastWaitForEventsThread => lastWaitForEventsThread; + /// public void WaitForEvents(TimeSpan timeout, CancellationToken cancellationToken) { + lastWaitForEventsThread = Thread.CurrentThread; LastWaitForEventsThreadName = Thread.CurrentThread.Name; LastWaitForEventsOnThreadPool = Thread.CurrentThread.IsThreadPoolThread; // Same knobs as the async form (signal-only, recorded timeout, backing queue); the drain diff --git a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs index 112176e3..c2561d5f 100644 --- a/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs +++ b/src/ZB.MOM.WW.MxGateway.Worker/Ipc/WorkerPipeSession.cs @@ -360,6 +360,14 @@ public sealed class WorkerPipeSession _logger?.Information( "WorkerRecordInfoPin", new Dictionary { ["status"] = pinStatus }); + + // The gateway does not capture worker stderr, so the log line above is rarely + // seen; WorkerReady carries the outcome to the gateway, which logs it and shows it + // on the dashboard. A runtime that already filled the field keeps its own value. + if (string.IsNullOrEmpty(ready.RecordInfoPinStatus)) + { + ready.RecordInfoPinStatus = pinStatus; + } } await WriteWorkerReadyAsync(ready, cancellationToken).ConfigureAwait(false); @@ -609,8 +617,12 @@ public sealed class WorkerPipeSession /// there. A dedicated thread parks on the signal instead, and with a synchronous flush the /// steady-state event path schedules no managed thread-pool work at all. The pipe write itself /// stays overlapped; its completion runs on an I/O completion thread, not a worker thread. + /// Internal so the completion mapping can be tested directly; production code calls it only + /// from . /// - private Task RunEventDrainLoopAsync(CancellationToken cancellationToken) + /// Token that stops the loop; the returned task then ends Canceled. + /// A task that mirrors the loop's outcome. + internal Task RunEventDrainLoopAsync(CancellationToken cancellationToken) { TaskCompletionSource completion = new(TaskCreationOptions.RunContinuationsAsynchronously); Thread thread = new(() => @@ -618,7 +630,18 @@ public sealed class WorkerPipeSession try { RunEventDrainLoop(cancellationToken); - completion.TrySetResult(true); + + // The loop returns only once its token is cancelled. Report that as Canceled + // whether the loop noticed at its head or inside a wait (which throws), so a + // stopped drain has one outcome rather than a timing-dependent pair. + if (cancellationToken.IsCancellationRequested) + { + completion.TrySetCanceled(cancellationToken); + } + else + { + completion.TrySetResult(true); + } } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) {