feat(worker,gateway): report record-info pin outcome in WorkerReady; drain-thread completion tests
The worker logged its MXSTATUS_PROXY record-info pin outcome only to its own stderr, which the gateway's launcher does not capture, so operators never saw whether the pin took. - Contract (additive, internal IPC): WorkerReady.record_info_pin_status = 5. No protocol version bump; an older peer ignores it or sends it empty. Regenerated Contracts/Generated, the client protoset, the Rust vendored proto, and the Go/Python/Java worker bindings (real delta only). - Worker: the handshake copies MxStatusProxyRecordInfoPin.Status into WorkerReady unless the runtime already set it. - Gateway: WorkerClient records it (IWorkerClient.RecordInfoPinStatus, default null) and logs it at Information when pinned, Warning otherwise; empty is not logged. Shown as "Record-info pin" in the dashboard session details Worker table via DashboardSessionSummary.WorkerRecordInfoPinStatus. - Tests: drain-thread cancellation (task ends Canceled; RunAsync cancels, runtime disposed, thread exits, no EventDrain stop-failure log) and fault (exception on the thread faults RunAsync with the same instance) paths; WorkerReady carries the pin status; gateway logging/recording and the dashboard projection. - Docs: gateway.md STA thread model (drain thread, pin step in COM lifetime), MxAccessWorkerInstanceDesign, GatewayProcessDesign, GatewayDashboardDesign, WorkerFrameProtocol.
This commit is contained in:
@@ -49,61 +49,62 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto {
|
||||
"CRIXCg9nYXRld2F5X3ZlcnNpb24YAyABKAkSFwoPbWF4X2ZyYW1lX2J5dGVz",
|
||||
"GAQgASgNImkKC1dvcmtlckhlbGxvEhgKEHByb3RvY29sX3ZlcnNpb24YASAB",
|
||||
"KA0SDQoFbm9uY2UYAiABKAkSGQoRd29ya2VyX3Byb2Nlc3NfaWQYAyABKAUS",
|
||||
"FgoOd29ya2VyX3ZlcnNpb24YBCABKAkijgEKC1dvcmtlclJlYWR5EhkKEXdv",
|
||||
"FgoOd29ya2VyX3ZlcnNpb24YBCABKAkirgEKC1dvcmtlclJlYWR5EhkKEXdv",
|
||||
"cmtlcl9wcm9jZXNzX2lkGAEgASgFEhcKD214YWNjZXNzX3Byb2dpZBgCIAEo",
|
||||
"CRIWCg5teGFjY2Vzc19jbHNpZBgDIAEoCRIzCg9yZWFkeV90aW1lc3RhbXAY",
|
||||
"BCABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wIncKDVdvcmtlckNv",
|
||||
"bW1hbmQSLwoHY29tbWFuZBgBIAEoCzIeLm14YWNjZXNzX2dhdGV3YXkudjEu",
|
||||
"TXhDb21tYW5kEjUKEWVucXVldWVfdGltZXN0YW1wGAIgASgLMhouZ29vZ2xl",
|
||||
"LnByb3RvYnVmLlRpbWVzdGFtcCKBAQoSV29ya2VyQ29tbWFuZFJlcGx5EjIK",
|
||||
"BXJlcGx5GAEgASgLMiMubXhhY2Nlc3NfZ2F0ZXdheS52MS5NeENvbW1hbmRS",
|
||||
"ZXBseRI3ChNjb21wbGV0ZWRfdGltZXN0YW1wGAIgASgLMhouZ29vZ2xlLnBy",
|
||||
"b3RvYnVmLlRpbWVzdGFtcCIeCgxXb3JrZXJDYW5jZWwSDgoGcmVhc29uGAEg",
|
||||
"ASgJIlEKDldvcmtlclNodXRkb3duEi8KDGdyYWNlX3BlcmlvZBgBIAEoCzIZ",
|
||||
"Lmdvb2dsZS5wcm90b2J1Zi5EdXJhdGlvbhIOCgZyZWFzb24YAiABKAkiSAoR",
|
||||
"V29ya2VyU2h1dGRvd25BY2sSMwoGc3RhdHVzGAEgASgLMiMubXhhY2Nlc3Nf",
|
||||
"Z2F0ZXdheS52MS5Qcm90b2NvbFN0YXR1cyI6CgtXb3JrZXJFdmVudBIrCgVl",
|
||||
"dmVudBgBIAEoCzIcLm14YWNjZXNzX2dhdGV3YXkudjEuTXhFdmVudCKlAgoP",
|
||||
"V29ya2VySGVhcnRiZWF0EhkKEXdvcmtlcl9wcm9jZXNzX2lkGAEgASgFEi4K",
|
||||
"BXN0YXRlGAIgASgOMh8ubXhhY2Nlc3Nfd29ya2VyLnYxLldvcmtlclN0YXRl",
|
||||
"Ej8KG2xhc3Rfc3RhX2FjdGl2aXR5X3RpbWVzdGFtcBgDIAEoCzIaLmdvb2ds",
|
||||
"ZS5wcm90b2J1Zi5UaW1lc3RhbXASHQoVcGVuZGluZ19jb21tYW5kX2NvdW50",
|
||||
"GAQgASgNEiIKGm91dGJvdW5kX2V2ZW50X3F1ZXVlX2RlcHRoGAUgASgNEhsK",
|
||||
"E2xhc3RfZXZlbnRfc2VxdWVuY2UYBiABKAQSJgoeY3VycmVudF9jb21tYW5k",
|
||||
"X2NvcnJlbGF0aW9uX2lkGAcgASgJIvQBCgtXb3JrZXJGYXVsdBI5CghjYXRl",
|
||||
"Z29yeRgBIAEoDjInLm14YWNjZXNzX3dvcmtlci52MS5Xb3JrZXJGYXVsdENh",
|
||||
"dGVnb3J5EhYKDmNvbW1hbmRfbWV0aG9kGAIgASgJEhQKB2hyZXN1bHQYAyAB",
|
||||
"KAVIAIgBARIWCg5leGNlcHRpb25fdHlwZRgEIAEoCRIaChJkaWFnbm9zdGlj",
|
||||
"X21lc3NhZ2UYBSABKAkSPAoPcHJvdG9jb2xfc3RhdHVzGAYgASgLMiMubXhh",
|
||||
"Y2Nlc3NfZ2F0ZXdheS52MS5Qcm90b2NvbFN0YXR1c0IKCghfaHJlc3VsdCqX",
|
||||
"AgoLV29ya2VyU3RhdGUSHAoYV09SS0VSX1NUQVRFX1VOU1BFQ0lGSUVEEAAS",
|
||||
"GQoVV09SS0VSX1NUQVRFX1NUQVJUSU5HEAESHAoYV09SS0VSX1NUQVRFX0hB",
|
||||
"TkRTSEFLSU5HEAISIQodV09SS0VSX1NUQVRFX0lOSVRJQUxJWklOR19TVEEQ",
|
||||
"AxIWChJXT1JLRVJfU1RBVEVfUkVBRFkQBBIiCh5XT1JLRVJfU1RBVEVfRVhF",
|
||||
"Q1VUSU5HX0NPTU1BTkQQBRIeChpXT1JLRVJfU1RBVEVfU0hVVFRJTkdfRE9X",
|
||||
"ThAGEhgKFFdPUktFUl9TVEFURV9TVE9QUEVEEAcSGAoUV09SS0VSX1NUQVRF",
|
||||
"X0ZBVUxURUQQCCrHBAoTV29ya2VyRmF1bHRDYXRlZ29yeRIlCiFXT1JLRVJf",
|
||||
"RkFVTFRfQ0FURUdPUllfVU5TUEVDSUZJRUQQABIrCidXT1JLRVJfRkFVTFRf",
|
||||
"Q0FURUdPUllfSU5WQUxJRF9BUkdVTUVOVFMQARI3CjNXT1JLRVJfRkFVTFRf",
|
||||
"Q0FURUdPUllfR0FURVdBWV9BVVRIRU5USUNBVElPTl9GQUlMRUQQAhIrCidX",
|
||||
"T1JLRVJfRkFVTFRfQ0FURUdPUllfUFJPVE9DT0xfTUlTTUFUQ0gQAxIsCihX",
|
||||
"T1JLRVJfRkFVTFRfQ0FURUdPUllfUFJPVE9DT0xfVklPTEFUSU9OEAQSKwon",
|
||||
"V09SS0VSX0ZBVUxUX0NBVEVHT1JZX1BJUEVfRElTQ09OTkVDVEVEEAUSMgou",
|
||||
"V09SS0VSX0ZBVUxUX0NBVEVHT1JZX01YQUNDRVNTX0NSRUFUSU9OX0ZBSUxF",
|
||||
"RBAGEjEKLVdPUktFUl9GQVVMVF9DQVRFR09SWV9NWEFDQ0VTU19DT01NQU5E",
|
||||
"X0ZBSUxFRBAHEjoKNldPUktFUl9GQVVMVF9DQVRFR09SWV9NWEFDQ0VTU19F",
|
||||
"VkVOVF9DT05WRVJTSU9OX0ZBSUxFRBAIEiIKHldPUktFUl9GQVVMVF9DQVRF",
|
||||
"R09SWV9TVEFfSFVORxAJEigKJFdPUktFUl9GQVVMVF9DQVRFR09SWV9RVUVV",
|
||||
"RV9PVkVSRkxPVxAKEioKJldPUktFUl9GQVVMVF9DQVRFR09SWV9TSFVURE9X",
|
||||
"Tl9USU1FT1VUEAtCJqoCI1pCLk1PTS5XVy5NeEdhdGV3YXkuQ29udHJhY3Rz",
|
||||
"LlByb3RvYgZwcm90bzM="));
|
||||
"BCABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEh4KFnJlY29yZF9p",
|
||||
"bmZvX3Bpbl9zdGF0dXMYBSABKAkidwoNV29ya2VyQ29tbWFuZBIvCgdjb21t",
|
||||
"YW5kGAEgASgLMh4ubXhhY2Nlc3NfZ2F0ZXdheS52MS5NeENvbW1hbmQSNQoR",
|
||||
"ZW5xdWV1ZV90aW1lc3RhbXAYAiABKAsyGi5nb29nbGUucHJvdG9idWYuVGlt",
|
||||
"ZXN0YW1wIoEBChJXb3JrZXJDb21tYW5kUmVwbHkSMgoFcmVwbHkYASABKAsy",
|
||||
"Iy5teGFjY2Vzc19nYXRld2F5LnYxLk14Q29tbWFuZFJlcGx5EjcKE2NvbXBs",
|
||||
"ZXRlZF90aW1lc3RhbXAYAiABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0",
|
||||
"YW1wIh4KDFdvcmtlckNhbmNlbBIOCgZyZWFzb24YASABKAkiUQoOV29ya2Vy",
|
||||
"U2h1dGRvd24SLwoMZ3JhY2VfcGVyaW9kGAEgASgLMhkuZ29vZ2xlLnByb3Rv",
|
||||
"YnVmLkR1cmF0aW9uEg4KBnJlYXNvbhgCIAEoCSJIChFXb3JrZXJTaHV0ZG93",
|
||||
"bkFjaxIzCgZzdGF0dXMYASABKAsyIy5teGFjY2Vzc19nYXRld2F5LnYxLlBy",
|
||||
"b3RvY29sU3RhdHVzIjoKC1dvcmtlckV2ZW50EisKBWV2ZW50GAEgASgLMhwu",
|
||||
"bXhhY2Nlc3NfZ2F0ZXdheS52MS5NeEV2ZW50IqUCCg9Xb3JrZXJIZWFydGJl",
|
||||
"YXQSGQoRd29ya2VyX3Byb2Nlc3NfaWQYASABKAUSLgoFc3RhdGUYAiABKA4y",
|
||||
"Hy5teGFjY2Vzc193b3JrZXIudjEuV29ya2VyU3RhdGUSPwobbGFzdF9zdGFf",
|
||||
"YWN0aXZpdHlfdGltZXN0YW1wGAMgASgLMhouZ29vZ2xlLnByb3RvYnVmLlRp",
|
||||
"bWVzdGFtcBIdChVwZW5kaW5nX2NvbW1hbmRfY291bnQYBCABKA0SIgoab3V0",
|
||||
"Ym91bmRfZXZlbnRfcXVldWVfZGVwdGgYBSABKA0SGwoTbGFzdF9ldmVudF9z",
|
||||
"ZXF1ZW5jZRgGIAEoBBImCh5jdXJyZW50X2NvbW1hbmRfY29ycmVsYXRpb25f",
|
||||
"aWQYByABKAki9AEKC1dvcmtlckZhdWx0EjkKCGNhdGVnb3J5GAEgASgOMicu",
|
||||
"bXhhY2Nlc3Nfd29ya2VyLnYxLldvcmtlckZhdWx0Q2F0ZWdvcnkSFgoOY29t",
|
||||
"bWFuZF9tZXRob2QYAiABKAkSFAoHaHJlc3VsdBgDIAEoBUgAiAEBEhYKDmV4",
|
||||
"Y2VwdGlvbl90eXBlGAQgASgJEhoKEmRpYWdub3N0aWNfbWVzc2FnZRgFIAEo",
|
||||
"CRI8Cg9wcm90b2NvbF9zdGF0dXMYBiABKAsyIy5teGFjY2Vzc19nYXRld2F5",
|
||||
"LnYxLlByb3RvY29sU3RhdHVzQgoKCF9ocmVzdWx0KpcCCgtXb3JrZXJTdGF0",
|
||||
"ZRIcChhXT1JLRVJfU1RBVEVfVU5TUEVDSUZJRUQQABIZChVXT1JLRVJfU1RB",
|
||||
"VEVfU1RBUlRJTkcQARIcChhXT1JLRVJfU1RBVEVfSEFORFNIQUtJTkcQAhIh",
|
||||
"Ch1XT1JLRVJfU1RBVEVfSU5JVElBTElaSU5HX1NUQRADEhYKEldPUktFUl9T",
|
||||
"VEFURV9SRUFEWRAEEiIKHldPUktFUl9TVEFURV9FWEVDVVRJTkdfQ09NTUFO",
|
||||
"RBAFEh4KGldPUktFUl9TVEFURV9TSFVUVElOR19ET1dOEAYSGAoUV09SS0VS",
|
||||
"X1NUQVRFX1NUT1BQRUQQBxIYChRXT1JLRVJfU1RBVEVfRkFVTFRFRBAIKscE",
|
||||
"ChNXb3JrZXJGYXVsdENhdGVnb3J5EiUKIVdPUktFUl9GQVVMVF9DQVRFR09S",
|
||||
"WV9VTlNQRUNJRklFRBAAEisKJ1dPUktFUl9GQVVMVF9DQVRFR09SWV9JTlZB",
|
||||
"TElEX0FSR1VNRU5UUxABEjcKM1dPUktFUl9GQVVMVF9DQVRFR09SWV9HQVRF",
|
||||
"V0FZX0FVVEhFTlRJQ0FUSU9OX0ZBSUxFRBACEisKJ1dPUktFUl9GQVVMVF9D",
|
||||
"QVRFR09SWV9QUk9UT0NPTF9NSVNNQVRDSBADEiwKKFdPUktFUl9GQVVMVF9D",
|
||||
"QVRFR09SWV9QUk9UT0NPTF9WSU9MQVRJT04QBBIrCidXT1JLRVJfRkFVTFRf",
|
||||
"Q0FURUdPUllfUElQRV9ESVNDT05ORUNURUQQBRIyCi5XT1JLRVJfRkFVTFRf",
|
||||
"Q0FURUdPUllfTVhBQ0NFU1NfQ1JFQVRJT05fRkFJTEVEEAYSMQotV09SS0VS",
|
||||
"X0ZBVUxUX0NBVEVHT1JZX01YQUNDRVNTX0NPTU1BTkRfRkFJTEVEEAcSOgo2",
|
||||
"V09SS0VSX0ZBVUxUX0NBVEVHT1JZX01YQUNDRVNTX0VWRU5UX0NPTlZFUlNJ",
|
||||
"T05fRkFJTEVEEAgSIgoeV09SS0VSX0ZBVUxUX0NBVEVHT1JZX1NUQV9IVU5H",
|
||||
"EAkSKAokV09SS0VSX0ZBVUxUX0NBVEVHT1JZX1FVRVVFX09WRVJGTE9XEAoS",
|
||||
"KgomV09SS0VSX0ZBVUxUX0NBVEVHT1JZX1NIVVRET1dOX1RJTUVPVVQQC0Im",
|
||||
"qgIjWkIuTU9NLldXLk14R2F0ZXdheS5Db250cmFjdHMuUHJvdG9iBnByb3Rv",
|
||||
"Mw=="));
|
||||
descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
|
||||
new pbr::FileDescriptor[] { global::Google.Protobuf.WellKnownTypes.DurationReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.TimestampReflection.Descriptor, global::ZB.MOM.WW.MxGateway.Contracts.Proto.MxaccessGatewayReflection.Descriptor, },
|
||||
new pbr::GeneratedClrTypeInfo(new[] {typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerState), typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerFaultCategory), }, null, new pbr::GeneratedClrTypeInfo[] {
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerEnvelope), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerEnvelope.Parser, new[]{ "ProtocolVersion", "SessionId", "Sequence", "CorrelationId", "GatewayHello", "WorkerHello", "WorkerReady", "WorkerCommand", "WorkerCommandReply", "WorkerCancel", "WorkerShutdown", "WorkerShutdownAck", "WorkerEvent", "WorkerHeartbeat", "WorkerFault" }, new[]{ "Body" }, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.GatewayHello), global::ZB.MOM.WW.MxGateway.Contracts.Proto.GatewayHello.Parser, new[]{ "SupportedProtocolVersion", "Nonce", "GatewayVersion", "MaxFrameBytes" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerHello), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerHello.Parser, new[]{ "ProtocolVersion", "Nonce", "WorkerProcessId", "WorkerVersion" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerReady), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerReady.Parser, new[]{ "WorkerProcessId", "MxaccessProgid", "MxaccessClsid", "ReadyTimestamp" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerReady), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerReady.Parser, new[]{ "WorkerProcessId", "MxaccessProgid", "MxaccessClsid", "ReadyTimestamp", "RecordInfoPinStatus" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerCommand), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerCommand.Parser, new[]{ "Command", "EnqueueTimestamp" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerCommandReply), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerCommandReply.Parser, new[]{ "Reply", "CompletedTimestamp" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerCancel), global::ZB.MOM.WW.MxGateway.Contracts.Proto.WorkerCancel.Parser, new[]{ "Reason" }, null, null, null, null),
|
||||
@@ -1744,6 +1745,7 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto {
|
||||
mxaccessProgid_ = other.mxaccessProgid_;
|
||||
mxaccessClsid_ = other.mxaccessClsid_;
|
||||
readyTimestamp_ = other.readyTimestamp_ != null ? other.readyTimestamp_.Clone() : null;
|
||||
recordInfoPinStatus_ = other.recordInfoPinStatus_;
|
||||
_unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields);
|
||||
}
|
||||
|
||||
@@ -1801,6 +1803,26 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto {
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Field number for the "record_info_pin_status" field.</summary>
|
||||
public const int RecordInfoPinStatusFieldNumber = 5;
|
||||
private string recordInfoPinStatus_ = "";
|
||||
/// <summary>
|
||||
/// Outcome of the worker's process-wide MXSTATUS_PROXY record-info pin
|
||||
/// (MxStatusProxyRecordInfoPin): "pinned {library-guid} vM.m record {record-guid}" on
|
||||
/// success, or the failing call and HRESULT / exception type otherwise. Empty when the
|
||||
/// worker made no attempt (an older worker, or a runtime that never created the
|
||||
/// MXAccess COM object). Diagnostic only: the gateway logs it and shows it on the
|
||||
/// dashboard, and never gates readiness on it, because the pin is a cache warm-up.
|
||||
/// </summary>
|
||||
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
|
||||
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
|
||||
public string RecordInfoPinStatus {
|
||||
get { return recordInfoPinStatus_; }
|
||||
set {
|
||||
recordInfoPinStatus_ = pb::ProtoPreconditions.CheckNotNull(value, "value");
|
||||
}
|
||||
}
|
||||
|
||||
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
|
||||
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
|
||||
public override bool Equals(object other) {
|
||||
@@ -1820,6 +1842,7 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto {
|
||||
if (MxaccessProgid != other.MxaccessProgid) return false;
|
||||
if (MxaccessClsid != other.MxaccessClsid) return false;
|
||||
if (!object.Equals(ReadyTimestamp, other.ReadyTimestamp)) return false;
|
||||
if (RecordInfoPinStatus != other.RecordInfoPinStatus) return false;
|
||||
return Equals(_unknownFields, other._unknownFields);
|
||||
}
|
||||
|
||||
@@ -1831,6 +1854,7 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto {
|
||||
if (MxaccessProgid.Length != 0) hash ^= MxaccessProgid.GetHashCode();
|
||||
if (MxaccessClsid.Length != 0) hash ^= MxaccessClsid.GetHashCode();
|
||||
if (readyTimestamp_ != null) hash ^= ReadyTimestamp.GetHashCode();
|
||||
if (RecordInfoPinStatus.Length != 0) hash ^= RecordInfoPinStatus.GetHashCode();
|
||||
if (_unknownFields != null) {
|
||||
hash ^= _unknownFields.GetHashCode();
|
||||
}
|
||||
@@ -1865,6 +1889,10 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto {
|
||||
output.WriteRawTag(34);
|
||||
output.WriteMessage(ReadyTimestamp);
|
||||
}
|
||||
if (RecordInfoPinStatus.Length != 0) {
|
||||
output.WriteRawTag(42);
|
||||
output.WriteString(RecordInfoPinStatus);
|
||||
}
|
||||
if (_unknownFields != null) {
|
||||
_unknownFields.WriteTo(output);
|
||||
}
|
||||
@@ -1891,6 +1919,10 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto {
|
||||
output.WriteRawTag(34);
|
||||
output.WriteMessage(ReadyTimestamp);
|
||||
}
|
||||
if (RecordInfoPinStatus.Length != 0) {
|
||||
output.WriteRawTag(42);
|
||||
output.WriteString(RecordInfoPinStatus);
|
||||
}
|
||||
if (_unknownFields != null) {
|
||||
_unknownFields.WriteTo(ref output);
|
||||
}
|
||||
@@ -1913,6 +1945,9 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto {
|
||||
if (readyTimestamp_ != null) {
|
||||
size += 1 + pb::CodedOutputStream.ComputeMessageSize(ReadyTimestamp);
|
||||
}
|
||||
if (RecordInfoPinStatus.Length != 0) {
|
||||
size += 1 + pb::CodedOutputStream.ComputeStringSize(RecordInfoPinStatus);
|
||||
}
|
||||
if (_unknownFields != null) {
|
||||
size += _unknownFields.CalculateSize();
|
||||
}
|
||||
@@ -1940,6 +1975,9 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto {
|
||||
}
|
||||
ReadyTimestamp.MergeFrom(other.ReadyTimestamp);
|
||||
}
|
||||
if (other.RecordInfoPinStatus.Length != 0) {
|
||||
RecordInfoPinStatus = other.RecordInfoPinStatus;
|
||||
}
|
||||
_unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields);
|
||||
}
|
||||
|
||||
@@ -1978,6 +2016,10 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto {
|
||||
input.ReadMessage(ReadyTimestamp);
|
||||
break;
|
||||
}
|
||||
case 42: {
|
||||
RecordInfoPinStatus = input.ReadString();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -2016,6 +2058,10 @@ namespace ZB.MOM.WW.MxGateway.Contracts.Proto {
|
||||
input.ReadMessage(ReadyTimestamp);
|
||||
break;
|
||||
}
|
||||
case 42: {
|
||||
RecordInfoPinStatus = input.ReadString();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,6 +65,13 @@ message WorkerReady {
|
||||
string mxaccess_progid = 2;
|
||||
string mxaccess_clsid = 3;
|
||||
google.protobuf.Timestamp ready_timestamp = 4;
|
||||
// Outcome of the worker's process-wide MXSTATUS_PROXY record-info pin
|
||||
// (MxStatusProxyRecordInfoPin): "pinned {library-guid} vM.m record {record-guid}" on
|
||||
// success, or the failing call and HRESULT / exception type otherwise. Empty when the
|
||||
// worker made no attempt (an older worker, or a runtime that never created the
|
||||
// MXAccess COM object). Diagnostic only: the gateway logs it and shows it on the
|
||||
// dashboard, and never gates readiness on it, because the pin is a cache warm-up.
|
||||
string record_info_pin_status = 5;
|
||||
}
|
||||
|
||||
message WorkerCommand {
|
||||
|
||||
@@ -102,6 +102,7 @@ else
|
||||
<tr><th scope="row">Process id</th><td>@(CurrentSession.WorkerProcessId?.ToString(System.Globalization.CultureInfo.InvariantCulture) ?? "-")</td></tr>
|
||||
<tr><th scope="row">State</th><td><StatusBadge Text="@(CurrentSession.WorkerState?.ToString() ?? "-")" /></td></tr>
|
||||
<tr><th scope="row">Last heartbeat</th><td>@DashboardDisplay.DateTime(CurrentSession.LastWorkerHeartbeatAt)</td></tr>
|
||||
<tr><th scope="row">Record-info pin</th><td>@DashboardDisplay.Text(CurrentSession.WorkerRecordInfoPinStatus)</td></tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
@@ -17,4 +17,5 @@ public sealed record DashboardSessionSummary(
|
||||
WorkerClientState? WorkerState,
|
||||
DateTimeOffset? LastWorkerHeartbeatAt,
|
||||
long EventsReceived,
|
||||
string? LastFault);
|
||||
string? LastFault,
|
||||
string? WorkerRecordInfoPinStatus = null);
|
||||
|
||||
@@ -285,7 +285,8 @@ public sealed class DashboardSnapshotService : IDashboardSnapshotService
|
||||
WorkerState: workerClient?.State,
|
||||
LastWorkerHeartbeatAt: workerClient?.LastHeartbeatAt,
|
||||
EventsReceived: eventsReceived,
|
||||
LastFault: DashboardRedactor.Redact(session.FinalFault));
|
||||
LastFault: DashboardRedactor.Redact(session.FinalFault),
|
||||
WorkerRecordInfoPinStatus: workerClient?.RecordInfoPinStatus);
|
||||
}
|
||||
|
||||
private static DashboardWorkerSummary CreateWorkerSummary(GatewaySession session)
|
||||
|
||||
@@ -27,6 +27,14 @@ public interface IWorkerClient : IAsyncDisposable
|
||||
/// </returns>
|
||||
Task? GetStateChangedSignal() => null;
|
||||
|
||||
/// <summary>
|
||||
/// Outcome of the worker's <c>MXSTATUS_PROXY</c> record-info pin, as reported in
|
||||
/// <see cref="WorkerReady"/>, or <see langword="null"/> before the handshake or when the
|
||||
/// worker did not report one. Diagnostic only; the default implementation returns
|
||||
/// <see langword="null"/>.
|
||||
/// </summary>
|
||||
string? RecordInfoPinStatus => null;
|
||||
|
||||
/// <summary>UTC timestamp of the most recent heartbeat from the worker.</summary>
|
||||
DateTimeOffset LastHeartbeatAt { get; }
|
||||
|
||||
|
||||
@@ -13,6 +13,8 @@ namespace ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
public sealed class WorkerClient : IWorkerClient
|
||||
{
|
||||
private const string GatewayVersionFallback = "unknown";
|
||||
// Prefix of a successful MxStatusProxyRecordInfoPin.Status in the worker; anything else is a failure.
|
||||
private const string RecordInfoPinnedPrefix = "pinned ";
|
||||
private static readonly TimeSpan DisposeTaskTimeout = TimeSpan.FromSeconds(5);
|
||||
private readonly object _syncRoot = new();
|
||||
private readonly WorkerClientConnection _connection;
|
||||
@@ -55,6 +57,7 @@ public sealed class WorkerClient : IWorkerClient
|
||||
private TaskCompletionSource _stateChanged = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
private DateTimeOffset _lastHeartbeatAt;
|
||||
private int? _processId;
|
||||
private string? _recordInfoPinStatus;
|
||||
private int _eventQueueDepth;
|
||||
private int _eventsReaderClaimed;
|
||||
private Task? _readLoopTask;
|
||||
@@ -161,6 +164,9 @@ public sealed class WorkerClient : IWorkerClient
|
||||
}
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public string? RecordInfoPinStatus => Volatile.Read(ref _recordInfoPinStatus);
|
||||
|
||||
/// <inheritdoc />
|
||||
public DateTimeOffset LastHeartbeatAt
|
||||
{
|
||||
@@ -1074,6 +1080,45 @@ public sealed class WorkerClient : IWorkerClient
|
||||
DateTimeOffset readyAt = _timeProvider.GetUtcNow();
|
||||
DateTimeOffset launchedAt = _connection.ProcessHandle?.LaunchedAt ?? readyAt;
|
||||
_metrics?.WorkerStarted(readyAt - launchedAt);
|
||||
RecordRecordInfoPinStatus(ready.RecordInfoPinStatus);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Records and logs the worker's <c>MXSTATUS_PROXY</c> record-info pin outcome from
|
||||
/// <see cref="WorkerReady"/>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The worker logs the outcome too, but to its own stderr, which the gateway does not
|
||||
/// capture; this is the line operators actually see. A failed pin is a warning rather than a
|
||||
/// fault: MXAccess still works, but every data change then pays a type-library reload, which
|
||||
/// is the dominant worker CPU cost at high event rates. An empty status means an older
|
||||
/// worker, so nothing is logged.
|
||||
/// </remarks>
|
||||
/// <param name="status">The status string the worker reported; empty when absent.</param>
|
||||
private void RecordRecordInfoPinStatus(string status)
|
||||
{
|
||||
if (string.IsNullOrEmpty(status))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
Volatile.Write(ref _recordInfoPinStatus, status);
|
||||
if (status.StartsWith(RecordInfoPinnedPrefix, StringComparison.Ordinal))
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"Worker {ProcessId} for session {SessionId} pinned the MXSTATUS_PROXY record info: {RecordInfoPinStatus}",
|
||||
ProcessId,
|
||||
SessionId,
|
||||
status);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Worker {ProcessId} for session {SessionId} could not pin the MXSTATUS_PROXY record info; each data change will reload the MXAccess type library: {RecordInfoPinStatus}",
|
||||
ProcessId,
|
||||
SessionId,
|
||||
status);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Updates the heartbeat timestamp and process ID.</summary>
|
||||
|
||||
@@ -182,6 +182,31 @@ public sealed class DashboardSnapshotServiceTests
|
||||
Assert.Equal("worker pipe disconnected", fault.Message);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The worker's record-info pin outcome, reported in <c>WorkerReady</c>, reaches the session
|
||||
/// summary the session-details page renders, since the worker's own log line is not captured.
|
||||
/// </summary>
|
||||
[Fact]
|
||||
public void GetSnapshot_ProjectsWorkerRecordInfoPinStatus()
|
||||
{
|
||||
const string pinStatus = "LoadRegTypeLib failed (HRESULT 0x8002801D)";
|
||||
SessionRegistry registry = new();
|
||||
GatewaySession session = CreateSession(
|
||||
"session-pin",
|
||||
"client-one",
|
||||
DateTimeOffset.Parse("2026-04-26T10:00:00Z", CultureInfo.InvariantCulture));
|
||||
session.AttachWorkerClient(
|
||||
new FakeWorkerClient("session-pin", 1301, WorkerClientState.Ready) { RecordInfoPinStatus = pinStatus });
|
||||
session.MarkReady();
|
||||
registry.TryAdd(session);
|
||||
using GatewayMetrics metrics = new();
|
||||
DashboardSnapshotService service = CreateService(registry, metrics);
|
||||
|
||||
DashboardSessionSummary summary = Assert.Single(service.GetSnapshot().Sessions);
|
||||
|
||||
Assert.Equal(pinStatus, summary.WorkerRecordInfoPinStatus);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies snapshot metrics include the cumulative alarm provider switch count.
|
||||
/// </summary>
|
||||
@@ -929,6 +954,9 @@ public sealed class DashboardSnapshotServiceTests
|
||||
/// <inheritdoc />
|
||||
public WorkerClientState State { get; private set; } = state;
|
||||
|
||||
/// <inheritdoc />
|
||||
public string? RecordInfoPinStatus { get; init; }
|
||||
|
||||
/// <inheritdoc />
|
||||
public DateTimeOffset LastHeartbeatAt { get; } = DateTimeOffset.Parse("2026-04-26T10:02:00Z", CultureInfo.InvariantCulture);
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using System.Buffers.Binary;
|
||||
using System.IO.Pipes;
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.MxGateway.Contracts;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Metrics;
|
||||
@@ -108,11 +109,13 @@ public sealed class FakeWorkerHarness : IAsyncDisposable
|
||||
/// <param name="options">Configuration options for the worker client.</param>
|
||||
/// <param name="metrics">Gateway metrics collector.</param>
|
||||
/// <param name="timeProvider">Time provider for timestamps.</param>
|
||||
/// <param name="logger">Logger for the worker client; null uses a no-op logger.</param>
|
||||
/// <returns>A configured worker client connected to this harness.</returns>
|
||||
public WorkerClient CreateClient(
|
||||
WorkerClientOptions? options = null,
|
||||
GatewayMetrics? metrics = null,
|
||||
TimeProvider? timeProvider = null)
|
||||
TimeProvider? timeProvider = null,
|
||||
ILogger<WorkerClient>? logger = null)
|
||||
{
|
||||
if (_gatewayStream is null)
|
||||
{
|
||||
@@ -125,7 +128,7 @@ public sealed class FakeWorkerHarness : IAsyncDisposable
|
||||
_gatewayStream,
|
||||
_frameOptions);
|
||||
|
||||
return new WorkerClient(connection, options, metrics, timeProvider);
|
||||
return new WorkerClient(connection, options, metrics, timeProvider, logger);
|
||||
}
|
||||
|
||||
/// <summary>Completes the worker startup handshake by reading the gateway hello and sending worker hello and ready.</summary>
|
||||
@@ -229,12 +232,14 @@ public sealed class FakeWorkerHarness : IAsyncDisposable
|
||||
/// <param name="mxaccessProgid">MXAccess COM ProgID.</param>
|
||||
/// <param name="mxaccessClsid">MXAccess COM CLSID.</param>
|
||||
/// <param name="cancellationToken">Token to cancel the asynchronous operation.</param>
|
||||
/// <param name="recordInfoPinStatus">Record-info pin outcome to report; empty mimics an older worker.</param>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
public async Task SendWorkerReadyAsync(
|
||||
int workerProcessId = DefaultWorkerProcessId,
|
||||
string mxaccessProgid = "LMXProxy.LMXProxyServer.1",
|
||||
string mxaccessClsid = "{C30B52F5-2CB5-4760-AF0A-3A344A7EB5DC}",
|
||||
CancellationToken cancellationToken = default)
|
||||
CancellationToken cancellationToken = default,
|
||||
string recordInfoPinStatus = "")
|
||||
{
|
||||
await _writer.WriteAsync(
|
||||
CreateEnvelope(
|
||||
@@ -245,6 +250,7 @@ public sealed class FakeWorkerHarness : IAsyncDisposable
|
||||
MxaccessProgid = mxaccessProgid,
|
||||
MxaccessClsid = mxaccessClsid,
|
||||
ReadyTimestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
|
||||
RecordInfoPinStatus = recordInfoPinStatus,
|
||||
}),
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.MxGateway.Contracts.Proto;
|
||||
using ZB.MOM.WW.MxGateway.Server.Workers;
|
||||
using ZB.MOM.WW.MxGateway.Tests.Gateway.Workers.Fakes;
|
||||
using ZB.MOM.WW.MxGateway.Tests.TestSupport;
|
||||
|
||||
namespace ZB.MOM.WW.MxGateway.Tests.Gateway.Workers;
|
||||
|
||||
/// <summary>
|
||||
/// The worker reports its <c>MXSTATUS_PROXY</c> record-info pin outcome in <see cref="WorkerReady"/>
|
||||
/// because the gateway never sees the worker's own stderr log. These tests pin how the gateway
|
||||
/// surfaces it: kept on the client, logged at Information on success and Warning on failure,
|
||||
/// and silently absent for an older worker that sends no status.
|
||||
/// </summary>
|
||||
public sealed class WorkerClientRecordInfoPinTests
|
||||
{
|
||||
private const string PinnedStatus =
|
||||
"pinned {77e896ec-b3e3-4de4-b96f-f32570164211} v3.2 record {a162d9e4-9019-42f3-a087-79264007e542}";
|
||||
|
||||
private static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(5);
|
||||
|
||||
/// <summary>A successful pin is recorded on the client and logged at Information.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StartAsync_WorkerReportsPinned_RecordsStatusAndLogsInformation()
|
||||
{
|
||||
RecordingLogger<WorkerClient> logger = new();
|
||||
await using FakeWorkerHarness harness = await FakeWorkerHarness.CreateConnectedPairAsync();
|
||||
await using WorkerClient client = harness.CreateClient(logger: logger);
|
||||
|
||||
await StartWithReadyAsync(client, harness, PinnedStatus);
|
||||
|
||||
Assert.Equal(PinnedStatus, client.RecordInfoPinStatus);
|
||||
(LogLevel level, string message, _) = Assert.Single(
|
||||
logger.Entries,
|
||||
entry => entry.Message.Contains("record info", StringComparison.Ordinal));
|
||||
Assert.Equal(LogLevel.Information, level);
|
||||
Assert.Contains(PinnedStatus, message, StringComparison.Ordinal);
|
||||
Assert.Contains(FakeWorkerHarness.DefaultSessionId, message, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A failed pin still lets the worker reach Ready (the pin is a cache warm-up, not a
|
||||
/// prerequisite) but is logged at Warning so the per-event CPU cost it implies is visible.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StartAsync_WorkerReportsPinFailure_StaysReadyAndLogsWarning()
|
||||
{
|
||||
const string failedStatus = "LoadRegTypeLib failed (HRESULT 0x8002801D)";
|
||||
RecordingLogger<WorkerClient> logger = new();
|
||||
await using FakeWorkerHarness harness = await FakeWorkerHarness.CreateConnectedPairAsync();
|
||||
await using WorkerClient client = harness.CreateClient(logger: logger);
|
||||
|
||||
await StartWithReadyAsync(client, harness, failedStatus);
|
||||
|
||||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||||
Assert.Equal(failedStatus, client.RecordInfoPinStatus);
|
||||
(LogLevel level, string message, _) = Assert.Single(
|
||||
logger.Entries,
|
||||
entry => entry.Message.Contains("record info", StringComparison.Ordinal));
|
||||
Assert.Equal(LogLevel.Warning, level);
|
||||
Assert.Contains(failedStatus, message, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
/// <summary>An older worker sends no status: nothing is recorded and nothing is logged about it.</summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task StartAsync_WorkerReportsNoStatus_RecordsNullAndDoesNotLog()
|
||||
{
|
||||
RecordingLogger<WorkerClient> logger = new();
|
||||
await using FakeWorkerHarness harness = await FakeWorkerHarness.CreateConnectedPairAsync();
|
||||
await using WorkerClient client = harness.CreateClient(logger: logger);
|
||||
|
||||
await StartWithReadyAsync(client, harness, recordInfoPinStatus: string.Empty);
|
||||
|
||||
Assert.Equal(WorkerClientState.Ready, client.State);
|
||||
Assert.Null(client.RecordInfoPinStatus);
|
||||
Assert.DoesNotContain(
|
||||
logger.Entries,
|
||||
entry => entry.Message.Contains("record info", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
private static async Task StartWithReadyAsync(
|
||||
WorkerClient client,
|
||||
FakeWorkerHarness harness,
|
||||
string recordInfoPinStatus)
|
||||
{
|
||||
Task startTask = client.StartAsync(CancellationToken.None);
|
||||
WorkerEnvelope gatewayHello = await harness.ReadGatewayEnvelopeAsync().WaitAsync(TestTimeout);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.GatewayHello, gatewayHello.BodyCase);
|
||||
await harness.SendWorkerHelloAsync().WaitAsync(TestTimeout);
|
||||
await harness.SendWorkerReadyAsync(recordInfoPinStatus: recordInfoPinStatus).WaitAsync(TestTimeout);
|
||||
await startTask.WaitAsync(TestTimeout);
|
||||
}
|
||||
}
|
||||
@@ -406,6 +406,151 @@ public sealed class WorkerPipeSessionTests
|
||||
await SendShutdownAndWaitAsync(pipePair, runTask, cancellation.Token);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The drain loop's task mirrors the dedicated thread's outcome: cancelling its token ends
|
||||
/// the task Canceled (not faulted, not still running), which is what lets the message loop's
|
||||
/// teardown treat it as an ordinary stop. Exercised directly, before any runtime session is
|
||||
/// attached, so the loop is in its runtime-less poll and only the cancellation can end it.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunEventDrainLoopAsync_WhenCancelled_EndsCanceled()
|
||||
{
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
using MemoryStream inbound = new();
|
||||
using MemoryStream outbound = new();
|
||||
WorkerPipeSession session = CreateSession(inbound, outbound, options);
|
||||
using CancellationTokenSource stop = new();
|
||||
|
||||
Task drainTask = session.RunEventDrainLoopAsync(stop.Token);
|
||||
Assert.False(drainTask.IsCompleted);
|
||||
stop.Cancel();
|
||||
|
||||
Task completedTask = await Task.WhenAny(drainTask, Task.Delay(TimeSpan.FromSeconds(5)));
|
||||
Assert.Same(drainTask, completedTask);
|
||||
Assert.True(drainTask.IsCanceled, $"Expected Canceled but the drain task ended {drainTask.Status}.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Cancelling the session while the dedicated drain thread is parked on the event signal
|
||||
/// ends <see cref="WorkerPipeSession.RunAsync"/> as cancelled, disposes the runtime, and lets
|
||||
/// the drain thread exit. Teardown must not log the drain task as faulted or overrunning its
|
||||
/// stop budget: a cancelled drain is a clean stop.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_WhenCancelledWhileDrainThreadParked_EndsCanceledAndDrainThreadExits()
|
||||
{
|
||||
using CancellationTokenSource timeout = new(TimeSpan.FromSeconds(10));
|
||||
using CancellationTokenSource sessionCancellation = new();
|
||||
using PipePair pipePair = await PipePair.CreateAsync(timeout.Token);
|
||||
RecordingWorkerLogger logger = new();
|
||||
FakeRuntimeSession runtime = new()
|
||||
{
|
||||
WaitForEventsOnSignalOnly = true,
|
||||
};
|
||||
WorkerPipeSession session = CreatePipeSession(
|
||||
pipePair.WorkerStream,
|
||||
runtime,
|
||||
new WorkerPipeSessionOptions
|
||||
{
|
||||
HeartbeatInterval = TimeSpan.FromMilliseconds(100),
|
||||
HeartbeatGrace = TimeSpan.FromSeconds(5),
|
||||
},
|
||||
logger);
|
||||
Task runTask = session.RunAsync(sessionCancellation.Token);
|
||||
await CompleteGatewayHandshakeAsync(pipePair, timeout.Token);
|
||||
Thread drainThread = await WaitForDrainThreadAsync(runtime, timeout.Token);
|
||||
|
||||
sessionCancellation.Cancel();
|
||||
|
||||
Task completedTask = await Task.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(5), timeout.Token));
|
||||
Assert.Same(runTask, completedTask);
|
||||
await Assert.ThrowsAnyAsync<OperationCanceledException>(async () => await runTask);
|
||||
Assert.True(runtime.Disposed);
|
||||
Assert.True(drainThread.Join(TimeSpan.FromSeconds(5)), "The dedicated drain thread did not exit.");
|
||||
Assert.DoesNotContain(
|
||||
logger.Events,
|
||||
entry => entry.Fields.TryGetValue("task", out object? task) && (task as string) == "EventDrain");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// An unexpected exception inside the loop on the dedicated drain thread faults the drain
|
||||
/// task with that same exception, so the message loop's <c>Task.WhenAny</c> supervision ends
|
||||
/// <see cref="WorkerPipeSession.RunAsync"/> with it exactly as it did when the loop was an
|
||||
/// async method: the session stops, the runtime is disposed, and the thread exits.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task RunAsync_WhenDrainLoopThrows_FaultsSessionWithTheSameException()
|
||||
{
|
||||
using CancellationTokenSource cancellation = new(TimeSpan.FromSeconds(10));
|
||||
using PipePair pipePair = await PipePair.CreateAsync(cancellation.Token);
|
||||
FakeRuntimeSession runtime = new()
|
||||
{
|
||||
WaitForEventsOnSignalOnly = true,
|
||||
};
|
||||
WorkerPipeSession session = CreatePipeSession(pipePair.WorkerStream, runtime);
|
||||
Task runTask = session.RunAsync(cancellation.Token);
|
||||
await CompleteGatewayHandshakeAsync(pipePair, cancellation.Token);
|
||||
Thread drainThread = await WaitForDrainThreadAsync(runtime, cancellation.Token);
|
||||
|
||||
InvalidOperationException scripted = new("scripted drain-loop failure");
|
||||
runtime.DrainEventsException = scripted;
|
||||
// Wake the parked thread; its next pass calls DrainEvents, which now throws.
|
||||
runtime.EnqueueEvent(CreateWorkerEvent(sequence: 4));
|
||||
|
||||
Task completedTask = await Task.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(5), cancellation.Token));
|
||||
Assert.Same(runTask, completedTask);
|
||||
InvalidOperationException observed =
|
||||
await Assert.ThrowsAsync<InvalidOperationException>(async () => await runTask);
|
||||
Assert.Same(scripted, observed);
|
||||
Assert.True(runtime.Disposed);
|
||||
Assert.True(drainThread.Join(TimeSpan.FromSeconds(5)), "The dedicated drain thread did not exit.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The handshake copies the process-wide record-info pin outcome into <c>WorkerReady</c>, so
|
||||
/// the gateway can log it: the worker's own log goes to stderr, which the gateway does not
|
||||
/// capture. Whether the pin succeeds depends on the host, so only the hand-off is asserted.
|
||||
/// </summary>
|
||||
/// <returns>A task that represents the asynchronous operation.</returns>
|
||||
[Fact]
|
||||
public async Task CompleteStartupHandshakeAsync_ReportsRecordInfoPinStatusInWorkerReady()
|
||||
{
|
||||
_ = MxStatusProxyRecordInfoPin.EnsurePinned();
|
||||
string? pinStatus = MxStatusProxyRecordInfoPin.Status;
|
||||
Assert.False(string.IsNullOrEmpty(pinStatus));
|
||||
WorkerFrameProtocolOptions options = CreateOptions();
|
||||
using MemoryStream inbound = new();
|
||||
await new WorkerFrameWriter(inbound, options).WriteAsync(CreateGatewayHelloEnvelope());
|
||||
inbound.Position = 0;
|
||||
using MemoryStream outbound = new();
|
||||
WorkerPipeSession session = CreateSession(inbound, outbound, options);
|
||||
|
||||
await session.CompleteStartupHandshakeAsync(
|
||||
_ => Task.FromResult(new WorkerReady { WorkerProcessId = 1234 }));
|
||||
|
||||
WorkerEnvelope[] written = ReadWrittenFrames(outbound, options);
|
||||
Assert.Equal(WorkerEnvelope.BodyOneofCase.WorkerReady, written[1].BodyCase);
|
||||
Assert.Equal(pinStatus, written[1].WorkerReady.RecordInfoPinStatus);
|
||||
}
|
||||
|
||||
// Waits until the drain loop has parked on the fake's event signal at least once and returns the
|
||||
// thread it parked on: the dedicated drain thread.
|
||||
private static async Task<Thread> WaitForDrainThreadAsync(
|
||||
FakeRuntimeSession runtime,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
Thread? drainThread;
|
||||
while ((drainThread = runtime.LastWaitForEventsThread) is null)
|
||||
{
|
||||
await Task.Delay(TimeSpan.FromMilliseconds(5), cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
return drainThread;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Verifies that a Ping control command is answered on the worker side
|
||||
/// (not dispatched to the STA) with an OK reply that echoes the ping
|
||||
|
||||
@@ -42,6 +42,8 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession
|
||||
// disposed SemaphoreSlim would turn that shutdown into an ObjectDisposedException.
|
||||
private readonly SemaphoreSlim eventSignal = new(0, 1);
|
||||
private TimeSpan? lastWaitForEventsTimeout;
|
||||
private volatile Exception? drainEventsException;
|
||||
private volatile Thread? lastWaitForEventsThread;
|
||||
private bool refreshStaActivityOnCapture;
|
||||
private WorkerRuntimeHeartbeatSnapshot snapshot = new(
|
||||
DateTimeOffset.UtcNow,
|
||||
@@ -235,9 +237,25 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession
|
||||
/// </summary>
|
||||
public int? LastDrainMaxTotalBytes { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets or sets an exception every <see cref="DrainEvents(uint)"/> call throws, standing in
|
||||
/// for an unexpected failure inside the event drain loop. Safe to set from the test thread
|
||||
/// while the drain thread is running.
|
||||
/// </summary>
|
||||
public Exception? DrainEventsException
|
||||
{
|
||||
get => drainEventsException;
|
||||
set => drainEventsException = value;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public IReadOnlyList<WorkerEvent> DrainEvents(uint maxEvents)
|
||||
{
|
||||
if (drainEventsException is { } drainException)
|
||||
{
|
||||
throw drainException;
|
||||
}
|
||||
|
||||
if (SuppressDrainForBatchSize is uint suppressed && maxEvents == suppressed)
|
||||
{
|
||||
return Array.Empty<WorkerEvent>();
|
||||
@@ -393,9 +411,16 @@ internal sealed class FakeRuntimeSession : IWorkerRuntimeSession
|
||||
/// <summary>Gets whether the most recent <see cref="WaitForEvents"/> call ran on a thread-pool thread.</summary>
|
||||
public bool? LastWaitForEventsOnThreadPool { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the thread that made the most recent <see cref="WaitForEvents"/> call, so a test can
|
||||
/// assert the dedicated drain thread has exited after the session stops.
|
||||
/// </summary>
|
||||
public Thread? LastWaitForEventsThread => lastWaitForEventsThread;
|
||||
|
||||
/// <inheritdoc />
|
||||
public void WaitForEvents(TimeSpan timeout, CancellationToken cancellationToken)
|
||||
{
|
||||
lastWaitForEventsThread = Thread.CurrentThread;
|
||||
LastWaitForEventsThreadName = Thread.CurrentThread.Name;
|
||||
LastWaitForEventsOnThreadPool = Thread.CurrentThread.IsThreadPoolThread;
|
||||
// Same knobs as the async form (signal-only, recorded timeout, backing queue); the drain
|
||||
|
||||
@@ -360,6 +360,14 @@ public sealed class WorkerPipeSession
|
||||
_logger?.Information(
|
||||
"WorkerRecordInfoPin",
|
||||
new Dictionary<string, object?> { ["status"] = pinStatus });
|
||||
|
||||
// The gateway does not capture worker stderr, so the log line above is rarely
|
||||
// seen; WorkerReady carries the outcome to the gateway, which logs it and shows it
|
||||
// on the dashboard. A runtime that already filled the field keeps its own value.
|
||||
if (string.IsNullOrEmpty(ready.RecordInfoPinStatus))
|
||||
{
|
||||
ready.RecordInfoPinStatus = pinStatus;
|
||||
}
|
||||
}
|
||||
|
||||
await WriteWorkerReadyAsync(ready, cancellationToken).ConfigureAwait(false);
|
||||
@@ -609,8 +617,12 @@ public sealed class WorkerPipeSession
|
||||
/// there. A dedicated thread parks on the signal instead, and with a synchronous flush the
|
||||
/// steady-state event path schedules no managed thread-pool work at all. The pipe write itself
|
||||
/// stays overlapped; its completion runs on an I/O completion thread, not a worker thread.
|
||||
/// Internal so the completion mapping can be tested directly; production code calls it only
|
||||
/// from <see cref="RunMessageLoopAsync"/>.
|
||||
/// </remarks>
|
||||
private Task RunEventDrainLoopAsync(CancellationToken cancellationToken)
|
||||
/// <param name="cancellationToken">Token that stops the loop; the returned task then ends Canceled.</param>
|
||||
/// <returns>A task that mirrors the loop's outcome.</returns>
|
||||
internal Task RunEventDrainLoopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
TaskCompletionSource<bool> completion = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
Thread thread = new(() =>
|
||||
@@ -618,7 +630,18 @@ public sealed class WorkerPipeSession
|
||||
try
|
||||
{
|
||||
RunEventDrainLoop(cancellationToken);
|
||||
completion.TrySetResult(true);
|
||||
|
||||
// The loop returns only once its token is cancelled. Report that as Canceled
|
||||
// whether the loop noticed at its head or inside a wait (which throws), so a
|
||||
// stopped drain has one outcome rather than a timing-dependent pair.
|
||||
if (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
completion.TrySetCanceled(cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
completion.TrySetResult(true);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user