fix(communication)+fix(siteruntime): carry DetachedScriptThreads/ScriptRunShedCount/SiteStreamAlarmDropCount across the health wire; restore drained counters on failed send (arch-review 2026-08-19 P8-1)

This commit is contained in:
Joseph Doherty
2026-08-19 14:47:04 -04:00
parent eca0ab768d
commit d0a748e632
9 changed files with 228 additions and 45 deletions
@@ -72,7 +72,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
"ZXkYASABKAkSQQoFdmFsdWUYAiABKAsyMi5zY2FkYWJyaWRnZS5jZW50cmFs", "ZXkYASABKAkSQQoFdmFsdWUYAiABKAsyMi5zY2FkYWJyaWRnZS5jZW50cmFs",
"Y29udHJvbC52MS5UYWdRdWFsaXR5Q291bnRzRHRvOgI4ASJQChFOb2RlU3Rh", "Y29udHJvbC52MS5UYWdRdWFsaXR5Q291bnRzRHRvOgI4ASJQChFOb2RlU3Rh",
"dHVzTGlzdER0bxI7CgVub2RlcxgBIAMoCzIsLnNjYWRhYnJpZGdlLmNlbnRy", "dHVzTGlzdER0bxI7CgVub2RlcxgBIAMoCzIsLnNjYWRhYnJpZGdlLmNlbnRy",
"YWxjb250cm9sLnYxLk5vZGVTdGF0dXNEdG8iyQ4KE1NpdGVIZWFsdGhSZXBv", "YWxjb250cm9sLnYxLk5vZGVTdGF0dXNEdG8irw8KE1NpdGVIZWFsdGhSZXBv",
"cnREdG8SDwoHc2l0ZV9pZBgBIAEoCRIXCg9zZXF1ZW5jZV9udW1iZXIYAiAB", "cnREdG8SDwoHc2l0ZV9pZBgBIAEoCRIXCg9zZXF1ZW5jZV9udW1iZXIYAiAB",
"KAMSNAoQcmVwb3J0X3RpbWVzdGFtcBgDIAEoCzIaLmdvb2dsZS5wcm90b2J1", "KAMSNAoQcmVwb3J0X3RpbWVzdGFtcBgDIAEoCzIaLmdvb2dsZS5wcm90b2J1",
"Zi5UaW1lc3RhbXAScAoYZGF0YV9jb25uZWN0aW9uX3N0YXR1c2VzGAQgAygL", "Zi5UaW1lc3RhbXAScAoYZGF0YV9jb25uZWN0aW9uX3N0YXR1c2VzGAQgAygL",
@@ -107,42 +107,45 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
"ASgLMhouZ29vZ2xlLnByb3RvYnVmLkJvb2xWYWx1ZRI7ChZsb2NhbF9kYl9v", "ASgLMhouZ29vZ2xlLnByb3RvYnVmLkJvb2xWYWx1ZRI7ChZsb2NhbF9kYl9v",
"cGxvZ19iYWNrbG9nGBwgASgLMhsuZ29vZ2xlLnByb3RvYnVmLkludDY0VmFs", "cGxvZ19iYWNrbG9nGBwgASgLMhsuZ29vZ2xlLnByb3RvYnVmLkludDY0VmFs",
"dWUSPQoZc3RhcnR1cF9yZWNvbmNpbGVyX2FjdGl2ZRgdIAEoCzIaLmdvb2ds", "dWUSPQoZc3RhcnR1cF9yZWNvbmNpbGVyX2FjdGl2ZRgdIAEoCzIaLmdvb2ds",
"ZS5wcm90b2J1Zi5Cb29sVmFsdWUacgobRGF0YUNvbm5lY3Rpb25TdGF0dXNl", "ZS5wcm90b2J1Zi5Cb29sVmFsdWUSHwoXZGV0YWNoZWRfc2NyaXB0X3RocmVh",
"c0VudHJ5EgsKA2tleRgBIAEoCRJCCgV2YWx1ZRgCIAEoDjIzLnNjYWRhYnJp", "ZHMYHiABKAUSHQoVc2NyaXB0X3J1bl9zaGVkX2NvdW50GB8gASgFEiQKHHNp",
"ZGdlLmNlbnRyYWxjb250cm9sLnYxLkNvbm5lY3Rpb25IZWFsdGhFbnVtOgI4", "dGVfc3RyZWFtX2FsYXJtX2Ryb3BfY291bnQYICABKAMacgobRGF0YUNvbm5l",
"ARpxChhUYWdSZXNvbHV0aW9uQ291bnRzRW50cnkSCwoDa2V5GAEgASgJEkQK", "Y3Rpb25TdGF0dXNlc0VudHJ5EgsKA2tleRgBIAEoCRJCCgV2YWx1ZRgCIAEo",
"BXZhbHVlGAIgASgLMjUuc2NhZGFicmlkZ2UuY2VudHJhbGNvbnRyb2wudjEu", "DjIzLnNjYWRhYnJpZGdlLmNlbnRyYWxjb250cm9sLnYxLkNvbm5lY3Rpb25I",
"VGFnUmVzb2x1dGlvblN0YXR1c0R0bzoCOAEaQgogU3RvcmVBbmRGb3J3YXJk", "ZWFsdGhFbnVtOgI4ARpxChhUYWdSZXNvbHV0aW9uQ291bnRzRW50cnkSCwoD",
"QnVmZmVyRGVwdGhzRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgF", "a2V5GAEgASgJEkQKBXZhbHVlGAIgASgLMjUuc2NhZGFicmlkZ2UuY2VudHJh",
"OgI4ASJjChZTaXRlSGVhbHRoUmVwb3J0QWNrRHRvEg8KB3NpdGVfaWQYASAB", "bGNvbnRyb2wudjEuVGFnUmVzb2x1dGlvblN0YXR1c0R0bzoCOAEaQgogU3Rv",
"KAkSFwoPc2VxdWVuY2VfbnVtYmVyGAIgASgDEhAKCGFjY2VwdGVkGAMgASgI", "cmVBbmRGb3J3YXJkQnVmZmVyRGVwdGhzRW50cnkSCwoDa2V5GAEgASgJEg0K",
"Eg0KBWVycm9yGAQgASgJIosBCgxIZWFydGJlYXREdG8SDwoHc2l0ZV9pZBgB", "BXZhbHVlGAIgASgFOgI4ASJjChZTaXRlSGVhbHRoUmVwb3J0QWNrRHRvEg8K",
"IAEoCRIVCg1ub2RlX2hvc3RuYW1lGAIgASgJEhEKCWlzX2FjdGl2ZRgDIAEo", "B3NpdGVfaWQYASABKAkSFwoPc2VxdWVuY2VfbnVtYmVyGAIgASgDEhAKCGFj",
"CBItCgl0aW1lc3RhbXAYBCABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0", "Y2VwdGVkGAMgASgIEg0KBWVycm9yGAQgASgJIosBCgxIZWFydGJlYXREdG8S",
"YW1wEhEKCXN5bnRoZXRpYxgFIAEoCCq9AQoUQ29ubmVjdGlvbkhlYWx0aEVu", "DwoHc2l0ZV9pZBgBIAEoCRIVCg1ub2RlX2hvc3RuYW1lGAIgASgJEhEKCWlz",
"dW0SIQodQ09OTkVDVElPTl9IRUFMVEhfVU5TUEVDSUZJRUQQABIfChtDT05O", "X2FjdGl2ZRgDIAEoCBItCgl0aW1lc3RhbXAYBCABKAsyGi5nb29nbGUucHJv",
"RUNUSU9OX0hFQUxUSF9DT05ORUNURUQQARIiCh5DT05ORUNUSU9OX0hFQUxU", "dG9idWYuVGltZXN0YW1wEhEKCXN5bnRoZXRpYxgFIAEoCCq9AQoUQ29ubmVj",
"SF9ESVNDT05ORUNURUQQAhIgChxDT05ORUNUSU9OX0hFQUxUSF9DT05ORUNU", "dGlvbkhlYWx0aEVudW0SIQodQ09OTkVDVElPTl9IRUFMVEhfVU5TUEVDSUZJ",
"SU5HEAMSGwoXQ09OTkVDVElPTl9IRUFMVEhfRVJST1IQBDKhBgoVQ2VudHJh", "RUQQABIfChtDT05ORUNUSU9OX0hFQUxUSF9DT05ORUNURUQQARIiCh5DT05O",
"bENvbnRyb2xTZXJ2aWNlEoMBChJTdWJtaXROb3RpZmljYXRpb24SNC5zY2Fk", "RUNUSU9OX0hFQUxUSF9ESVNDT05ORUNURUQQAhIgChxDT05ORUNUSU9OX0hF",
"YWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5Ob3RpZmljYXRpb25TdWJtaXRE", "QUxUSF9DT05ORUNUSU5HEAMSGwoXQ09OTkVDVElPTl9IRUFMVEhfRVJST1IQ",
"dG8aNy5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5Ob3RpZmljYXRp", "BDKhBgoVQ2VudHJhbENvbnRyb2xTZXJ2aWNlEoMBChJTdWJtaXROb3RpZmlj",
"b25TdWJtaXRBY2tEdG8SkgEKF1F1ZXJ5Tm90aWZpY2F0aW9uU3RhdHVzEjku", "YXRpb24SNC5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5Ob3RpZmlj",
"c2NhZGFicmlkZ2UuY2VudHJhbGNvbnRyb2wudjEuTm90aWZpY2F0aW9uU3Rh", "YXRpb25TdWJtaXREdG8aNy5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52",
"dHVzUXVlcnlEdG8aPC5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5O", "MS5Ob3RpZmljYXRpb25TdWJtaXRBY2tEdG8SkgEKF1F1ZXJ5Tm90aWZpY2F0",
"b3RpZmljYXRpb25TdGF0dXNSZXNwb25zZUR0bxJHChFJbmdlc3RBdWRpdEV2", "aW9uU3RhdHVzEjkuc2NhZGFicmlkZ2UuY2VudHJhbGNvbnRyb2wudjEuTm90",
"ZW50cxIbLnNpdGVzdHJlYW0uQXVkaXRFdmVudEJhdGNoGhUuc2l0ZXN0cmVh", "aWZpY2F0aW9uU3RhdHVzUXVlcnlEdG8aPC5zY2FkYWJyaWRnZS5jZW50cmFs",
"bS5Jbmdlc3RBY2sSUAoVSW5nZXN0Q2FjaGVkVGVsZW1ldHJ5EiAuc2l0ZXN0", "Y29udHJvbC52MS5Ob3RpZmljYXRpb25TdGF0dXNSZXNwb25zZUR0bxJHChFJ",
"cmVhbS5DYWNoZWRUZWxlbWV0cnlCYXRjaBoVLnNpdGVzdHJlYW0uSW5nZXN0", "bmdlc3RBdWRpdEV2ZW50cxIbLnNpdGVzdHJlYW0uQXVkaXRFdmVudEJhdGNo",
"QWNrEoABCg1SZWNvbmNpbGVTaXRlEjYuc2NhZGFicmlkZ2UuY2VudHJhbGNv", "GhUuc2l0ZXN0cmVhbS5Jbmdlc3RBY2sSUAoVSW5nZXN0Q2FjaGVkVGVsZW1l",
"bnRyb2wudjEuUmVjb25jaWxlU2l0ZVJlcXVlc3REdG8aNy5zY2FkYWJyaWRn", "dHJ5EiAuc2l0ZXN0cmVhbS5DYWNoZWRUZWxlbWV0cnlCYXRjaBoVLnNpdGVz",
"ZS5jZW50cmFsY29udHJvbC52MS5SZWNvbmNpbGVTaXRlUmVzcG9uc2VEdG8S", "dHJlYW0uSW5nZXN0QWNrEoABCg1SZWNvbmNpbGVTaXRlEjYuc2NhZGFicmlk",
"fQoQUmVwb3J0U2l0ZUhlYWx0aBIyLnNjYWRhYnJpZGdlLmNlbnRyYWxjb250", "Z2UuY2VudHJhbGNvbnRyb2wudjEuUmVjb25jaWxlU2l0ZVJlcXVlc3REdG8a",
"cm9sLnYxLlNpdGVIZWFsdGhSZXBvcnREdG8aNS5zY2FkYWJyaWRnZS5jZW50", "Ny5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5SZWNvbmNpbGVTaXRl",
"cmFsY29udHJvbC52MS5TaXRlSGVhbHRoUmVwb3J0QWNrRHRvElAKCUhlYXJ0", "UmVzcG9uc2VEdG8SfQoQUmVwb3J0U2l0ZUhlYWx0aBIyLnNjYWRhYnJpZGdl",
"YmVhdBIrLnNjYWRhYnJpZGdlLmNlbnRyYWxjb250cm9sLnYxLkhlYXJ0YmVh", "LmNlbnRyYWxjb250cm9sLnYxLlNpdGVIZWFsdGhSZXBvcnREdG8aNS5zY2Fk",
"dER0bxoWLmdvb2dsZS5wcm90b2J1Zi5FbXB0eUIrqgIoWkIuTU9NLldXLlNj", "YWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5TaXRlSGVhbHRoUmVwb3J0QWNr",
"YWRhQnJpZGdlLkNvbW11bmljYXRpb24uR3JwY2IGcHJvdG8z")); "RHRvElAKCUhlYXJ0YmVhdBIrLnNjYWRhYnJpZGdlLmNlbnRyYWxjb250cm9s",
"LnYxLkhlYXJ0YmVhdER0bxoWLmdvb2dsZS5wcm90b2J1Zi5FbXB0eUIrqgIo",
"WkIuTU9NLldXLlNjYWRhQnJpZGdlLkNvbW11bmljYXRpb24uR3JwY2IGcHJv",
"dG8z"));
descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
new pbr::FileDescriptor[] { global::Google.Protobuf.WellKnownTypes.EmptyReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.TimestampReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.WrappersReflection.Descriptor, global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SitestreamReflection.Descriptor, }, new pbr::FileDescriptor[] { global::Google.Protobuf.WellKnownTypes.EmptyReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.TimestampReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.WrappersReflection.Descriptor, global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SitestreamReflection.Descriptor, },
new pbr::GeneratedClrTypeInfo(new[] {typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.ConnectionHealthEnum), }, null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(new[] {typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.ConnectionHealthEnum), }, null, new pbr::GeneratedClrTypeInfo[] {
@@ -160,7 +163,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.ConnectionEndpointMapDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.ConnectionEndpointMapDto.Parser, new[]{ "Entries" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { null, }), new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.ConnectionEndpointMapDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.ConnectionEndpointMapDto.Parser, new[]{ "Entries" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { null, }),
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.TagQualityMapDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.TagQualityMapDto.Parser, new[]{ "Entries" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { null, }), new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.TagQualityMapDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.TagQualityMapDto.Parser, new[]{ "Entries" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { null, }),
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.NodeStatusListDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.NodeStatusListDto.Parser, new[]{ "Nodes" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.NodeStatusListDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.NodeStatusListDto.Parser, new[]{ "Nodes" }, null, null, null, null),
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportDto.Parser, new[]{ "SiteId", "SequenceNumber", "ReportTimestamp", "DataConnectionStatuses", "TagResolutionCounts", "ScriptErrorCount", "AlarmEvaluationErrorCount", "StoreAndForwardBufferDepths", "DeadLetterCount", "DeployedInstanceCount", "EnabledInstanceCount", "DisabledInstanceCount", "NodeRole", "NodeHostname", "DataConnectionEndpoints", "DataConnectionTagQuality", "ParkedMessageCount", "ClusterNodes", "SiteAuditWriteFailures", "AuditRedactionFailure", "SiteAuditBacklog", "SiteEventLogWriteFailures", "OldestParkedMessageAgeSeconds", "ScriptQueueDepth", "ScriptBusyThreads", "ScriptOldestBusyAgeSeconds", "LocalDbReplicationConnected", "LocalDbOplogBacklog", "StartupReconcilerActive" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { null, null, null, }), new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportDto.Parser, new[]{ "SiteId", "SequenceNumber", "ReportTimestamp", "DataConnectionStatuses", "TagResolutionCounts", "ScriptErrorCount", "AlarmEvaluationErrorCount", "StoreAndForwardBufferDepths", "DeadLetterCount", "DeployedInstanceCount", "EnabledInstanceCount", "DisabledInstanceCount", "NodeRole", "NodeHostname", "DataConnectionEndpoints", "DataConnectionTagQuality", "ParkedMessageCount", "ClusterNodes", "SiteAuditWriteFailures", "AuditRedactionFailure", "SiteAuditBacklog", "SiteEventLogWriteFailures", "OldestParkedMessageAgeSeconds", "ScriptQueueDepth", "ScriptBusyThreads", "ScriptOldestBusyAgeSeconds", "LocalDbReplicationConnected", "LocalDbOplogBacklog", "StartupReconcilerActive", "DetachedScriptThreads", "ScriptRunShedCount", "SiteStreamAlarmDropCount" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { null, null, null, }),
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportAckDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportAckDto.Parser, new[]{ "SiteId", "SequenceNumber", "Accepted", "Error" }, null, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportAckDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportAckDto.Parser, new[]{ "SiteId", "SequenceNumber", "Accepted", "Error" }, null, null, null, null),
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.HeartbeatDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.HeartbeatDto.Parser, new[]{ "SiteId", "NodeHostname", "IsActive", "Timestamp", "Synthetic" }, null, null, null, null) new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.HeartbeatDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.HeartbeatDto.Parser, new[]{ "SiteId", "NodeHostname", "IsActive", "Timestamp", "Synthetic" }, null, null, null, null)
})); }));
@@ -4268,6 +4271,9 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
LocalDbReplicationConnected = other.LocalDbReplicationConnected; LocalDbReplicationConnected = other.LocalDbReplicationConnected;
LocalDbOplogBacklog = other.LocalDbOplogBacklog; LocalDbOplogBacklog = other.LocalDbOplogBacklog;
StartupReconcilerActive = other.StartupReconcilerActive; StartupReconcilerActive = other.StartupReconcilerActive;
detachedScriptThreads_ = other.detachedScriptThreads_;
scriptRunShedCount_ = other.scriptRunShedCount_;
siteStreamAlarmDropCount_ = other.siteStreamAlarmDropCount_;
_unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields);
} }
@@ -4665,6 +4671,56 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
} }
/// <summary>Field number for the "detached_script_threads" field.</summary>
public const int DetachedScriptThreadsFieldNumber = 30;
private int detachedScriptThreads_;
/// <summary>
/// WP3.1: script-execution worker threads detached and replaced by the stuck-script
/// watchdog and not yet exited. Point-in-time gauge (like script_busy_threads above),
/// not interval-drained. Zero is the healthy state.
/// </summary>
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
public int DetachedScriptThreads {
get { return detachedScriptThreads_; }
set {
detachedScriptThreads_ = value;
}
}
/// <summary>Field number for the "script_run_shed_count" field.</summary>
public const int ScriptRunShedCountFieldNumber = 31;
private int scriptRunShedCount_;
/// <summary>
/// WP3.1: per-interval count of script/alarm on-trigger runs shed by
/// MaxConcurrentRunsPerScript. Interval-drained like script_error_count.
/// </summary>
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
public int ScriptRunShedCount {
get { return scriptRunShedCount_; }
set {
scriptRunShedCount_ = value;
}
}
/// <summary>Field number for the "site_stream_alarm_drop_count" field.</summary>
public const int SiteStreamAlarmDropCountFieldNumber = 32;
private long siteStreamAlarmDropCount_;
/// <summary>
/// WP2.6d: cumulative count of alarm state changes dropped from SiteStreamManager's
/// dedicated alarm publish queue because it was at capacity. Point-in-time
/// (not reset on collect), like site_event_log_write_failures.
/// </summary>
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
public long SiteStreamAlarmDropCount {
get { return siteStreamAlarmDropCount_; }
set {
siteStreamAlarmDropCount_ = value;
}
}
[global::System.Diagnostics.DebuggerNonUserCodeAttribute] [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)] [global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
public override bool Equals(object other) { public override bool Equals(object other) {
@@ -4709,6 +4765,9 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
if (LocalDbReplicationConnected != other.LocalDbReplicationConnected) return false; if (LocalDbReplicationConnected != other.LocalDbReplicationConnected) return false;
if (LocalDbOplogBacklog != other.LocalDbOplogBacklog) return false; if (LocalDbOplogBacklog != other.LocalDbOplogBacklog) return false;
if (StartupReconcilerActive != other.StartupReconcilerActive) return false; if (StartupReconcilerActive != other.StartupReconcilerActive) return false;
if (DetachedScriptThreads != other.DetachedScriptThreads) return false;
if (ScriptRunShedCount != other.ScriptRunShedCount) return false;
if (SiteStreamAlarmDropCount != other.SiteStreamAlarmDropCount) return false;
return Equals(_unknownFields, other._unknownFields); return Equals(_unknownFields, other._unknownFields);
} }
@@ -4745,6 +4804,9 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
if (localDbReplicationConnected_ != null) hash ^= LocalDbReplicationConnected.GetHashCode(); if (localDbReplicationConnected_ != null) hash ^= LocalDbReplicationConnected.GetHashCode();
if (localDbOplogBacklog_ != null) hash ^= LocalDbOplogBacklog.GetHashCode(); if (localDbOplogBacklog_ != null) hash ^= LocalDbOplogBacklog.GetHashCode();
if (startupReconcilerActive_ != null) hash ^= StartupReconcilerActive.GetHashCode(); if (startupReconcilerActive_ != null) hash ^= StartupReconcilerActive.GetHashCode();
if (DetachedScriptThreads != 0) hash ^= DetachedScriptThreads.GetHashCode();
if (ScriptRunShedCount != 0) hash ^= ScriptRunShedCount.GetHashCode();
if (SiteStreamAlarmDropCount != 0L) hash ^= SiteStreamAlarmDropCount.GetHashCode();
if (_unknownFields != null) { if (_unknownFields != null) {
hash ^= _unknownFields.GetHashCode(); hash ^= _unknownFields.GetHashCode();
} }
@@ -4865,6 +4927,18 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
if (startupReconcilerActive_ != null) { if (startupReconcilerActive_ != null) {
_single_startupReconcilerActive_codec.WriteTagAndValue(output, StartupReconcilerActive); _single_startupReconcilerActive_codec.WriteTagAndValue(output, StartupReconcilerActive);
} }
if (DetachedScriptThreads != 0) {
output.WriteRawTag(240, 1);
output.WriteInt32(DetachedScriptThreads);
}
if (ScriptRunShedCount != 0) {
output.WriteRawTag(248, 1);
output.WriteInt32(ScriptRunShedCount);
}
if (SiteStreamAlarmDropCount != 0L) {
output.WriteRawTag(128, 2);
output.WriteInt64(SiteStreamAlarmDropCount);
}
if (_unknownFields != null) { if (_unknownFields != null) {
_unknownFields.WriteTo(output); _unknownFields.WriteTo(output);
} }
@@ -4977,6 +5051,18 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
if (startupReconcilerActive_ != null) { if (startupReconcilerActive_ != null) {
_single_startupReconcilerActive_codec.WriteTagAndValue(ref output, StartupReconcilerActive); _single_startupReconcilerActive_codec.WriteTagAndValue(ref output, StartupReconcilerActive);
} }
if (DetachedScriptThreads != 0) {
output.WriteRawTag(240, 1);
output.WriteInt32(DetachedScriptThreads);
}
if (ScriptRunShedCount != 0) {
output.WriteRawTag(248, 1);
output.WriteInt32(ScriptRunShedCount);
}
if (SiteStreamAlarmDropCount != 0L) {
output.WriteRawTag(128, 2);
output.WriteInt64(SiteStreamAlarmDropCount);
}
if (_unknownFields != null) { if (_unknownFields != null) {
_unknownFields.WriteTo(ref output); _unknownFields.WriteTo(ref output);
} }
@@ -5068,6 +5154,15 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
if (startupReconcilerActive_ != null) { if (startupReconcilerActive_ != null) {
size += _single_startupReconcilerActive_codec.CalculateSizeWithTag(StartupReconcilerActive); size += _single_startupReconcilerActive_codec.CalculateSizeWithTag(StartupReconcilerActive);
} }
if (DetachedScriptThreads != 0) {
size += 2 + pb::CodedOutputStream.ComputeInt32Size(DetachedScriptThreads);
}
if (ScriptRunShedCount != 0) {
size += 2 + pb::CodedOutputStream.ComputeInt32Size(ScriptRunShedCount);
}
if (SiteStreamAlarmDropCount != 0L) {
size += 2 + pb::CodedOutputStream.ComputeInt64Size(SiteStreamAlarmDropCount);
}
if (_unknownFields != null) { if (_unknownFields != null) {
size += _unknownFields.CalculateSize(); size += _unknownFields.CalculateSize();
} }
@@ -5186,6 +5281,15 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
StartupReconcilerActive = other.StartupReconcilerActive; StartupReconcilerActive = other.StartupReconcilerActive;
} }
} }
if (other.DetachedScriptThreads != 0) {
DetachedScriptThreads = other.DetachedScriptThreads;
}
if (other.ScriptRunShedCount != 0) {
ScriptRunShedCount = other.ScriptRunShedCount;
}
if (other.SiteStreamAlarmDropCount != 0L) {
SiteStreamAlarmDropCount = other.SiteStreamAlarmDropCount;
}
_unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields);
} }
@@ -5351,6 +5455,18 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
} }
break; break;
} }
case 240: {
DetachedScriptThreads = input.ReadInt32();
break;
}
case 248: {
ScriptRunShedCount = input.ReadInt32();
break;
}
case 256: {
SiteStreamAlarmDropCount = input.ReadInt64();
break;
}
} }
} }
#endif #endif
@@ -5516,6 +5632,18 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
} }
break; break;
} }
case 240: {
DetachedScriptThreads = input.ReadInt32();
break;
}
case 248: {
ScriptRunShedCount = input.ReadInt32();
break;
}
case 256: {
SiteStreamAlarmDropCount = input.ReadInt64();
break;
}
} }
} }
} }
@@ -476,6 +476,9 @@ public static class CentralControlDtoMapper
LocalDbReplicationConnected = msg.LocalDbReplicationConnected, LocalDbReplicationConnected = msg.LocalDbReplicationConnected,
LocalDbOplogBacklog = msg.LocalDbOplogBacklog, LocalDbOplogBacklog = msg.LocalDbOplogBacklog,
StartupReconcilerActive = msg.StartupReconcilerActive, StartupReconcilerActive = msg.StartupReconcilerActive,
DetachedScriptThreads = msg.DetachedScriptThreads,
ScriptRunShedCount = msg.ScriptRunShedCount,
SiteStreamAlarmDropCount = msg.SiteStreamAlarmDropCount,
}; };
foreach (var (name, health) in msg.DataConnectionStatuses) foreach (var (name, health) in msg.DataConnectionStatuses)
@@ -651,6 +654,9 @@ public static class CentralControlDtoMapper
LocalDbReplicationConnected = dto.LocalDbReplicationConnected, LocalDbReplicationConnected = dto.LocalDbReplicationConnected,
LocalDbOplogBacklog = dto.LocalDbOplogBacklog, LocalDbOplogBacklog = dto.LocalDbOplogBacklog,
StartupReconcilerActive = dto.StartupReconcilerActive, StartupReconcilerActive = dto.StartupReconcilerActive,
DetachedScriptThreads = dto.DetachedScriptThreads,
ScriptRunShedCount = dto.ScriptRunShedCount,
SiteStreamAlarmDropCount = dto.SiteStreamAlarmDropCount,
}; };
} }
@@ -234,6 +234,17 @@ message SiteHealthReportDto {
// self-heal path, which is a misconfiguration a healthy-looking site would // self-heal path, which is a misconfiguration a healthy-looking site would
// otherwise hide entirely. // otherwise hide entirely.
google.protobuf.BoolValue startup_reconciler_active = 29; google.protobuf.BoolValue startup_reconciler_active = 29;
// WP3.1: script-execution worker threads detached and replaced by the stuck-script
// watchdog and not yet exited. Point-in-time gauge (like script_busy_threads above),
// not interval-drained. Zero is the healthy state.
int32 detached_script_threads = 30;
// WP3.1: per-interval count of script/alarm on-trigger runs shed by
// MaxConcurrentRunsPerScript. Interval-drained like script_error_count.
int32 script_run_shed_count = 31;
// WP2.6d: cumulative count of alarm state changes dropped from SiteStreamManager's
// dedicated alarm publish queue because it was at capacity. Point-in-time
// (not reset on collect), like site_event_log_write_failures.
int64 site_stream_alarm_drop_count = 32;
} }
// Central -> Site: health report ack, so a lost report is observable. // Central -> Site: health report ack, so a lost report is observable.
@@ -132,7 +132,8 @@ public class CentralHealthReportLoop : BackgroundService
alarmErrors: report.AlarmEvaluationErrorCount, alarmErrors: report.AlarmEvaluationErrorCount,
deadLetters: report.DeadLetterCount, deadLetters: report.DeadLetterCount,
siteAuditWriteFailures: report.SiteAuditWriteFailures, siteAuditWriteFailures: report.SiteAuditWriteFailures,
auditRedactionFailures: report.AuditRedactionFailure); auditRedactionFailures: report.AuditRedactionFailure,
scriptRunShedCount: report.ScriptRunShedCount);
throw; throw;
} }
@@ -182,7 +182,8 @@ public class HealthReportSender : BackgroundService
alarmErrors: report.AlarmEvaluationErrorCount, alarmErrors: report.AlarmEvaluationErrorCount,
deadLetters: report.DeadLetterCount, deadLetters: report.DeadLetterCount,
siteAuditWriteFailures: report.SiteAuditWriteFailures, siteAuditWriteFailures: report.SiteAuditWriteFailures,
auditRedactionFailures: report.AuditRedactionFailure); auditRedactionFailures: report.AuditRedactionFailure,
scriptRunShedCount: report.ScriptRunShedCount);
throw; throw;
} }
@@ -286,12 +286,19 @@ public interface ISiteHealthCollector
/// <param name="deadLetters">Dead letter count to add back.</param> /// <param name="deadLetters">Dead letter count to add back.</param>
/// <param name="siteAuditWriteFailures">Site audit write failure count to add back.</param> /// <param name="siteAuditWriteFailures">Site audit write failure count to add back.</param>
/// <param name="auditRedactionFailures">Audit redaction failure count to add back.</param> /// <param name="auditRedactionFailures">Audit redaction failure count to add back.</param>
/// <param name="scriptRunShedCount">
/// Interval-drained script/alarm run-shed count to add back (arch-review 2026-08-19
/// P8-1). Additive trailing parameter defaulting to 0 so existing callers keep
/// compiling; <c>DetachedScriptThreads</c>/<c>SiteStreamAlarmDropCount</c> are
/// point-in-time gauges, not drained on collect, so they need no restore path.
/// </param>
void AddIntervalCounters( void AddIntervalCounters(
int scriptErrors, int scriptErrors,
int alarmErrors, int alarmErrors,
int deadLetters, int deadLetters,
int siteAuditWriteFailures, int siteAuditWriteFailures,
int auditRedactionFailures) int auditRedactionFailures,
int scriptRunShedCount = 0)
{ {
// Default no-op so test fakes do not need to be updated. The real // Default no-op so test fakes do not need to be updated. The real
// SiteHealthCollector overrides this with the Interlocked.Add restore. // SiteHealthCollector overrides this with the Interlocked.Add restore.
@@ -245,19 +245,25 @@ public class SiteHealthCollector : ISiteHealthCollector
int alarmErrors, int alarmErrors,
int deadLetters, int deadLetters,
int siteAuditWriteFailures, int siteAuditWriteFailures,
int auditRedactionFailures) int auditRedactionFailures,
int scriptRunShedCount = 0)
{ {
// Each counter is restored atomically via // Each counter is restored atomically via
// Interlocked.Add so an increment that arrived during the failed Send // Interlocked.Add so an increment that arrived during the failed Send
// (and therefore accumulated against the zero left by CollectReport's // (and therefore accumulated against the zero left by CollectReport's
// Exchange) is correctly summed with the values being put back. No // Exchange) is correctly summed with the values being put back. No
// ordering between the five Adds is required — they target independent // ordering between the six Adds is required — they target independent
// fields. // fields. scriptRunShedCount joins this restore path (arch-review
// 2026-08-19 P8-1) because it is drained by CollectReport the same way
// as the other five; DetachedScriptThreads and SiteStreamAlarmDropCount
// are point-in-time gauges (CompareExchange/Read, never zeroed on
// collect) so a failed Send loses nothing for them to restore.
if (scriptErrors != 0) Interlocked.Add(ref _scriptErrorCount, scriptErrors); if (scriptErrors != 0) Interlocked.Add(ref _scriptErrorCount, scriptErrors);
if (alarmErrors != 0) Interlocked.Add(ref _alarmErrorCount, alarmErrors); if (alarmErrors != 0) Interlocked.Add(ref _alarmErrorCount, alarmErrors);
if (deadLetters != 0) Interlocked.Add(ref _deadLetterCount, deadLetters); if (deadLetters != 0) Interlocked.Add(ref _deadLetterCount, deadLetters);
if (siteAuditWriteFailures != 0) Interlocked.Add(ref _siteAuditWriteFailures, siteAuditWriteFailures); if (siteAuditWriteFailures != 0) Interlocked.Add(ref _siteAuditWriteFailures, siteAuditWriteFailures);
if (auditRedactionFailures != 0) Interlocked.Add(ref _auditRedactionFailures, auditRedactionFailures); if (auditRedactionFailures != 0) Interlocked.Add(ref _auditRedactionFailures, auditRedactionFailures);
if (scriptRunShedCount != 0) Interlocked.Add(ref _scriptRunShedCount, scriptRunShedCount);
} }
/// <inheritdoc /> /// <inheritdoc />
@@ -554,6 +554,11 @@ public class CentralControlDtoMapperTests
LocalDbReplicationConnected = true, LocalDbReplicationConnected = true,
LocalDbOplogBacklog = 250L, LocalDbOplogBacklog = 250L,
StartupReconcilerActive = true, StartupReconcilerActive = true,
// arch-review 2026-08-19 P8-1: these three never crossed the wire at all
// (mapper omitted them entirely, always reading back as 0) until this fix.
DetachedScriptThreads = 2,
ScriptRunShedCount = 13,
SiteStreamAlarmDropCount = 41L,
}; };
/// <summary> /// <summary>
@@ -603,6 +608,9 @@ public class CentralControlDtoMapperTests
Assert.Equal(expected.LocalDbReplicationConnected, actual.LocalDbReplicationConnected); Assert.Equal(expected.LocalDbReplicationConnected, actual.LocalDbReplicationConnected);
Assert.Equal(expected.LocalDbOplogBacklog, actual.LocalDbOplogBacklog); Assert.Equal(expected.LocalDbOplogBacklog, actual.LocalDbOplogBacklog);
Assert.Equal(expected.StartupReconcilerActive, actual.StartupReconcilerActive); Assert.Equal(expected.StartupReconcilerActive, actual.StartupReconcilerActive);
Assert.Equal(expected.DetachedScriptThreads, actual.DetachedScriptThreads);
Assert.Equal(expected.ScriptRunShedCount, actual.ScriptRunShedCount);
Assert.Equal(expected.SiteStreamAlarmDropCount, actual.SiteStreamAlarmDropCount);
} }
/// <summary> /// <summary>
@@ -333,6 +333,16 @@ public class HealthReportSenderTests
/// un-sent report only, and the next successful report shipped with the /// un-sent report only, and the next successful report shipped with the
/// counters at zero. /// counters at zero.
/// </summary> /// </summary>
/// <remarks>
/// Extended for arch-review 2026-08-19 P8-1: <c>ScriptRunShedCount</c> is drained
/// by <c>CollectReport</c> the exact same way as the five counters above, but
/// <c>AddIntervalCounters</c> used to take only those five — a failed send
/// destructively lost the shed count forever. <c>DetachedScriptThreads</c> and
/// <c>SiteStreamAlarmDropCount</c> are deliberately NOT exercised here: they are
/// point-in-time gauges (set via <c>Interlocked.CompareExchange</c>/<c>Read</c>,
/// never zeroed by <c>CollectReport</c>), so a failed send loses nothing for them
/// to restore.
/// </remarks>
[Fact] [Fact]
public async Task SendFailure_PreservesIntervalCountersForNextReport() public async Task SendFailure_PreservesIntervalCountersForNextReport()
{ {
@@ -341,7 +351,7 @@ public class HealthReportSenderTests
collector.SetActiveNode(true); collector.SetActiveNode(true);
// Pre-populate every per-interval counter so the restore path on each // Pre-populate every per-interval counter so the restore path on each
// field is exercised — script error, alarm error, dead letter, site // field is exercised — script error, alarm error, dead letter, site
// audit write failure, audit redaction failure. // audit write failure, audit redaction failure, script run shed.
collector.IncrementScriptError(); collector.IncrementScriptError();
collector.IncrementScriptError(); collector.IncrementScriptError();
collector.IncrementAlarmError(); collector.IncrementAlarmError();
@@ -351,6 +361,10 @@ public class HealthReportSenderTests
collector.IncrementSiteAuditWriteFailures(); collector.IncrementSiteAuditWriteFailures();
collector.IncrementAuditRedactionFailure(); collector.IncrementAuditRedactionFailure();
collector.IncrementAuditRedactionFailure(); collector.IncrementAuditRedactionFailure();
collector.IncrementScriptRunShed();
collector.IncrementScriptRunShed();
collector.IncrementScriptRunShed();
collector.IncrementScriptRunShed();
var options = Options.Create(new HealthMonitoringOptions var options = Options.Create(new HealthMonitoringOptions
{ {
@@ -383,6 +397,7 @@ public class HealthReportSenderTests
Assert.Equal(3, firstSuccess.DeadLetterCount); Assert.Equal(3, firstSuccess.DeadLetterCount);
Assert.Equal(1, firstSuccess.SiteAuditWriteFailures); Assert.Equal(1, firstSuccess.SiteAuditWriteFailures);
Assert.Equal(2, firstSuccess.AuditRedactionFailure); Assert.Equal(2, firstSuccess.AuditRedactionFailure);
Assert.Equal(4, firstSuccess.ScriptRunShedCount);
} }
/// <summary> /// <summary>