dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

fix(communication)+fix(siteruntime): carry DetachedScriptThreads/ScriptRunShedCount/SiteStreamAlarmDropCount across the health wire; restore drained counters on failed send (arch-review 2026-08-19 P8-1)

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-19 14:47:04 -04:00
parent eca0ab768d
commit d0a748e632
9 changed files with 228 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.Communication/CentralControlGrpc/CentralControl.cs
+166 -38
View File
@@ -72,7 +72,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
"ZXkYASABKAkSQQoFdmFsdWUYAiABKAsyMi5zY2FkYWJyaWRnZS5jZW50cmFs",
"Y29udHJvbC52MS5UYWdRdWFsaXR5Q291bnRzRHRvOgI4ASJQChFOb2RlU3Rh",
"dHVzTGlzdER0bxI7CgVub2RlcxgBIAMoCzIsLnNjYWRhYnJpZGdlLmNlbnRy",
"YWxjb250cm9sLnYxLk5vZGVTdGF0dXNEdG8iyQ4KE1NpdGVIZWFsdGhSZXBv",
"YWxjb250cm9sLnYxLk5vZGVTdGF0dXNEdG8irw8KE1NpdGVIZWFsdGhSZXBv",
"cnREdG8SDwoHc2l0ZV9pZBgBIAEoCRIXCg9zZXF1ZW5jZV9udW1iZXIYAiAB",
"KAMSNAoQcmVwb3J0X3RpbWVzdGFtcBgDIAEoCzIaLmdvb2dsZS5wcm90b2J1",
"Zi5UaW1lc3RhbXAScAoYZGF0YV9jb25uZWN0aW9uX3N0YXR1c2VzGAQgAygL",
@@ -107,42 +107,45 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
"ASgLMhouZ29vZ2xlLnByb3RvYnVmLkJvb2xWYWx1ZRI7ChZsb2NhbF9kYl9v",
"cGxvZ19iYWNrbG9nGBwgASgLMhsuZ29vZ2xlLnByb3RvYnVmLkludDY0VmFs",
"dWUSPQoZc3RhcnR1cF9yZWNvbmNpbGVyX2FjdGl2ZRgdIAEoCzIaLmdvb2ds",
"ZS5wcm90b2J1Zi5Cb29sVmFsdWUacgobRGF0YUNvbm5lY3Rpb25TdGF0dXNl",
"c0VudHJ5EgsKA2tleRgBIAEoCRJCCgV2YWx1ZRgCIAEoDjIzLnNjYWRhYnJp",
"ZGdlLmNlbnRyYWxjb250cm9sLnYxLkNvbm5lY3Rpb25IZWFsdGhFbnVtOgI4",
"ARpxChhUYWdSZXNvbHV0aW9uQ291bnRzRW50cnkSCwoDa2V5GAEgASgJEkQK",
"BXZhbHVlGAIgASgLMjUuc2NhZGFicmlkZ2UuY2VudHJhbGNvbnRyb2wudjEu",
"VGFnUmVzb2x1dGlvblN0YXR1c0R0bzoCOAEaQgogU3RvcmVBbmRGb3J3YXJk",
"QnVmZmVyRGVwdGhzRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgF",
"OgI4ASJjChZTaXRlSGVhbHRoUmVwb3J0QWNrRHRvEg8KB3NpdGVfaWQYASAB",
"KAkSFwoPc2VxdWVuY2VfbnVtYmVyGAIgASgDEhAKCGFjY2VwdGVkGAMgASgI",
"Eg0KBWVycm9yGAQgASgJIosBCgxIZWFydGJlYXREdG8SDwoHc2l0ZV9pZBgB",
"IAEoCRIVCg1ub2RlX2hvc3RuYW1lGAIgASgJEhEKCWlzX2FjdGl2ZRgDIAEo",
"CBItCgl0aW1lc3RhbXAYBCABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0",
"YW1wEhEKCXN5bnRoZXRpYxgFIAEoCCq9AQoUQ29ubmVjdGlvbkhlYWx0aEVu",
"dW0SIQodQ09OTkVDVElPTl9IRUFMVEhfVU5TUEVDSUZJRUQQABIfChtDT05O",
"RUNUSU9OX0hFQUxUSF9DT05ORUNURUQQARIiCh5DT05ORUNUSU9OX0hFQUxU",
"SF9ESVNDT05ORUNURUQQAhIgChxDT05ORUNUSU9OX0hFQUxUSF9DT05ORUNU",
"SU5HEAMSGwoXQ09OTkVDVElPTl9IRUFMVEhfRVJST1IQBDKhBgoVQ2VudHJh",
"bENvbnRyb2xTZXJ2aWNlEoMBChJTdWJtaXROb3RpZmljYXRpb24SNC5zY2Fk",
"YWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5Ob3RpZmljYXRpb25TdWJtaXRE",
"dG8aNy5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5Ob3RpZmljYXRp",
"b25TdWJtaXRBY2tEdG8SkgEKF1F1ZXJ5Tm90aWZpY2F0aW9uU3RhdHVzEjku",
"c2NhZGFicmlkZ2UuY2VudHJhbGNvbnRyb2wudjEuTm90aWZpY2F0aW9uU3Rh",
"dHVzUXVlcnlEdG8aPC5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5O",
"b3RpZmljYXRpb25TdGF0dXNSZXNwb25zZUR0bxJHChFJbmdlc3RBdWRpdEV2",
"ZW50cxIbLnNpdGVzdHJlYW0uQXVkaXRFdmVudEJhdGNoGhUuc2l0ZXN0cmVh",
"bS5Jbmdlc3RBY2sSUAoVSW5nZXN0Q2FjaGVkVGVsZW1ldHJ5EiAuc2l0ZXN0",
"cmVhbS5DYWNoZWRUZWxlbWV0cnlCYXRjaBoVLnNpdGVzdHJlYW0uSW5nZXN0",
"QWNrEoABCg1SZWNvbmNpbGVTaXRlEjYuc2NhZGFicmlkZ2UuY2VudHJhbGNv",
"bnRyb2wudjEuUmVjb25jaWxlU2l0ZVJlcXVlc3REdG8aNy5zY2FkYWJyaWRn",
"ZS5jZW50cmFsY29udHJvbC52MS5SZWNvbmNpbGVTaXRlUmVzcG9uc2VEdG8S",
"fQoQUmVwb3J0U2l0ZUhlYWx0aBIyLnNjYWRhYnJpZGdlLmNlbnRyYWxjb250",
"cm9sLnYxLlNpdGVIZWFsdGhSZXBvcnREdG8aNS5zY2FkYWJyaWRnZS5jZW50",
"cmFsY29udHJvbC52MS5TaXRlSGVhbHRoUmVwb3J0QWNrRHRvElAKCUhlYXJ0",
"YmVhdBIrLnNjYWRhYnJpZGdlLmNlbnRyYWxjb250cm9sLnYxLkhlYXJ0YmVh",
"dER0bxoWLmdvb2dsZS5wcm90b2J1Zi5FbXB0eUIrqgIoWkIuTU9NLldXLlNj",
"YWRhQnJpZGdlLkNvbW11bmljYXRpb24uR3JwY2IGcHJvdG8z"));
"ZS5wcm90b2J1Zi5Cb29sVmFsdWUSHwoXZGV0YWNoZWRfc2NyaXB0X3RocmVh",
"ZHMYHiABKAUSHQoVc2NyaXB0X3J1bl9zaGVkX2NvdW50GB8gASgFEiQKHHNp",
"dGVfc3RyZWFtX2FsYXJtX2Ryb3BfY291bnQYICABKAMacgobRGF0YUNvbm5l",
"Y3Rpb25TdGF0dXNlc0VudHJ5EgsKA2tleRgBIAEoCRJCCgV2YWx1ZRgCIAEo",
"DjIzLnNjYWRhYnJpZGdlLmNlbnRyYWxjb250cm9sLnYxLkNvbm5lY3Rpb25I",
"ZWFsdGhFbnVtOgI4ARpxChhUYWdSZXNvbHV0aW9uQ291bnRzRW50cnkSCwoD",
"a2V5GAEgASgJEkQKBXZhbHVlGAIgASgLMjUuc2NhZGFicmlkZ2UuY2VudHJh",
"bGNvbnRyb2wudjEuVGFnUmVzb2x1dGlvblN0YXR1c0R0bzoCOAEaQgogU3Rv",
"cmVBbmRGb3J3YXJkQnVmZmVyRGVwdGhzRW50cnkSCwoDa2V5GAEgASgJEg0K",
"BXZhbHVlGAIgASgFOgI4ASJjChZTaXRlSGVhbHRoUmVwb3J0QWNrRHRvEg8K",
"B3NpdGVfaWQYASABKAkSFwoPc2VxdWVuY2VfbnVtYmVyGAIgASgDEhAKCGFj",
"Y2VwdGVkGAMgASgIEg0KBWVycm9yGAQgASgJIosBCgxIZWFydGJlYXREdG8S",
"DwoHc2l0ZV9pZBgBIAEoCRIVCg1ub2RlX2hvc3RuYW1lGAIgASgJEhEKCWlz",
"X2FjdGl2ZRgDIAEoCBItCgl0aW1lc3RhbXAYBCABKAsyGi5nb29nbGUucHJv",
"dG9idWYuVGltZXN0YW1wEhEKCXN5bnRoZXRpYxgFIAEoCCq9AQoUQ29ubmVj",
"dGlvbkhlYWx0aEVudW0SIQodQ09OTkVDVElPTl9IRUFMVEhfVU5TUEVDSUZJ",
"RUQQABIfChtDT05ORUNUSU9OX0hFQUxUSF9DT05ORUNURUQQARIiCh5DT05O",
"RUNUSU9OX0hFQUxUSF9ESVNDT05ORUNURUQQAhIgChxDT05ORUNUSU9OX0hF",
"QUxUSF9DT05ORUNUSU5HEAMSGwoXQ09OTkVDVElPTl9IRUFMVEhfRVJST1IQ",
"BDKhBgoVQ2VudHJhbENvbnRyb2xTZXJ2aWNlEoMBChJTdWJtaXROb3RpZmlj",
"YXRpb24SNC5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5Ob3RpZmlj",
"YXRpb25TdWJtaXREdG8aNy5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52",
"MS5Ob3RpZmljYXRpb25TdWJtaXRBY2tEdG8SkgEKF1F1ZXJ5Tm90aWZpY2F0",
"aW9uU3RhdHVzEjkuc2NhZGFicmlkZ2UuY2VudHJhbGNvbnRyb2wudjEuTm90",
"aWZpY2F0aW9uU3RhdHVzUXVlcnlEdG8aPC5zY2FkYWJyaWRnZS5jZW50cmFs",
"Y29udHJvbC52MS5Ob3RpZmljYXRpb25TdGF0dXNSZXNwb25zZUR0bxJHChFJ",
"bmdlc3RBdWRpdEV2ZW50cxIbLnNpdGVzdHJlYW0uQXVkaXRFdmVudEJhdGNo",
"GhUuc2l0ZXN0cmVhbS5Jbmdlc3RBY2sSUAoVSW5nZXN0Q2FjaGVkVGVsZW1l",
"dHJ5EiAuc2l0ZXN0cmVhbS5DYWNoZWRUZWxlbWV0cnlCYXRjaBoVLnNpdGVz",
"dHJlYW0uSW5nZXN0QWNrEoABCg1SZWNvbmNpbGVTaXRlEjYuc2NhZGFicmlk",
"Z2UuY2VudHJhbGNvbnRyb2wudjEuUmVjb25jaWxlU2l0ZVJlcXVlc3REdG8a",
"Ny5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5SZWNvbmNpbGVTaXRl",
"UmVzcG9uc2VEdG8SfQoQUmVwb3J0U2l0ZUhlYWx0aBIyLnNjYWRhYnJpZGdl",
"LmNlbnRyYWxjb250cm9sLnYxLlNpdGVIZWFsdGhSZXBvcnREdG8aNS5zY2Fk",
"YWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5TaXRlSGVhbHRoUmVwb3J0QWNr",
"RHRvElAKCUhlYXJ0YmVhdBIrLnNjYWRhYnJpZGdlLmNlbnRyYWxjb250cm9s",
"LnYxLkhlYXJ0YmVhdER0bxoWLmdvb2dsZS5wcm90b2J1Zi5FbXB0eUIrqgIo",
"WkIuTU9NLldXLlNjYWRhQnJpZGdlLkNvbW11bmljYXRpb24uR3JwY2IGcHJv",
"dG8z"));
descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
new pbr::FileDescriptor[] { global::Google.Protobuf.WellKnownTypes.EmptyReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.TimestampReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.WrappersReflection.Descriptor, global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SitestreamReflection.Descriptor, },
new pbr::GeneratedClrTypeInfo(new[] {typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.ConnectionHealthEnum), }, null, new pbr::GeneratedClrTypeInfo[] {
@@ -160,7 +163,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.ConnectionEndpointMapDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.ConnectionEndpointMapDto.Parser, new[]{ "Entries" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { null, }),
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.TagQualityMapDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.TagQualityMapDto.Parser, new[]{ "Entries" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { null, }),
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.NodeStatusListDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.NodeStatusListDto.Parser, new[]{ "Nodes" }, null, null, null, null),
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportDto.Parser, new[]{ "SiteId", "SequenceNumber", "ReportTimestamp", "DataConnectionStatuses", "TagResolutionCounts", "ScriptErrorCount", "AlarmEvaluationErrorCount", "StoreAndForwardBufferDepths", "DeadLetterCount", "DeployedInstanceCount", "EnabledInstanceCount", "DisabledInstanceCount", "NodeRole", "NodeHostname", "DataConnectionEndpoints", "DataConnectionTagQuality", "ParkedMessageCount", "ClusterNodes", "SiteAuditWriteFailures", "AuditRedactionFailure", "SiteAuditBacklog", "SiteEventLogWriteFailures", "OldestParkedMessageAgeSeconds", "ScriptQueueDepth", "ScriptBusyThreads", "ScriptOldestBusyAgeSeconds", "LocalDbReplicationConnected", "LocalDbOplogBacklog", "StartupReconcilerActive" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { null, null, null, }),
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportDto.Parser, new[]{ "SiteId", "SequenceNumber", "ReportTimestamp", "DataConnectionStatuses", "TagResolutionCounts", "ScriptErrorCount", "AlarmEvaluationErrorCount", "StoreAndForwardBufferDepths", "DeadLetterCount", "DeployedInstanceCount", "EnabledInstanceCount", "DisabledInstanceCount", "NodeRole", "NodeHostname", "DataConnectionEndpoints", "DataConnectionTagQuality", "ParkedMessageCount", "ClusterNodes", "SiteAuditWriteFailures", "AuditRedactionFailure", "SiteAuditBacklog", "SiteEventLogWriteFailures", "OldestParkedMessageAgeSeconds", "ScriptQueueDepth", "ScriptBusyThreads", "ScriptOldestBusyAgeSeconds", "LocalDbReplicationConnected", "LocalDbOplogBacklog", "StartupReconcilerActive", "DetachedScriptThreads", "ScriptRunShedCount", "SiteStreamAlarmDropCount" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { null, null, null, }),
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportAckDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportAckDto.Parser, new[]{ "SiteId", "SequenceNumber", "Accepted", "Error" }, null, null, null, null),
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.HeartbeatDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.HeartbeatDto.Parser, new[]{ "SiteId", "NodeHostname", "IsActive", "Timestamp", "Synthetic" }, null, null, null, null)
}));
@@ -4268,6 +4271,9 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
LocalDbReplicationConnected = other.LocalDbReplicationConnected;
LocalDbOplogBacklog = other.LocalDbOplogBacklog;
StartupReconcilerActive = other.StartupReconcilerActive;
detachedScriptThreads_ = other.detachedScriptThreads_;
scriptRunShedCount_ = other.scriptRunShedCount_;
siteStreamAlarmDropCount_ = other.siteStreamAlarmDropCount_;
_unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields);
}
@@ -4665,6 +4671,56 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
}
/// <summary>Field number for the "detached_script_threads" field.</summary>
public const int DetachedScriptThreadsFieldNumber = 30;
private int detachedScriptThreads_;
/// <summary>
/// WP3.1: script-execution worker threads detached and replaced by the stuck-script
/// watchdog and not yet exited. Point-in-time gauge (like script_busy_threads above),
/// not interval-drained. Zero is the healthy state.
/// </summary>
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
public int DetachedScriptThreads {
get { return detachedScriptThreads_; }
set {
detachedScriptThreads_ = value;
}
}
/// <summary>Field number for the "script_run_shed_count" field.</summary>
public const int ScriptRunShedCountFieldNumber = 31;
private int scriptRunShedCount_;
/// <summary>
/// WP3.1: per-interval count of script/alarm on-trigger runs shed by
/// MaxConcurrentRunsPerScript. Interval-drained like script_error_count.
/// </summary>
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
public int ScriptRunShedCount {
get { return scriptRunShedCount_; }
set {
scriptRunShedCount_ = value;
}
}
/// <summary>Field number for the "site_stream_alarm_drop_count" field.</summary>
public const int SiteStreamAlarmDropCountFieldNumber = 32;
private long siteStreamAlarmDropCount_;
/// <summary>
/// WP2.6d: cumulative count of alarm state changes dropped from SiteStreamManager's
/// dedicated alarm publish queue because it was at capacity. Point-in-time
/// (not reset on collect), like site_event_log_write_failures.
/// </summary>
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
public long SiteStreamAlarmDropCount {
get { return siteStreamAlarmDropCount_; }
set {
siteStreamAlarmDropCount_ = value;
}
}
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
public override bool Equals(object other) {
@@ -4709,6 +4765,9 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
if (LocalDbReplicationConnected != other.LocalDbReplicationConnected) return false;
if (LocalDbOplogBacklog != other.LocalDbOplogBacklog) return false;
if (StartupReconcilerActive != other.StartupReconcilerActive) return false;
if (DetachedScriptThreads != other.DetachedScriptThreads) return false;
if (ScriptRunShedCount != other.ScriptRunShedCount) return false;
if (SiteStreamAlarmDropCount != other.SiteStreamAlarmDropCount) return false;
return Equals(_unknownFields, other._unknownFields);
}
@@ -4745,6 +4804,9 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
if (localDbReplicationConnected_ != null) hash ^= LocalDbReplicationConnected.GetHashCode();
if (localDbOplogBacklog_ != null) hash ^= LocalDbOplogBacklog.GetHashCode();
if (startupReconcilerActive_ != null) hash ^= StartupReconcilerActive.GetHashCode();
if (DetachedScriptThreads != 0) hash ^= DetachedScriptThreads.GetHashCode();
if (ScriptRunShedCount != 0) hash ^= ScriptRunShedCount.GetHashCode();
if (SiteStreamAlarmDropCount != 0L) hash ^= SiteStreamAlarmDropCount.GetHashCode();
if (_unknownFields != null) {
hash ^= _unknownFields.GetHashCode();
}
@@ -4865,6 +4927,18 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
if (startupReconcilerActive_ != null) {
_single_startupReconcilerActive_codec.WriteTagAndValue(output, StartupReconcilerActive);
}
if (DetachedScriptThreads != 0) {
output.WriteRawTag(240, 1);
output.WriteInt32(DetachedScriptThreads);
}
if (ScriptRunShedCount != 0) {
output.WriteRawTag(248, 1);
output.WriteInt32(ScriptRunShedCount);
}
if (SiteStreamAlarmDropCount != 0L) {
output.WriteRawTag(128, 2);
output.WriteInt64(SiteStreamAlarmDropCount);
}
if (_unknownFields != null) {
_unknownFields.WriteTo(output);
}
@@ -4977,6 +5051,18 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
if (startupReconcilerActive_ != null) {
_single_startupReconcilerActive_codec.WriteTagAndValue(ref output, StartupReconcilerActive);
}
if (DetachedScriptThreads != 0) {
output.WriteRawTag(240, 1);
output.WriteInt32(DetachedScriptThreads);
}
if (ScriptRunShedCount != 0) {
output.WriteRawTag(248, 1);
output.WriteInt32(ScriptRunShedCount);
}
if (SiteStreamAlarmDropCount != 0L) {
output.WriteRawTag(128, 2);
output.WriteInt64(SiteStreamAlarmDropCount);
}
if (_unknownFields != null) {
_unknownFields.WriteTo(ref output);
}
@@ -5068,6 +5154,15 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
if (startupReconcilerActive_ != null) {
size += _single_startupReconcilerActive_codec.CalculateSizeWithTag(StartupReconcilerActive);
}
if (DetachedScriptThreads != 0) {
size += 2 + pb::CodedOutputStream.ComputeInt32Size(DetachedScriptThreads);
}
if (ScriptRunShedCount != 0) {
size += 2 + pb::CodedOutputStream.ComputeInt32Size(ScriptRunShedCount);
}
if (SiteStreamAlarmDropCount != 0L) {
size += 2 + pb::CodedOutputStream.ComputeInt64Size(SiteStreamAlarmDropCount);
}
if (_unknownFields != null) {
size += _unknownFields.CalculateSize();
}
@@ -5186,6 +5281,15 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
StartupReconcilerActive = other.StartupReconcilerActive;
}
}
if (other.DetachedScriptThreads != 0) {
DetachedScriptThreads = other.DetachedScriptThreads;
}
if (other.ScriptRunShedCount != 0) {
ScriptRunShedCount = other.ScriptRunShedCount;
}
if (other.SiteStreamAlarmDropCount != 0L) {
SiteStreamAlarmDropCount = other.SiteStreamAlarmDropCount;
}
_unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields);
}
@@ -5351,6 +5455,18 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
}
break;
}
case 240: {
DetachedScriptThreads = input.ReadInt32();
break;
}
case 248: {
ScriptRunShedCount = input.ReadInt32();
break;
}
case 256: {
SiteStreamAlarmDropCount = input.ReadInt64();
break;
}
}
}
#endif
@@ -5516,6 +5632,18 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
}
break;
}
case 240: {
DetachedScriptThreads = input.ReadInt32();
break;
}
case 248: {
ScriptRunShedCount = input.ReadInt32();
break;
}
case 256: {
SiteStreamAlarmDropCount = input.ReadInt64();
break;
}
}
}
}
src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/CentralControlDtoMapper.cs
+6
View File
@@ -476,6 +476,9 @@ public static class CentralControlDtoMapper
LocalDbReplicationConnected = msg.LocalDbReplicationConnected,
LocalDbOplogBacklog = msg.LocalDbOplogBacklog,
StartupReconcilerActive = msg.StartupReconcilerActive,
DetachedScriptThreads = msg.DetachedScriptThreads,
ScriptRunShedCount = msg.ScriptRunShedCount,
SiteStreamAlarmDropCount = msg.SiteStreamAlarmDropCount,
};
foreach (var (name, health) in msg.DataConnectionStatuses)
@@ -651,6 +654,9 @@ public static class CentralControlDtoMapper
LocalDbReplicationConnected = dto.LocalDbReplicationConnected,
LocalDbOplogBacklog = dto.LocalDbOplogBacklog,
StartupReconcilerActive = dto.StartupReconcilerActive,
DetachedScriptThreads = dto.DetachedScriptThreads,
ScriptRunShedCount = dto.ScriptRunShedCount,
SiteStreamAlarmDropCount = dto.SiteStreamAlarmDropCount,
};
}
src/ZB.MOM.WW.ScadaBridge.Communication/Protos/central_control.proto
+11
View File
@@ -234,6 +234,17 @@ message SiteHealthReportDto {
// self-heal path, which is a misconfiguration a healthy-looking site would
// otherwise hide entirely.
google.protobuf.BoolValue startup_reconciler_active = 29;
// WP3.1: script-execution worker threads detached and replaced by the stuck-script
// watchdog and not yet exited. Point-in-time gauge (like script_busy_threads above),
// not interval-drained. Zero is the healthy state.
int32 detached_script_threads = 30;
// WP3.1: per-interval count of script/alarm on-trigger runs shed by
// MaxConcurrentRunsPerScript. Interval-drained like script_error_count.
int32 script_run_shed_count = 31;
// WP2.6d: cumulative count of alarm state changes dropped from SiteStreamManager's
// dedicated alarm publish queue because it was at capacity. Point-in-time
// (not reset on collect), like site_event_log_write_failures.
int64 site_stream_alarm_drop_count = 32;
}
// Central -> Site: health report ack, so a lost report is observable.
src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/CentralHealthReportLoop.cs
+2 -1
View File
@@ -132,7 +132,8 @@ public class CentralHealthReportLoop : BackgroundService
alarmErrors: report.AlarmEvaluationErrorCount,
deadLetters: report.DeadLetterCount,
siteAuditWriteFailures: report.SiteAuditWriteFailures,
auditRedactionFailures: report.AuditRedactionFailure);
auditRedactionFailures: report.AuditRedactionFailure,
scriptRunShedCount: report.ScriptRunShedCount);
throw;
}
src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/HealthReportSender.cs
+2 -1
View File
@@ -182,7 +182,8 @@ public class HealthReportSender : BackgroundService
alarmErrors: report.AlarmEvaluationErrorCount,
deadLetters: report.DeadLetterCount,
siteAuditWriteFailures: report.SiteAuditWriteFailures,
auditRedactionFailures: report.AuditRedactionFailure);
auditRedactionFailures: report.AuditRedactionFailure,
scriptRunShedCount: report.ScriptRunShedCount);
throw;
}
src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/ISiteHealthCollector.cs
+8 -1
View File
@@ -286,12 +286,19 @@ public interface ISiteHealthCollector
/// <param name="deadLetters">Dead letter count to add back.</param>
/// <param name="siteAuditWriteFailures">Site audit write failure count to add back.</param>
/// <param name="auditRedactionFailures">Audit redaction failure count to add back.</param>
/// <param name="scriptRunShedCount">
/// Interval-drained script/alarm run-shed count to add back (arch-review 2026-08-19
/// P8-1). Additive trailing parameter defaulting to 0 so existing callers keep
/// compiling; <c>DetachedScriptThreads</c>/<c>SiteStreamAlarmDropCount</c> are
/// point-in-time gauges, not drained on collect, so they need no restore path.
/// </param>
void AddIntervalCounters(
int scriptErrors,
int alarmErrors,
int deadLetters,
int siteAuditWriteFailures,
int auditRedactionFailures)
int auditRedactionFailures,
int scriptRunShedCount = 0)
{
// Default no-op so test fakes do not need to be updated. The real
// SiteHealthCollector overrides this with the Interlocked.Add restore.
src/ZB.MOM.WW.ScadaBridge.HealthMonitoring/SiteHealthCollector.cs
+9 -3
View File
@@ -245,19 +245,25 @@ public class SiteHealthCollector : ISiteHealthCollector
int alarmErrors,
int deadLetters,
int siteAuditWriteFailures,
int auditRedactionFailures)
int auditRedactionFailures,
int scriptRunShedCount = 0)
{
// Each counter is restored atomically via
// Interlocked.Add so an increment that arrived during the failed Send
// (and therefore accumulated against the zero left by CollectReport's
// Exchange) is correctly summed with the values being put back. No
// ordering between the five Adds is required — they target independent
// fields.
// ordering between the six Adds is required — they target independent
// fields. scriptRunShedCount joins this restore path (arch-review
// 2026-08-19 P8-1) because it is drained by CollectReport the same way
// as the other five; DetachedScriptThreads and SiteStreamAlarmDropCount
// are point-in-time gauges (CompareExchange/Read, never zeroed on
// collect) so a failed Send loses nothing for them to restore.
if (scriptErrors != 0) Interlocked.Add(ref _scriptErrorCount, scriptErrors);
if (alarmErrors != 0) Interlocked.Add(ref _alarmErrorCount, alarmErrors);
if (deadLetters != 0) Interlocked.Add(ref _deadLetterCount, deadLetters);
if (siteAuditWriteFailures != 0) Interlocked.Add(ref _siteAuditWriteFailures, siteAuditWriteFailures);
if (auditRedactionFailures != 0) Interlocked.Add(ref _auditRedactionFailures, auditRedactionFailures);
if (scriptRunShedCount != 0) Interlocked.Add(ref _scriptRunShedCount, scriptRunShedCount);
}
/// <inheritdoc />
tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/Grpc/CentralControlDtoMapperTests.cs
+8
View File
@@ -554,6 +554,11 @@ public class CentralControlDtoMapperTests
LocalDbReplicationConnected = true,
LocalDbOplogBacklog = 250L,
StartupReconcilerActive = true,
// arch-review 2026-08-19 P8-1: these three never crossed the wire at all
// (mapper omitted them entirely, always reading back as 0) until this fix.
DetachedScriptThreads = 2,
ScriptRunShedCount = 13,
SiteStreamAlarmDropCount = 41L,
};
/// <summary>
@@ -603,6 +608,9 @@ public class CentralControlDtoMapperTests
Assert.Equal(expected.LocalDbReplicationConnected, actual.LocalDbReplicationConnected);
Assert.Equal(expected.LocalDbOplogBacklog, actual.LocalDbOplogBacklog);
Assert.Equal(expected.StartupReconcilerActive, actual.StartupReconcilerActive);
Assert.Equal(expected.DetachedScriptThreads, actual.DetachedScriptThreads);
Assert.Equal(expected.ScriptRunShedCount, actual.ScriptRunShedCount);
Assert.Equal(expected.SiteStreamAlarmDropCount, actual.SiteStreamAlarmDropCount);
}
/// <summary>
tests/ZB.MOM.WW.ScadaBridge.HealthMonitoring.Tests/HealthReportSenderTests.cs
+16 -1
View File
@@ -333,6 +333,16 @@ public class HealthReportSenderTests
/// un-sent report only, and the next successful report shipped with the
/// counters at zero.
/// </summary>
/// <remarks>
/// Extended for arch-review 2026-08-19 P8-1: <c>ScriptRunShedCount</c> is drained
/// by <c>CollectReport</c> the exact same way as the five counters above, but
/// <c>AddIntervalCounters</c> used to take only those five — a failed send
/// destructively lost the shed count forever. <c>DetachedScriptThreads</c> and
/// <c>SiteStreamAlarmDropCount</c> are deliberately NOT exercised here: they are
/// point-in-time gauges (set via <c>Interlocked.CompareExchange</c>/<c>Read</c>,
/// never zeroed by <c>CollectReport</c>), so a failed send loses nothing for them
/// to restore.
/// </remarks>
[Fact]
public async Task SendFailure_PreservesIntervalCountersForNextReport()
{
@@ -341,7 +351,7 @@ public class HealthReportSenderTests
collector.SetActiveNode(true);
// Pre-populate every per-interval counter so the restore path on each
// field is exercised — script error, alarm error, dead letter, site
// audit write failure, audit redaction failure.
// audit write failure, audit redaction failure, script run shed.
collector.IncrementScriptError();
collector.IncrementScriptError();
collector.IncrementAlarmError();
@@ -351,6 +361,10 @@ public class HealthReportSenderTests
collector.IncrementSiteAuditWriteFailures();
collector.IncrementAuditRedactionFailure();
collector.IncrementAuditRedactionFailure();
collector.IncrementScriptRunShed();
collector.IncrementScriptRunShed();
collector.IncrementScriptRunShed();
collector.IncrementScriptRunShed();
var options = Options.Create(new HealthMonitoringOptions
{
@@ -383,6 +397,7 @@ public class HealthReportSenderTests
Assert.Equal(3, firstSuccess.DeadLetterCount);
Assert.Equal(1, firstSuccess.SiteAuditWriteFailures);
Assert.Equal(2, firstSuccess.AuditRedactionFailure);
Assert.Equal(4, firstSuccess.ScriptRunShedCount);
}
/// <summary>