Merge rows 80+73+71: nullable LocalDb health fields on unreplicated sites (Host supplies ReplicationIsConfigured), StartupReconcilerActive on the health report (proto field 29) with central No-self-heal badge, standby active-node ERR downgraded via narrowly-keyed Serilog filter
# Conflicts: # CLAUDE.md
This commit is contained in:
Joseph Doherty
@@ -267,6 +267,14 @@
|
||||
Replication backlog
|
||||
</span>
|
||||
}
|
||||
@if (IsStartupReconcilerMissing(state))
|
||||
{
|
||||
<span class="badge bg-warning text-dark"
|
||||
data-test="startup-reconciler-badge"
|
||||
title="This site reported that a node booted without its SiteReconciliationActor (no IDeploymentConfigFetcher), so the startup self-heal is disabled: a deploy that lands while the node is DOWN will not be recovered on restart. Config still reaches the node via LocalDb replication while it is up. Check the node's configuration.">
|
||||
No self-heal
|
||||
</span>
|
||||
}
|
||||
@if (!state.IsOnline && state.LastStatusChangeAt is { } changedAt)
|
||||
{
|
||||
<small class="text-muted">offline since @changedAt.ToString("u")</small>
|
||||
@@ -566,6 +574,22 @@
|
||||
&& state.LocalDbBacklogAboveSince is { } since
|
||||
&& DateTimeOffset.UtcNow - since >= HealthOptions.Value.LocalDbBacklogSustainedFor;
|
||||
|
||||
// Register row 73: the "No self-heal" badge.
|
||||
//
|
||||
// Warning-level treatment, deliberately not an error and never an offline-style state:
|
||||
// the node still receives config through LocalDb replication while it is up, so what it
|
||||
// has lost is recovery from a deploy that landed while it was DOWN. That is degraded,
|
||||
// not down — which is exactly what a warning badge says.
|
||||
//
|
||||
// Tri-state, and only the explicit false is badged. Null means "not reported": the
|
||||
// central self-report row (where a site reconciler is meaningless) and any site node on
|
||||
// a pre-row-73 build both send null, and badging those would fire on every site the
|
||||
// moment central is upgraded ahead of the sites. Offline sites are skipped for the same
|
||||
// reason the backlog badge skips them — the Offline badge is the signal that matters.
|
||||
private bool IsStartupReconcilerMissing(SiteHealthState state) =>
|
||||
state.IsOnline
|
||||
&& state.LatestReport?.StartupReconcilerActive == false;
|
||||
|
||||
// Online central nodes, from the same ClusterNodes list the Nodes column renders.
|
||||
// Fewer than two means no standby, which disables the manual-failover control. This is
|
||||
// a display-side guard only — AkkaManualFailoverService re-checks against live cluster
|
||||
|
||||
@@ -103,12 +103,23 @@ public record SiteHealthReport(
|
||||
/// or <see langword="null"/> when replication is not wired on this node.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Nullable on purpose, and <see langword="false"/> is NOT the same as null.
|
||||
/// Replication ships default-OFF, so a site node with no peer configured reports
|
||||
/// <see langword="false"/> — that is a healthy, expected state, not an outage. Only
|
||||
/// a node that was configured with a peer and is reporting <see langword="false"/>
|
||||
/// is degraded, and distinguishing those two cases is the operator's job, not this
|
||||
/// field's.
|
||||
/// <b>Null means "no reading"</b> — replication is not configured on this node
|
||||
/// (the default-OFF posture), or the reporter has not run yet.
|
||||
/// <b><see langword="false"/> means "configured, and the peer session is down"</b>,
|
||||
/// which is a genuine fault worth investigating.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// The site side must never publish <see langword="false"/> for an unconfigured node
|
||||
/// (deferred-work register row 80): the replication engine is registered
|
||||
/// unconditionally and answers <c>Connected=false</c> while inert, so passing that
|
||||
/// through made every deliberately-unreplicated site look like a broken replicated
|
||||
/// one, and starved central's null branches (conditional KPI emission,
|
||||
/// sustained-badge anchor exclusion) of the null they were built for.
|
||||
/// <c>LocalDbReplicationStatusReporter</c> gates on the Host-supplied
|
||||
/// <c>SiteLocalDbSetup.ReplicationIsConfigured</c> to keep that distinction honest.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public bool? LocalDbReplicationConnected { get; init; }
|
||||
|
||||
@@ -134,6 +145,34 @@ public record SiteHealthReport(
|
||||
/// 0 so existing producers/tests that don't populate it stay valid.
|
||||
/// </summary>
|
||||
public long SiteStreamAlarmDropCount { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether this node spawned its <c>SiteReconciliationActor</c> at boot — the per-node
|
||||
/// startup self-heal that lets a standby which was DOWN during a deploy recover its
|
||||
/// deployed inventory (and, since register row 67, restart the Instance Actors a
|
||||
/// boot-time load missed).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// Tri-state, and the distinction is the whole point (deferred-work register row 73).
|
||||
/// <b>Null = not reported</b>: a central self-report, where the notion does not apply,
|
||||
/// or a site node running a pre-row-73 build. <b><see langword="true"/></b> = the
|
||||
/// reconciler is running. <b><see langword="false"/></b> = the node booted WITHOUT it,
|
||||
/// because no <c>IDeploymentConfigFetcher</c> was resolvable.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// That false case used to be invisible: <c>AkkaHostedService</c> logged a single boot
|
||||
/// Warning and spawned nothing, so a misconfigured site reported itself perfectly
|
||||
/// healthy forever while being unable to self-heal. Central badges it rather than
|
||||
/// failing the site — replication remains the primary path, so the node is degraded,
|
||||
/// not down.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Stamped once at boot by the Host (which owns the spawn decision) via
|
||||
/// <c>ISiteHealthCollector.SetStartupReconcilerActive</c>; it is not polled.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public bool? StartupReconcilerActive { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -72,7 +72,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
"ZXkYASABKAkSQQoFdmFsdWUYAiABKAsyMi5zY2FkYWJyaWRnZS5jZW50cmFs",
|
||||
"Y29udHJvbC52MS5UYWdRdWFsaXR5Q291bnRzRHRvOgI4ASJQChFOb2RlU3Rh",
|
||||
"dHVzTGlzdER0bxI7CgVub2RlcxgBIAMoCzIsLnNjYWRhYnJpZGdlLmNlbnRy",
|
||||
"YWxjb250cm9sLnYxLk5vZGVTdGF0dXNEdG8iig4KE1NpdGVIZWFsdGhSZXBv",
|
||||
"YWxjb250cm9sLnYxLk5vZGVTdGF0dXNEdG8iyQ4KE1NpdGVIZWFsdGhSZXBv",
|
||||
"cnREdG8SDwoHc2l0ZV9pZBgBIAEoCRIXCg9zZXF1ZW5jZV9udW1iZXIYAiAB",
|
||||
"KAMSNAoQcmVwb3J0X3RpbWVzdGFtcBgDIAEoCzIaLmdvb2dsZS5wcm90b2J1",
|
||||
"Zi5UaW1lc3RhbXAScAoYZGF0YV9jb25uZWN0aW9uX3N0YXR1c2VzGAQgAygL",
|
||||
@@ -106,42 +106,43 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
"YmxlVmFsdWUSQgoebG9jYWxfZGJfcmVwbGljYXRpb25fY29ubmVjdGVkGBsg",
|
||||
"ASgLMhouZ29vZ2xlLnByb3RvYnVmLkJvb2xWYWx1ZRI7ChZsb2NhbF9kYl9v",
|
||||
"cGxvZ19iYWNrbG9nGBwgASgLMhsuZ29vZ2xlLnByb3RvYnVmLkludDY0VmFs",
|
||||
"dWUacgobRGF0YUNvbm5lY3Rpb25TdGF0dXNlc0VudHJ5EgsKA2tleRgBIAEo",
|
||||
"CRJCCgV2YWx1ZRgCIAEoDjIzLnNjYWRhYnJpZGdlLmNlbnRyYWxjb250cm9s",
|
||||
"LnYxLkNvbm5lY3Rpb25IZWFsdGhFbnVtOgI4ARpxChhUYWdSZXNvbHV0aW9u",
|
||||
"Q291bnRzRW50cnkSCwoDa2V5GAEgASgJEkQKBXZhbHVlGAIgASgLMjUuc2Nh",
|
||||
"ZGFicmlkZ2UuY2VudHJhbGNvbnRyb2wudjEuVGFnUmVzb2x1dGlvblN0YXR1",
|
||||
"c0R0bzoCOAEaQgogU3RvcmVBbmRGb3J3YXJkQnVmZmVyRGVwdGhzRW50cnkS",
|
||||
"CwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgFOgI4ASJjChZTaXRlSGVhbHRo",
|
||||
"UmVwb3J0QWNrRHRvEg8KB3NpdGVfaWQYASABKAkSFwoPc2VxdWVuY2VfbnVt",
|
||||
"YmVyGAIgASgDEhAKCGFjY2VwdGVkGAMgASgIEg0KBWVycm9yGAQgASgJIosB",
|
||||
"CgxIZWFydGJlYXREdG8SDwoHc2l0ZV9pZBgBIAEoCRIVCg1ub2RlX2hvc3Ru",
|
||||
"YW1lGAIgASgJEhEKCWlzX2FjdGl2ZRgDIAEoCBItCgl0aW1lc3RhbXAYBCAB",
|
||||
"KAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEhEKCXN5bnRoZXRpYxgF",
|
||||
"IAEoCCq9AQoUQ29ubmVjdGlvbkhlYWx0aEVudW0SIQodQ09OTkVDVElPTl9I",
|
||||
"RUFMVEhfVU5TUEVDSUZJRUQQABIfChtDT05ORUNUSU9OX0hFQUxUSF9DT05O",
|
||||
"RUNURUQQARIiCh5DT05ORUNUSU9OX0hFQUxUSF9ESVNDT05ORUNURUQQAhIg",
|
||||
"ChxDT05ORUNUSU9OX0hFQUxUSF9DT05ORUNUSU5HEAMSGwoXQ09OTkVDVElP",
|
||||
"Tl9IRUFMVEhfRVJST1IQBDKhBgoVQ2VudHJhbENvbnRyb2xTZXJ2aWNlEoMB",
|
||||
"ChJTdWJtaXROb3RpZmljYXRpb24SNC5zY2FkYWJyaWRnZS5jZW50cmFsY29u",
|
||||
"dHJvbC52MS5Ob3RpZmljYXRpb25TdWJtaXREdG8aNy5zY2FkYWJyaWRnZS5j",
|
||||
"ZW50cmFsY29udHJvbC52MS5Ob3RpZmljYXRpb25TdWJtaXRBY2tEdG8SkgEK",
|
||||
"F1F1ZXJ5Tm90aWZpY2F0aW9uU3RhdHVzEjkuc2NhZGFicmlkZ2UuY2VudHJh",
|
||||
"bGNvbnRyb2wudjEuTm90aWZpY2F0aW9uU3RhdHVzUXVlcnlEdG8aPC5zY2Fk",
|
||||
"YWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5Ob3RpZmljYXRpb25TdGF0dXNS",
|
||||
"ZXNwb25zZUR0bxJHChFJbmdlc3RBdWRpdEV2ZW50cxIbLnNpdGVzdHJlYW0u",
|
||||
"QXVkaXRFdmVudEJhdGNoGhUuc2l0ZXN0cmVhbS5Jbmdlc3RBY2sSUAoVSW5n",
|
||||
"ZXN0Q2FjaGVkVGVsZW1ldHJ5EiAuc2l0ZXN0cmVhbS5DYWNoZWRUZWxlbWV0",
|
||||
"cnlCYXRjaBoVLnNpdGVzdHJlYW0uSW5nZXN0QWNrEoABCg1SZWNvbmNpbGVT",
|
||||
"aXRlEjYuc2NhZGFicmlkZ2UuY2VudHJhbGNvbnRyb2wudjEuUmVjb25jaWxl",
|
||||
"U2l0ZVJlcXVlc3REdG8aNy5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52",
|
||||
"MS5SZWNvbmNpbGVTaXRlUmVzcG9uc2VEdG8SfQoQUmVwb3J0U2l0ZUhlYWx0",
|
||||
"aBIyLnNjYWRhYnJpZGdlLmNlbnRyYWxjb250cm9sLnYxLlNpdGVIZWFsdGhS",
|
||||
"ZXBvcnREdG8aNS5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5TaXRl",
|
||||
"SGVhbHRoUmVwb3J0QWNrRHRvElAKCUhlYXJ0YmVhdBIrLnNjYWRhYnJpZGdl",
|
||||
"LmNlbnRyYWxjb250cm9sLnYxLkhlYXJ0YmVhdER0bxoWLmdvb2dsZS5wcm90",
|
||||
"b2J1Zi5FbXB0eUIrqgIoWkIuTU9NLldXLlNjYWRhQnJpZGdlLkNvbW11bmlj",
|
||||
"YXRpb24uR3JwY2IGcHJvdG8z"));
|
||||
"dWUSPQoZc3RhcnR1cF9yZWNvbmNpbGVyX2FjdGl2ZRgdIAEoCzIaLmdvb2ds",
|
||||
"ZS5wcm90b2J1Zi5Cb29sVmFsdWUacgobRGF0YUNvbm5lY3Rpb25TdGF0dXNl",
|
||||
"c0VudHJ5EgsKA2tleRgBIAEoCRJCCgV2YWx1ZRgCIAEoDjIzLnNjYWRhYnJp",
|
||||
"ZGdlLmNlbnRyYWxjb250cm9sLnYxLkNvbm5lY3Rpb25IZWFsdGhFbnVtOgI4",
|
||||
"ARpxChhUYWdSZXNvbHV0aW9uQ291bnRzRW50cnkSCwoDa2V5GAEgASgJEkQK",
|
||||
"BXZhbHVlGAIgASgLMjUuc2NhZGFicmlkZ2UuY2VudHJhbGNvbnRyb2wudjEu",
|
||||
"VGFnUmVzb2x1dGlvblN0YXR1c0R0bzoCOAEaQgogU3RvcmVBbmRGb3J3YXJk",
|
||||
"QnVmZmVyRGVwdGhzRW50cnkSCwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgF",
|
||||
"OgI4ASJjChZTaXRlSGVhbHRoUmVwb3J0QWNrRHRvEg8KB3NpdGVfaWQYASAB",
|
||||
"KAkSFwoPc2VxdWVuY2VfbnVtYmVyGAIgASgDEhAKCGFjY2VwdGVkGAMgASgI",
|
||||
"Eg0KBWVycm9yGAQgASgJIosBCgxIZWFydGJlYXREdG8SDwoHc2l0ZV9pZBgB",
|
||||
"IAEoCRIVCg1ub2RlX2hvc3RuYW1lGAIgASgJEhEKCWlzX2FjdGl2ZRgDIAEo",
|
||||
"CBItCgl0aW1lc3RhbXAYBCABKAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0",
|
||||
"YW1wEhEKCXN5bnRoZXRpYxgFIAEoCCq9AQoUQ29ubmVjdGlvbkhlYWx0aEVu",
|
||||
"dW0SIQodQ09OTkVDVElPTl9IRUFMVEhfVU5TUEVDSUZJRUQQABIfChtDT05O",
|
||||
"RUNUSU9OX0hFQUxUSF9DT05ORUNURUQQARIiCh5DT05ORUNUSU9OX0hFQUxU",
|
||||
"SF9ESVNDT05ORUNURUQQAhIgChxDT05ORUNUSU9OX0hFQUxUSF9DT05ORUNU",
|
||||
"SU5HEAMSGwoXQ09OTkVDVElPTl9IRUFMVEhfRVJST1IQBDKhBgoVQ2VudHJh",
|
||||
"bENvbnRyb2xTZXJ2aWNlEoMBChJTdWJtaXROb3RpZmljYXRpb24SNC5zY2Fk",
|
||||
"YWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5Ob3RpZmljYXRpb25TdWJtaXRE",
|
||||
"dG8aNy5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5Ob3RpZmljYXRp",
|
||||
"b25TdWJtaXRBY2tEdG8SkgEKF1F1ZXJ5Tm90aWZpY2F0aW9uU3RhdHVzEjku",
|
||||
"c2NhZGFicmlkZ2UuY2VudHJhbGNvbnRyb2wudjEuTm90aWZpY2F0aW9uU3Rh",
|
||||
"dHVzUXVlcnlEdG8aPC5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5O",
|
||||
"b3RpZmljYXRpb25TdGF0dXNSZXNwb25zZUR0bxJHChFJbmdlc3RBdWRpdEV2",
|
||||
"ZW50cxIbLnNpdGVzdHJlYW0uQXVkaXRFdmVudEJhdGNoGhUuc2l0ZXN0cmVh",
|
||||
"bS5Jbmdlc3RBY2sSUAoVSW5nZXN0Q2FjaGVkVGVsZW1ldHJ5EiAuc2l0ZXN0",
|
||||
"cmVhbS5DYWNoZWRUZWxlbWV0cnlCYXRjaBoVLnNpdGVzdHJlYW0uSW5nZXN0",
|
||||
"QWNrEoABCg1SZWNvbmNpbGVTaXRlEjYuc2NhZGFicmlkZ2UuY2VudHJhbGNv",
|
||||
"bnRyb2wudjEuUmVjb25jaWxlU2l0ZVJlcXVlc3REdG8aNy5zY2FkYWJyaWRn",
|
||||
"ZS5jZW50cmFsY29udHJvbC52MS5SZWNvbmNpbGVTaXRlUmVzcG9uc2VEdG8S",
|
||||
"fQoQUmVwb3J0U2l0ZUhlYWx0aBIyLnNjYWRhYnJpZGdlLmNlbnRyYWxjb250",
|
||||
"cm9sLnYxLlNpdGVIZWFsdGhSZXBvcnREdG8aNS5zY2FkYWJyaWRnZS5jZW50",
|
||||
"cmFsY29udHJvbC52MS5TaXRlSGVhbHRoUmVwb3J0QWNrRHRvElAKCUhlYXJ0",
|
||||
"YmVhdBIrLnNjYWRhYnJpZGdlLmNlbnRyYWxjb250cm9sLnYxLkhlYXJ0YmVh",
|
||||
"dER0bxoWLmdvb2dsZS5wcm90b2J1Zi5FbXB0eUIrqgIoWkIuTU9NLldXLlNj",
|
||||
"YWRhQnJpZGdlLkNvbW11bmljYXRpb24uR3JwY2IGcHJvdG8z"));
|
||||
descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
|
||||
new pbr::FileDescriptor[] { global::Google.Protobuf.WellKnownTypes.EmptyReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.TimestampReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.WrappersReflection.Descriptor, global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SitestreamReflection.Descriptor, },
|
||||
new pbr::GeneratedClrTypeInfo(new[] {typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.ConnectionHealthEnum), }, null, new pbr::GeneratedClrTypeInfo[] {
|
||||
@@ -159,7 +160,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.ConnectionEndpointMapDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.ConnectionEndpointMapDto.Parser, new[]{ "Entries" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { null, }),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.TagQualityMapDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.TagQualityMapDto.Parser, new[]{ "Entries" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { null, }),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.NodeStatusListDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.NodeStatusListDto.Parser, new[]{ "Nodes" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportDto.Parser, new[]{ "SiteId", "SequenceNumber", "ReportTimestamp", "DataConnectionStatuses", "TagResolutionCounts", "ScriptErrorCount", "AlarmEvaluationErrorCount", "StoreAndForwardBufferDepths", "DeadLetterCount", "DeployedInstanceCount", "EnabledInstanceCount", "DisabledInstanceCount", "NodeRole", "NodeHostname", "DataConnectionEndpoints", "DataConnectionTagQuality", "ParkedMessageCount", "ClusterNodes", "SiteAuditWriteFailures", "AuditRedactionFailure", "SiteAuditBacklog", "SiteEventLogWriteFailures", "OldestParkedMessageAgeSeconds", "ScriptQueueDepth", "ScriptBusyThreads", "ScriptOldestBusyAgeSeconds", "LocalDbReplicationConnected", "LocalDbOplogBacklog" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { null, null, null, }),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportDto.Parser, new[]{ "SiteId", "SequenceNumber", "ReportTimestamp", "DataConnectionStatuses", "TagResolutionCounts", "ScriptErrorCount", "AlarmEvaluationErrorCount", "StoreAndForwardBufferDepths", "DeadLetterCount", "DeployedInstanceCount", "EnabledInstanceCount", "DisabledInstanceCount", "NodeRole", "NodeHostname", "DataConnectionEndpoints", "DataConnectionTagQuality", "ParkedMessageCount", "ClusterNodes", "SiteAuditWriteFailures", "AuditRedactionFailure", "SiteAuditBacklog", "SiteEventLogWriteFailures", "OldestParkedMessageAgeSeconds", "ScriptQueueDepth", "ScriptBusyThreads", "ScriptOldestBusyAgeSeconds", "LocalDbReplicationConnected", "LocalDbOplogBacklog", "StartupReconcilerActive" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { null, null, null, }),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportAckDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportAckDto.Parser, new[]{ "SiteId", "SequenceNumber", "Accepted", "Error" }, null, null, null, null),
|
||||
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.HeartbeatDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.HeartbeatDto.Parser, new[]{ "SiteId", "NodeHostname", "IsActive", "Timestamp", "Synthetic" }, null, null, null, null)
|
||||
}));
|
||||
@@ -4266,6 +4267,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
ScriptOldestBusyAgeSeconds = other.ScriptOldestBusyAgeSeconds;
|
||||
LocalDbReplicationConnected = other.LocalDbReplicationConnected;
|
||||
LocalDbOplogBacklog = other.LocalDbOplogBacklog;
|
||||
StartupReconcilerActive = other.StartupReconcilerActive;
|
||||
_unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields);
|
||||
}
|
||||
|
||||
@@ -4642,6 +4644,27 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
}
|
||||
|
||||
|
||||
/// <summary>Field number for the "startup_reconciler_active" field.</summary>
|
||||
public const int StartupReconcilerActiveFieldNumber = 29;
|
||||
private static readonly pb::FieldCodec<bool?> _single_startupReconcilerActive_codec = pb::FieldCodec.ForStructWrapper<bool>(234);
|
||||
private bool? startupReconcilerActive_;
|
||||
/// <summary>
|
||||
/// Whether this node spawned its SiteReconciliationActor at boot. Absent means
|
||||
/// "not reported" — a central self-report (where it is meaningless) or a site
|
||||
/// node on a pre-row-73 build. false means the node booted WITHOUT the startup
|
||||
/// self-heal path, which is a misconfiguration a healthy-looking site would
|
||||
/// otherwise hide entirely.
|
||||
/// </summary>
|
||||
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
|
||||
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
|
||||
public bool? StartupReconcilerActive {
|
||||
get { return startupReconcilerActive_; }
|
||||
set {
|
||||
startupReconcilerActive_ = value;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
|
||||
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
|
||||
public override bool Equals(object other) {
|
||||
@@ -4685,6 +4708,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
if (!pbc::ProtobufEqualityComparers.BitwiseNullableDoubleEqualityComparer.Equals(ScriptOldestBusyAgeSeconds, other.ScriptOldestBusyAgeSeconds)) return false;
|
||||
if (LocalDbReplicationConnected != other.LocalDbReplicationConnected) return false;
|
||||
if (LocalDbOplogBacklog != other.LocalDbOplogBacklog) return false;
|
||||
if (StartupReconcilerActive != other.StartupReconcilerActive) return false;
|
||||
return Equals(_unknownFields, other._unknownFields);
|
||||
}
|
||||
|
||||
@@ -4720,6 +4744,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
if (scriptOldestBusyAgeSeconds_ != null) hash ^= pbc::ProtobufEqualityComparers.BitwiseNullableDoubleEqualityComparer.GetHashCode(ScriptOldestBusyAgeSeconds);
|
||||
if (localDbReplicationConnected_ != null) hash ^= LocalDbReplicationConnected.GetHashCode();
|
||||
if (localDbOplogBacklog_ != null) hash ^= LocalDbOplogBacklog.GetHashCode();
|
||||
if (startupReconcilerActive_ != null) hash ^= StartupReconcilerActive.GetHashCode();
|
||||
if (_unknownFields != null) {
|
||||
hash ^= _unknownFields.GetHashCode();
|
||||
}
|
||||
@@ -4837,6 +4862,9 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
if (localDbOplogBacklog_ != null) {
|
||||
_single_localDbOplogBacklog_codec.WriteTagAndValue(output, LocalDbOplogBacklog);
|
||||
}
|
||||
if (startupReconcilerActive_ != null) {
|
||||
_single_startupReconcilerActive_codec.WriteTagAndValue(output, StartupReconcilerActive);
|
||||
}
|
||||
if (_unknownFields != null) {
|
||||
_unknownFields.WriteTo(output);
|
||||
}
|
||||
@@ -4946,6 +4974,9 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
if (localDbOplogBacklog_ != null) {
|
||||
_single_localDbOplogBacklog_codec.WriteTagAndValue(ref output, LocalDbOplogBacklog);
|
||||
}
|
||||
if (startupReconcilerActive_ != null) {
|
||||
_single_startupReconcilerActive_codec.WriteTagAndValue(ref output, StartupReconcilerActive);
|
||||
}
|
||||
if (_unknownFields != null) {
|
||||
_unknownFields.WriteTo(ref output);
|
||||
}
|
||||
@@ -5034,6 +5065,9 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
if (localDbOplogBacklog_ != null) {
|
||||
size += _single_localDbOplogBacklog_codec.CalculateSizeWithTag(LocalDbOplogBacklog);
|
||||
}
|
||||
if (startupReconcilerActive_ != null) {
|
||||
size += _single_startupReconcilerActive_codec.CalculateSizeWithTag(StartupReconcilerActive);
|
||||
}
|
||||
if (_unknownFields != null) {
|
||||
size += _unknownFields.CalculateSize();
|
||||
}
|
||||
@@ -5147,6 +5181,11 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
LocalDbOplogBacklog = other.LocalDbOplogBacklog;
|
||||
}
|
||||
}
|
||||
if (other.startupReconcilerActive_ != null) {
|
||||
if (startupReconcilerActive_ == null || other.StartupReconcilerActive != false) {
|
||||
StartupReconcilerActive = other.StartupReconcilerActive;
|
||||
}
|
||||
}
|
||||
_unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields);
|
||||
}
|
||||
|
||||
@@ -5305,6 +5344,13 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 234: {
|
||||
bool? value = _single_startupReconcilerActive_codec.Read(input);
|
||||
if (startupReconcilerActive_ == null || value != false) {
|
||||
StartupReconcilerActive = value;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -5463,6 +5509,13 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
|
||||
}
|
||||
break;
|
||||
}
|
||||
case 234: {
|
||||
bool? value = _single_startupReconcilerActive_codec.Read(ref input);
|
||||
if (startupReconcilerActive_ == null || value != false) {
|
||||
StartupReconcilerActive = value;
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -475,6 +475,7 @@ public static class CentralControlDtoMapper
|
||||
ScriptOldestBusyAgeSeconds = msg.ScriptOldestBusyAgeSeconds,
|
||||
LocalDbReplicationConnected = msg.LocalDbReplicationConnected,
|
||||
LocalDbOplogBacklog = msg.LocalDbOplogBacklog,
|
||||
StartupReconcilerActive = msg.StartupReconcilerActive,
|
||||
};
|
||||
|
||||
foreach (var (name, health) in msg.DataConnectionStatuses)
|
||||
@@ -649,6 +650,7 @@ public static class CentralControlDtoMapper
|
||||
ScriptOldestBusyAgeSeconds = dto.ScriptOldestBusyAgeSeconds,
|
||||
LocalDbReplicationConnected = dto.LocalDbReplicationConnected,
|
||||
LocalDbOplogBacklog = dto.LocalDbOplogBacklog,
|
||||
StartupReconcilerActive = dto.StartupReconcilerActive,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -228,6 +228,12 @@ message SiteHealthReportDto {
|
||||
// Absent means UNKNOWN, never zero — a failed backlog read rendered as 0
|
||||
// would report a broken replication pair as perfectly healthy.
|
||||
google.protobuf.Int64Value local_db_oplog_backlog = 28;
|
||||
// Whether this node spawned its SiteReconciliationActor at boot. Absent means
|
||||
// "not reported" — a central self-report (where it is meaningless) or a site
|
||||
// node on a pre-row-73 build. false means the node booted WITHOUT the startup
|
||||
// self-heal path, which is a misconfiguration a healthy-looking site would
|
||||
// otherwise hide entirely.
|
||||
google.protobuf.BoolValue startup_reconciler_active = 29;
|
||||
}
|
||||
|
||||
// Central -> Site: health report ack, so a lost report is observable.
|
||||
|
||||
@@ -162,14 +162,38 @@ public interface ISiteHealthCollector
|
||||
/// service. Point-in-time: values are NOT reset on <see cref="CollectReport"/>.
|
||||
/// </summary>
|
||||
/// <param name="connected">
|
||||
/// Whether a sync session is currently running. <see langword="false"/> is the
|
||||
/// normal state on a node with no peer configured — replication ships default-OFF.
|
||||
/// Whether a sync session is currently running, or <see langword="null"/> when there
|
||||
/// is no reading to give — which is the state of a node with replication UNCONFIGURED
|
||||
/// (register row 80). <see langword="false"/> means "replication is configured on this
|
||||
/// node and the peer session is currently down", i.e. a genuine fault worth a badge;
|
||||
/// reporting that on a deliberately-unreplicated site reads as a fault that isn't one.
|
||||
/// </param>
|
||||
/// <param name="oplogBacklog">
|
||||
/// Unacked oplog entries, or <see langword="null"/> when unknown. Pass null through
|
||||
/// unchanged: a failed poll rendered as 0 would report a broken pair as healthy.
|
||||
/// Unacked oplog entries, or <see langword="null"/> when unknown (failed poll) or not
|
||||
/// applicable (replication unconfigured). Pass null through unchanged: a failed poll
|
||||
/// rendered as 0 would report a broken pair as healthy.
|
||||
/// </param>
|
||||
void SetLocalDbReplicationStatus(bool connected, long? oplogBacklog)
|
||||
void SetLocalDbReplicationStatus(bool? connected, long? oplogBacklog)
|
||||
{
|
||||
// Default no-op so test fakes do not need to be updated.
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Record whether this node spawned its <c>SiteReconciliationActor</c> at boot, for the
|
||||
/// <c>StartupReconcilerActive</c> field of every subsequent <see cref="CollectReport"/>
|
||||
/// (deferred-work register row 73).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Called ONCE at boot by the Host, which owns the spawn decision — this is a latched
|
||||
/// boot fact, not a polled gauge, so there is no reporter hosted service behind it.
|
||||
/// A collector that is never told (central's self-report, or any test host) leaves the
|
||||
/// field null, which reads as "not reported" rather than "reconciler absent".
|
||||
/// </remarks>
|
||||
/// <param name="active">
|
||||
/// <see langword="true"/> when the reconciler was created; <see langword="false"/> when
|
||||
/// it was skipped because no <c>IDeploymentConfigFetcher</c> was available.
|
||||
/// </param>
|
||||
void SetStartupReconcilerActive(bool active)
|
||||
{
|
||||
// Default no-op so test fakes do not need to be updated.
|
||||
}
|
||||
|
||||
@@ -22,6 +22,23 @@ namespace ZB.MOM.WW.ScadaBridge.HealthMonitoring;
|
||||
/// report a replication pair that cannot read its own oplog as perfectly healthy.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>An UNCONFIGURED node reports null for both fields</b> (register row 80). The
|
||||
/// replication engine is registered unconditionally and sits inert without a peer, so
|
||||
/// <c>ISyncStatus</c> answers <c>Connected=false</c> / <c>OplogBacklog=0</c> on a site
|
||||
/// that was never meant to replicate at all. Publishing that pair is actively misleading:
|
||||
/// a dashboard reader sees "connected: false" on a healthy unreplicated site and reads a
|
||||
/// fault, and the central null-branch machinery (conditional KPI emission, sustained-badge
|
||||
/// anchor exclusion) never receives the null it was built for. The
|
||||
/// <c>replicationConfiguredProvider</c> gate therefore short-circuits the probe and pushes
|
||||
/// <c>(null, null)</c> — "no reading", per the documented nullable semantics.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The gate is supplied by the Host, not read from configuration here</b>, mirroring
|
||||
/// <c>SiteEventLogReplicationCheck</c>: the Host owns
|
||||
/// <c>SiteLocalDbSetup.ReplicationIsConfigured</c> (the PeerAddress-OR-ApiKey rule) and
|
||||
/// HealthMonitoring must not learn to read LocalDb configuration itself.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Cadence.</b> 30 s, matching <see cref="SiteEventLogFailureCountReporter"/> and
|
||||
/// <c>SiteAuditBacklogReporter</c>. Any exception during a probe is logged and swallowed;
|
||||
/// the next tick retries.
|
||||
@@ -34,6 +51,7 @@ public sealed class LocalDbReplicationStatusReporter : IHostedService, IDisposab
|
||||
|
||||
private readonly Func<bool> _connectedProvider;
|
||||
private readonly Func<long?> _oplogBacklogProvider;
|
||||
private readonly Func<bool> _replicationConfiguredProvider;
|
||||
private readonly ISiteHealthCollector _collector;
|
||||
private readonly ILogger<LocalDbReplicationStatusReporter> _logger;
|
||||
private readonly TimeSpan _refreshInterval;
|
||||
@@ -43,18 +61,26 @@ public sealed class LocalDbReplicationStatusReporter : IHostedService, IDisposab
|
||||
/// <summary>Initializes a new instance of <see cref="LocalDbReplicationStatusReporter"/>.</summary>
|
||||
/// <param name="connectedProvider">Reads whether a peer sync session is currently running.</param>
|
||||
/// <param name="oplogBacklogProvider">Reads the unacked oplog backlog, or null when unknown.</param>
|
||||
/// <param name="replicationConfiguredProvider">
|
||||
/// Reads whether replication is configured on this node at all (the Host's
|
||||
/// <c>SiteLocalDbSetup.ReplicationIsConfigured</c>). When it answers false, both fields
|
||||
/// are published as null rather than as the engine's inert false/0 defaults.
|
||||
/// </param>
|
||||
/// <param name="collector">The site health collector receiving the snapshot.</param>
|
||||
/// <param name="logger">Logger instance.</param>
|
||||
/// <param name="refreshInterval">Poll interval override; defaults to 30 s.</param>
|
||||
public LocalDbReplicationStatusReporter(
|
||||
Func<bool> connectedProvider,
|
||||
Func<long?> oplogBacklogProvider,
|
||||
Func<bool> replicationConfiguredProvider,
|
||||
ISiteHealthCollector collector,
|
||||
ILogger<LocalDbReplicationStatusReporter> logger,
|
||||
TimeSpan? refreshInterval = null)
|
||||
{
|
||||
_connectedProvider = connectedProvider ?? throw new ArgumentNullException(nameof(connectedProvider));
|
||||
_oplogBacklogProvider = oplogBacklogProvider ?? throw new ArgumentNullException(nameof(oplogBacklogProvider));
|
||||
_replicationConfiguredProvider = replicationConfiguredProvider
|
||||
?? throw new ArgumentNullException(nameof(replicationConfiguredProvider));
|
||||
_collector = collector ?? throw new ArgumentNullException(nameof(collector));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_refreshInterval = refreshInterval ?? DefaultRefreshInterval;
|
||||
@@ -114,6 +140,14 @@ public sealed class LocalDbReplicationStatusReporter : IHostedService, IDisposab
|
||||
{
|
||||
try
|
||||
{
|
||||
if (!_replicationConfiguredProvider())
|
||||
{
|
||||
// Replication unconfigured: "no reading", NOT "disconnected with an empty
|
||||
// backlog". Publishing the inert engine's false/0 here is register row 80.
|
||||
_collector.SetLocalDbReplicationStatus(null, null);
|
||||
return;
|
||||
}
|
||||
|
||||
_collector.SetLocalDbReplicationStatus(
|
||||
_connectedProvider(), _oplogBacklogProvider());
|
||||
}
|
||||
|
||||
@@ -170,19 +170,33 @@ public static class ServiceCollectionExtensions
|
||||
/// a reference on the replication library, matching
|
||||
/// <see cref="AddSiteEventLogHealthMetricsBridge"/>. Pass the backlog through as
|
||||
/// nullable — coercing null to 0 would report an unreadable oplog as a healthy one.
|
||||
/// <para>
|
||||
/// <paramref name="replicationConfiguredProvider"/> is the register-row-80 gate: the
|
||||
/// replication engine is registered unconditionally and answers <c>false</c>/<c>0</c>
|
||||
/// while inert, so without it a deliberately-unreplicated site publishes
|
||||
/// "disconnected, empty backlog" instead of the documented "no reading" null. The
|
||||
/// caller supplies it — the Host owns <c>SiteLocalDbSetup.ReplicationIsConfigured</c>,
|
||||
/// exactly as it supplies <c>SiteEventLogReplicationCheck</c> to SiteEventLogging.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="services">The service collection to register into.</param>
|
||||
/// <param name="connectedProvider">Given the root provider, returns a reader for "a sync session is running".</param>
|
||||
/// <param name="oplogBacklogProvider">Given the root provider, returns a reader for the unacked backlog (null = unknown).</param>
|
||||
/// <param name="replicationConfiguredProvider">
|
||||
/// Given the root provider, returns a reader for "replication is configured on this node".
|
||||
/// False makes the reporter publish null for BOTH fields.
|
||||
/// </param>
|
||||
/// <returns>The same <see cref="IServiceCollection"/> for chaining.</returns>
|
||||
public static IServiceCollection AddLocalDbReplicationHealthBridge(
|
||||
this IServiceCollection services,
|
||||
Func<IServiceProvider, Func<bool>> connectedProvider,
|
||||
Func<IServiceProvider, Func<long?>> oplogBacklogProvider)
|
||||
Func<IServiceProvider, Func<long?>> oplogBacklogProvider,
|
||||
Func<IServiceProvider, Func<bool>> replicationConfiguredProvider)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(connectedProvider);
|
||||
ArgumentNullException.ThrowIfNull(oplogBacklogProvider);
|
||||
ArgumentNullException.ThrowIfNull(replicationConfiguredProvider);
|
||||
|
||||
if (services.Any(d => d.ServiceType == typeof(LocalDbReplicationHealthBridgeMarker)))
|
||||
{
|
||||
@@ -193,6 +207,7 @@ public static class ServiceCollectionExtensions
|
||||
services.AddHostedService(sp => new LocalDbReplicationStatusReporter(
|
||||
connectedProvider(sp),
|
||||
oplogBacklogProvider(sp),
|
||||
replicationConfiguredProvider(sp),
|
||||
sp.GetRequiredService<ISiteHealthCollector>(),
|
||||
sp.GetRequiredService<ILogger<LocalDbReplicationStatusReporter>>()));
|
||||
|
||||
|
||||
@@ -20,8 +20,18 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
private long _siteEventLogWriteFailures;
|
||||
// One volatile tuple rather than two independent fields: a torn read that paired a
|
||||
// fresh Connected with a stale backlog would be indistinguishable from a real state.
|
||||
// Null = the reporter has not yet run (or replication is not wired).
|
||||
private volatile Tuple<bool, long?>? _localDbReplicationStatus;
|
||||
// Null = the reporter has not yet run. Once it HAS run, an unreplicated node latches
|
||||
// (null, null) — "no reading" — rather than the inert engine's (false, 0); see
|
||||
// LocalDbReplicationStatusReporter and register row 80.
|
||||
private volatile Tuple<bool?, long?>? _localDbReplicationStatus;
|
||||
// Latched once at boot by the Host (register row 73). A tri-state int rather than a
|
||||
// bool? because `volatile` cannot be applied to Nullable<bool> (a struct) and this is
|
||||
// written on the startup thread but read on the report-sender thread:
|
||||
// 0 = never told -> report null ("not reported": central self-report, or a test host)
|
||||
// 1 = reconciler spawned
|
||||
// -1 = reconciler SKIPPED (no IDeploymentConfigFetcher)
|
||||
// Null must stay distinct from -1: "we don't know" is not "self-heal is missing".
|
||||
private volatile int _startupReconcilerActive;
|
||||
private readonly ConcurrentDictionary<string, ConnectionHealth> _connectionStatuses = new();
|
||||
private readonly ConcurrentDictionary<string, TagResolutionStatus> _tagResolutionCounts = new();
|
||||
private readonly ConcurrentDictionary<string, string> _connectionEndpoints = new();
|
||||
@@ -109,11 +119,17 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetLocalDbReplicationStatus(bool connected, long? oplogBacklog)
|
||||
public void SetLocalDbReplicationStatus(bool? connected, long? oplogBacklog)
|
||||
{
|
||||
_localDbReplicationStatus = Tuple.Create(connected, oplogBacklog);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SetStartupReconcilerActive(bool active)
|
||||
{
|
||||
_startupReconcilerActive = active ? 1 : -1;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void UpdateConnectionHealth(string connectionName, ConnectionHealth health)
|
||||
{
|
||||
@@ -256,6 +272,7 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
// Single read of the volatile tuple — two reads could straddle a reporter tick and
|
||||
// pair a fresh Connected with a stale backlog.
|
||||
var localDbReplication = _localDbReplicationStatus;
|
||||
var startupReconciler = _startupReconcilerActive;
|
||||
|
||||
// Snapshot current connection and tag resolution state
|
||||
var connectionStatuses = new Dictionary<string, ConnectionHealth>(_connectionStatuses);
|
||||
@@ -307,7 +324,10 @@ public class SiteHealthCollector : ISiteHealthCollector
|
||||
// with an empty backlog".
|
||||
LocalDbReplicationConnected = localDbReplication?.Item1,
|
||||
LocalDbOplogBacklog = localDbReplication?.Item2,
|
||||
SiteStreamAlarmDropCount = Interlocked.Read(ref _siteStreamAlarmDropCount)
|
||||
SiteStreamAlarmDropCount = Interlocked.Read(ref _siteStreamAlarmDropCount),
|
||||
// Tri-state: 0 stays null ("not reported"), which must never collapse into
|
||||
// false ("self-heal is missing"). Register row 73.
|
||||
StartupReconcilerActive = startupReconciler == 0 ? null : startupReconciler > 0
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1029,6 +1029,14 @@ akka {{
|
||||
// singleton, which starts the Instance Actor if the boot-time load missed it. Going
|
||||
// through the PROXY (not a local ActorOf) keeps Instance Actors under the singleton
|
||||
// regardless of which node's reconcile pass found the gap.
|
||||
//
|
||||
// Whichever branch is taken is LATCHED onto the site health report as
|
||||
// StartupReconcilerActive (deferred-work register row 73). Before that, the skip
|
||||
// was a single boot Warning and nothing else: a node that silently lost its entire
|
||||
// self-heal path went on reporting itself perfectly healthy forever, and the only
|
||||
// evidence was one line scrolled off the top of a log. Central badges the false
|
||||
// case at Warning rather than failing the site — replication is still the primary
|
||||
// path, so the node is degraded, not down.
|
||||
if (deploymentConfigFetcher != null)
|
||||
{
|
||||
var reconcileLogger = _serviceProvider.GetRequiredService<ILoggerFactory>()
|
||||
@@ -1045,15 +1053,50 @@ akka {{
|
||||
null,
|
||||
dmProxy)),
|
||||
"site-reconciliation");
|
||||
siteHealthCollector?.SetStartupReconcilerActive(true);
|
||||
_logger.LogInformation(
|
||||
"SiteReconciliationActor created (per-node startup self-heal) for site {SiteId} node {Node}",
|
||||
_nodeOptions.SiteId, _nodeOptions.NodeName);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"No IDeploymentConfigFetcher available; SiteReconciliationActor not created — "
|
||||
+ "startup self-heal disabled (replication remains the primary path)");
|
||||
siteHealthCollector?.SetStartupReconcilerActive(false);
|
||||
|
||||
// Raised from Warning to Error (register row 73). This is a misconfiguration,
|
||||
// not a degraded-but-expected state: no configuration makes the fetcher
|
||||
// legitimately absent on a site node, and the consequence is that neither the
|
||||
// file heal nor the row-67 runtime heal will ever run on this node.
|
||||
_logger.LogError(
|
||||
"No IDeploymentConfigFetcher available; SiteReconciliationActor not created for "
|
||||
+ "site {SiteId} node {Node} — startup self-heal is DISABLED (replication "
|
||||
+ "remains the primary path). Surfaced centrally as StartupReconcilerActive=false.",
|
||||
_nodeOptions.SiteId, _nodeOptions.NodeName);
|
||||
|
||||
// Also record it where an operator reviewing the site after the fact will find
|
||||
// it — the boot log line is gone by then, but the site event log is queryable
|
||||
// from the Central UI. Best-effort: a failed event write must never keep the
|
||||
// node from finishing startup, which is why this is fire-and-forget with an
|
||||
// observed continuation rather than an await.
|
||||
var bootEventLogger = _serviceProvider
|
||||
.GetService<ZB.MOM.WW.ScadaBridge.SiteEventLogging.ISiteEventLogger>();
|
||||
_ = bootEventLogger?.LogEventAsync(
|
||||
"deployment",
|
||||
"Error",
|
||||
null,
|
||||
"AkkaHostedService",
|
||||
"Startup self-heal disabled: SiteReconciliationActor was not created "
|
||||
+ "because no IDeploymentConfigFetcher is registered.",
|
||||
$"Node {_nodeOptions.NodeName} of site {_nodeOptions.SiteId} cannot "
|
||||
+ "reconcile its deployed inventory at boot. Config deploys still reach "
|
||||
+ "this node via LocalDb replication when it is up; a deploy that lands "
|
||||
+ "while it is DOWN will not be recovered on restart.")
|
||||
.ContinueWith(
|
||||
t => _logger.LogWarning(
|
||||
t.Exception,
|
||||
"Failed to write the startup-self-heal-disabled site event."),
|
||||
CancellationToken.None,
|
||||
TaskContinuationOptions.OnlyOnFaulted,
|
||||
TaskScheduler.Default);
|
||||
}
|
||||
|
||||
// Audit Log — site-side telemetry actor that drains the SQLite
|
||||
|
||||
@@ -85,12 +85,91 @@ public static class LoggerConfigurationFactory
|
||||
return new LoggerConfiguration()
|
||||
.ReadFrom.Configuration(configuration)
|
||||
.MinimumLevel.Is(minimumLevel)
|
||||
// Register row 71. Drop ONLY the framework's Error line for the by-design
|
||||
// standby verdict of the active-node probe. See IsExpectedStandbyActiveNodeProbe
|
||||
// for why this cannot hide a genuine failure.
|
||||
.Filter.ByExcluding(IsExpectedStandbyActiveNodeProbe)
|
||||
.Enrich.WithProperty("SiteId", siteId)
|
||||
.Enrich.WithProperty("NodeHostname", nodeHostname)
|
||||
.Enrich.WithProperty("NodeRole", nodeRole)
|
||||
.Enrich.With(new TraceContextEnricher());
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Logger category that emits the per-probe "Health check {HealthCheckName} with status
|
||||
/// {HealthStatus} …" line. It is <c>DefaultHealthCheckService</c>, not
|
||||
/// <c>HealthCheckPublisherHostedService</c>: this host registers no
|
||||
/// <c>IHealthCheckPublisher</c>, so the checks run only on an inbound HTTP probe of
|
||||
/// <c>/health/*</c>.
|
||||
/// </summary>
|
||||
internal const string HealthCheckServiceSourceContext =
|
||||
"Microsoft.Extensions.Diagnostics.HealthChecks.DefaultHealthCheckService";
|
||||
|
||||
/// <summary>Name both the site and central <c>/health/active</c> probes register under.</summary>
|
||||
internal const string ActiveNodeCheckName = "active-node";
|
||||
|
||||
/// <summary>
|
||||
/// Suppresses the standby node's correct-by-design <c>active-node</c> Error line
|
||||
/// (deferred-work register row 71).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>The problem.</b> <c>DefaultHealthCheckService</c> logs one line per check per probe,
|
||||
/// picking the level from the verdict — Debug for Healthy, Warning for Degraded,
|
||||
/// <b>Error for Unhealthy</b>. Both active-node checks
|
||||
/// (<c>SitePairActiveNodeHealthCheck</c> on a site, the shared <c>ActiveNodeHealthCheck</c>
|
||||
/// on central) answer Unhealthy on the standby, because 503 is exactly the contract
|
||||
/// <c>/health/active</c> publishes to Traefik and the family dashboard. So a perfectly
|
||||
/// healthy standby logged ERR on every probe — every 10 s on a site node, every 5 s on a
|
||||
/// central node behind Traefik's 5 s active-node probe. That floods the standby's log and
|
||||
/// trains operators to ignore ERR.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Why a filter and not a level override.</b> Serilog's <c>MinimumLevel.Override</c>
|
||||
/// can only RAISE a per-source floor; it cannot rewrite one Error event as Information.
|
||||
/// A <c>LogEvent</c>'s level is immutable, so an enricher cannot do it either. Excluding
|
||||
/// the single by-design event is the surgical option, and it leaves the verdict fully
|
||||
/// observable where it belongs: the HTTP status of <c>/health/active</c> itself, the
|
||||
/// cluster view on the readiness probe, and the health dashboard.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Why this cannot hide a genuine failure.</b> The predicate is keyed on the check
|
||||
/// NAME, not on the category, so a failing <c>localdb</c>, <c>akka-cluster</c>,
|
||||
/// <c>database</c>, or <c>required-singletons</c> check still logs at Error untouched.
|
||||
/// And the active-node check is a pure "am I the oldest Up member" verdict with no error
|
||||
/// path of its own — the ACTIVE node always answers Healthy, so the only event this can
|
||||
/// ever match is a standby reporting that it is a standby. Genuine loss of the pair
|
||||
/// surfaces on the readiness checks, which are deliberately out of scope here.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// Matching on <see cref="LogEventLevel.Error"/> rather than on the <c>HealthStatus</c>
|
||||
/// property avoids depending on how the enum renders into the event; the level→verdict
|
||||
/// mapping above already makes Error equivalent to Unhealthy.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
/// <param name="logEvent">The candidate log event.</param>
|
||||
/// <returns><see langword="true"/> when the event should be dropped.</returns>
|
||||
internal static bool IsExpectedStandbyActiveNodeProbe(LogEvent logEvent)
|
||||
{
|
||||
if (logEvent is null || logEvent.Level != LogEventLevel.Error)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return MatchesScalar(logEvent, "SourceContext", HealthCheckServiceSourceContext)
|
||||
&& MatchesScalar(logEvent, "HealthCheckName", ActiveNodeCheckName);
|
||||
}
|
||||
|
||||
/// <summary>Tests one structured property for an exact scalar string value.</summary>
|
||||
/// <param name="logEvent">The event whose properties are inspected.</param>
|
||||
/// <param name="propertyName">The structured property name.</param>
|
||||
/// <param name="expected">The value the property must equal.</param>
|
||||
/// <returns><see langword="true"/> on an exact ordinal match.</returns>
|
||||
private static bool MatchesScalar(LogEvent logEvent, string propertyName, string expected) =>
|
||||
logEvent.Properties.TryGetValue(propertyName, out var value)
|
||||
&& value is ScalarValue { Value: string actual }
|
||||
&& string.Equals(actual, expected, StringComparison.Ordinal);
|
||||
|
||||
/// <summary>
|
||||
/// Parses a Serilog <see cref="LogEventLevel"/> name, falling back to
|
||||
/// <see cref="LogEventLevel.Information"/> for null/blank/unrecognised values.
|
||||
|
||||
@@ -137,14 +137,27 @@ public static class SiteServiceRegistration
|
||||
sp => () => sp.GetRequiredService<ISiteEventLogger>().FailedWriteCount);
|
||||
|
||||
// LocalDb replication — bridge ISyncStatus onto the site health report. Registered
|
||||
// unconditionally alongside the engine: on a node with no peer this reports
|
||||
// Connected=false with a real 0 backlog, which is the healthy default-OFF state.
|
||||
// OplogBacklog is passed through NULLABLE on purpose — null means the poll failed
|
||||
// or no provider is wired, and flattening it to 0 would render a replication pair
|
||||
// that cannot read its own oplog as perfectly healthy.
|
||||
// unconditionally alongside the engine, but what it PUBLISHES is gated on whether
|
||||
// this node is configured to replicate at all (register row 80).
|
||||
//
|
||||
// The engine is inert without a peer, so ISyncStatus answers Connected=false with a
|
||||
// real 0 backlog on a site that was never meant to replicate. Publishing that pair
|
||||
// contradicts the documented nullable semantics ("null means no reading, NOT
|
||||
// disconnected with an empty backlog"): a dashboard reader sees connected:false on a
|
||||
// healthy unreplicated site and reads a fault, and central's null-branch machinery
|
||||
// (conditional KPI emission, sustained-badge anchor exclusion) never receives the
|
||||
// null it was built for — so rig site-b/site-c emitted a misleading
|
||||
// localDbReplicationConnected=0 series. Unconfigured now publishes null/null.
|
||||
//
|
||||
// Delegating to SiteLocalDbSetup.ReplicationIsConfigured keeps the
|
||||
// PeerAddress-OR-ApiKey rule in ONE place and mirrors SiteEventLogReplicationCheck
|
||||
// below — HealthMonitoring must not learn to read LocalDb configuration itself.
|
||||
// Read once here (configuration is fixed for the process lifetime) and captured.
|
||||
var localDbReplicationConfigured = SiteLocalDbSetup.ReplicationIsConfigured(config);
|
||||
services.AddLocalDbReplicationHealthBridge(
|
||||
sp => () => sp.GetRequiredService<ISyncStatus>().Connected,
|
||||
sp => () => sp.GetRequiredService<ISyncStatus>().OplogBacklog);
|
||||
sp => () => sp.GetRequiredService<ISyncStatus>().OplogBacklog,
|
||||
_ => () => localDbReplicationConfigured);
|
||||
|
||||
// Audit Log — site-side hot-path writer + telemetry collaborators.
|
||||
// The SiteAuditTelemetryActor itself is registered by AkkaHostedService
|
||||
@@ -217,9 +230,9 @@ public static class SiteServiceRegistration
|
||||
// backlog genuinely rises at purge time and reads like a replication fault to anyone who
|
||||
// does not know a purge just ran. Delegating to SiteLocalDbSetup.ReplicationIsConfigured
|
||||
// keeps the PeerAddress-OR-ApiKey rule in one place — SiteEventLogging must not learn to
|
||||
// read LocalDb configuration itself.
|
||||
var replicationConfigured = SiteLocalDbSetup.ReplicationIsConfigured(config);
|
||||
SiteEventLogReplicationCheck replicationCheck = () => replicationConfigured;
|
||||
// read LocalDb configuration itself. Shares the single read taken for the LocalDb
|
||||
// replication health bridge above, so the two surfaces can never disagree.
|
||||
SiteEventLogReplicationCheck replicationCheck = () => localDbReplicationConfigured;
|
||||
services.AddSingleton(replicationCheck);
|
||||
|
||||
// Health checks — the shared ZB.MOM.WW.Health probes, mapped by MapZbHealth on the site's
|
||||
|
||||
Reference in New Issue
Block a user