dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

perf(comms+audit): close phase-2 residuals — direct ingest path, monotonic timeouts, synthetic probe, not-reporting set, cursor-exact audit pull

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-14 21:38:23 -04:00
parent 4cd1441984
commit a5882753dd
38 changed files with 1254 additions and 443 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.Communication/Actors/CentralCommunicationActor.cs
+36 -113
View File
@@ -67,36 +67,6 @@ public class CentralCommunicationActor : ReceiveActor
/// </summary>
private IActorRef? _notificationOutboxProxy;
/// <summary>
/// Proxy <see cref="IActorRef"/> for the central AuditLogIngestActor cluster
/// singleton. Set via <see cref="RegisterAuditIngest"/> — the Host creates the
/// singleton proxy after this actor and registers it (mirrors
/// <see cref="_notificationOutboxProxy"/>). Null until registration completes;
/// an audit ingest command arriving before then is answered with an empty
/// reply so the site keeps its rows Pending and retries.
///
/// Once registered, the handler Asks this proxy and pipes the reply straight
/// back to the caller. On an Ask timeout or a faulted reply, PipeTo forwards a
/// <see cref="Status.Failure"/> to the caller — the fault propagates rather
/// than being swallowed. This differs from the gRPC handler
/// (<c>SiteStreamGrpcServer</c>), which catches the exception and returns an
/// empty ack; here the faulted Ask is the transient signal the site relies on
/// (see <see cref="HandleIngestAuditEvents"/>).
/// </summary>
private IActorRef? _auditIngestProxy;
/// <summary>
/// Default Ask timeout for routing audit ingest commands to the
/// Effective Ask timeout for audit ingest routing. Defaults to
/// <see cref="Grpc.SiteStreamGrpcServer.AuditIngestAskTimeout"/> (30 s) — the two
/// audit-ingest entry points (the site stream server and the control plane) share one source of truth
/// for the timeout. Overridable via the constructor so tests can exercise the
/// timeout/fault path without waiting 30 s. When the window is exceeded the Ask
/// faults and that fault is piped back to the caller as a
/// <see cref="Status.Failure"/> (see <see cref="HandleIngestAuditEvents"/>).
/// </summary>
private readonly TimeSpan _auditIngestAskTimeout;
/// <summary>
/// DistributedPubSub topic used to fan health reports out to the peer
/// central node so both per-node aggregators stay in sync. See
@@ -111,12 +81,10 @@ public class CentralCommunicationActor : ReceiveActor
/// </summary>
/// <param name="serviceProvider">DI service provider for scoped repository and aggregator access.</param>
/// <param name="transport">The central→site command transport to route every <see cref="SiteEnvelope"/> through.</param>
/// <param name="auditIngestAskTimeout">Optional override for the audit-ingest Ask timeout (test hook).</param>
public CentralCommunicationActor(
IServiceProvider serviceProvider,
ISiteCommandTransport transport,
TimeSpan? auditIngestAskTimeout = null)
: this(serviceProvider, auditIngestAskTimeout)
ISiteCommandTransport transport)
: this(serviceProvider)
{
_transport = transport ?? throw new ArgumentNullException(nameof(transport));
}
@@ -125,13 +93,9 @@ public class CentralCommunicationActor : ReceiveActor
/// <see cref="_transport"/> is assigned by the delegating public constructor before any message
/// is dispatched.</summary>
/// <param name="serviceProvider">DI service provider.</param>
/// <param name="auditIngestAskTimeout">Optional audit-ingest Ask timeout override.</param>
private CentralCommunicationActor(
IServiceProvider serviceProvider,
TimeSpan? auditIngestAskTimeout)
private CentralCommunicationActor(IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
_auditIngestAskTimeout = auditIngestAskTimeout ?? Grpc.SiteStreamGrpcServer.AuditIngestAskTimeout;
// Site address cache loaded from database
Receive<SiteAddressCacheLoaded>(HandleSiteAddressCacheLoaded);
@@ -176,24 +140,11 @@ public class CentralCommunicationActor : ReceiveActor
// so the NotificationStatusResponse routes back to the querying site.
Receive<NotificationStatusQuery>(HandleNotificationStatusQuery);
// Audit Log: the Host registers the AuditLogIngestActor singleton
// proxy after this actor is created (the proxy cannot exist before this
// actor's construction).
Receive<RegisterAuditIngest>(msg =>
{
_auditIngestProxy = msg.AuditIngestActor;
_log.Info("Registered audit ingest proxy");
});
// Audit Log site→central ingest: a site forwards a batch of audit
// events to the central cluster. Ask the ingest proxy
// and pipe the IngestAuditEventsReply back to the original Sender (the
// site's transport path) so the site can flip its rows to Forwarded.
Receive<IngestAuditEventsCommand>(HandleIngestAuditEvents);
// Audit Log combined-telemetry ingest: routes to the same proxy
// the same way; the proxy replies with an IngestCachedTelemetryReply.
Receive<IngestCachedTelemetryCommand>(HandleIngestCachedTelemetry);
// Audit Log site→central ingest is NOT relayed here. The central-hosted
// CentralControlGrpcService Asks the audit-log-ingest singleton proxy directly
// (as SiteStreamGrpcServer always has), so this actor no longer sits in that
// path — the relay was a second hop with an identical 30 s Ask timeout that
// could only add latency. Do not reintroduce it.
// Startup reconciliation: a site node forwards its local deployed inventory on
// startup. Resolve the scoped ReconcileService, diff the
@@ -239,51 +190,6 @@ public class CentralCommunicationActor : ReceiveActor
_notificationOutboxProxy.Forward(msg);
}
private void HandleIngestAuditEvents(IngestAuditEventsCommand msg)
{
if (_auditIngestProxy == null)
{
// No ingest proxy registered yet (host startup race). Reply with an
// empty IngestAuditEventsReply so the site keeps its rows Pending and
// retries — the same behaviour as the gRPC handler's wiring-race path.
_log.Warning(
"Cannot route IngestAuditEventsCommand ({0} events) — audit ingest not available",
msg.Events.Count);
Sender.Tell(new IngestAuditEventsReply(Array.Empty<Guid>()));
return;
}
// Capture Sender before the async/PipeTo — Akka resets Sender between
// dispatches. The reply is piped straight back to the calling site node.
// On an Ask timeout or a faulted reply, PipeTo delivers a Status.Failure to
// replyTo: the fault propagates to the caller rather than being swallowed.
// The site's own Ask through this path then faults, and the site drain loop
// treats that as a transient failure — rows stay Pending and are retried on
// the next tick. (The gRPC handler instead returns an empty ack on fault;
// propagating the fault here is the cleaner transient signal.)
var replyTo = Sender;
_log.Debug("Routing IngestAuditEventsCommand ({0} events) to the audit ingest actor", msg.Events.Count);
_auditIngestProxy.Ask<IngestAuditEventsReply>(msg, _auditIngestAskTimeout)
.PipeTo(replyTo);
}
private void HandleIngestCachedTelemetry(IngestCachedTelemetryCommand msg)
{
if (_auditIngestProxy == null)
{
_log.Warning(
"Cannot route IngestCachedTelemetryCommand ({0} entries) — audit ingest not available",
msg.Entries.Count);
Sender.Tell(new IngestCachedTelemetryReply(Array.Empty<Guid>()));
return;
}
var replyTo = Sender;
_log.Debug("Routing IngestCachedTelemetryCommand ({0} entries) to the audit ingest actor", msg.Entries.Count);
_auditIngestProxy.Ask<IngestCachedTelemetryReply>(msg, _auditIngestAskTimeout)
.PipeTo(replyTo);
}
/// <summary>
/// Startup reconciliation (site→central): resolve the scoped
/// <see cref="ReconcileService"/> in a DI scope, diff the node's reported inventory
@@ -291,7 +197,7 @@ public class CentralCommunicationActor : ReceiveActor
/// back to the site node's transport path. The actor stays thin — all the diff
/// and staging logic lives in the service. Mirrors the DB-access pattern used by
/// <see cref="LoadSiteAddressesFromDb"/> (Task.Run + CreateScope + PipeTo) and the
/// Sender-preservation pattern of <see cref="HandleIngestAuditEvents"/>.
/// Sender-preservation pattern of <see cref="HandleNotificationSubmit"/>.
///
/// On a faulted task PipeTo delivers a <see cref="Status.Failure"/> to the node; its
/// Ask faults and it simply retries reconcile on the next startup — reconcile is
@@ -328,6 +234,21 @@ public class CentralCommunicationActor : ReceiveActor
private void HandleHeartbeat(HeartbeatMessage heartbeat)
{
// Synthetic heartbeats carry no liveness meaning — today only
// CentralChannelProvider's failback probe, which reuses this RPC to ask whether the
// preferred central endpoint answers again. It is emitted by a site's TRANSPORT layer,
// not by any node's heartbeat timer, so stamping it would let a site whose real
// heartbeats had stopped keep looking alive on the health dashboard for as long as its
// transport kept probing. Dropped before the local mark AND before the peer fan-out,
// so neither central node's aggregator ever sees it.
if (heartbeat.Synthetic)
{
_log.Debug(
"Ignoring synthetic heartbeat from site {0} (node '{1}') — probe traffic, not liveness",
heartbeat.SiteId, heartbeat.NodeHostname);
return;
}
MarkHeartbeatLocally(heartbeat);
// Fan the heartbeat out to the peer central node so BOTH aggregators mark
@@ -350,9 +271,17 @@ public class CentralCommunicationActor : ReceiveActor
/// <summary>
/// Marks a site heartbeat on the local aggregator without re-broadcasting.
/// Used for both site-originated heartbeats and peer-replicated ones.
/// A synthetic heartbeat is never marked — <see cref="HandleHeartbeat"/> already drops it
/// before the fan-out, so a replica can only carry one if a peer node predates the flag;
/// the guard here is belt-and-braces on the last hop before the aggregator.
/// </summary>
private void MarkHeartbeatLocally(HeartbeatMessage heartbeat)
{
if (heartbeat.Synthetic)
{
return;
}
var aggregator = _serviceProvider.GetService<ICentralHealthAggregator>();
aggregator?.MarkHeartbeat(heartbeat.SiteId, heartbeat.Timestamp);
}
@@ -623,13 +552,7 @@ public record DebugStreamTerminated(string SiteId, string CorrelationId);
/// </summary>
public record RegisterNotificationOutbox(IActorRef OutboxProxy);
/// <summary>
/// Registers the central AuditLogIngestActor singleton proxy with the
/// <see cref="CentralCommunicationActor"/> so site-forwarded
/// <see cref="IngestAuditEventsCommand"/> and <see cref="IngestCachedTelemetryCommand"/>
/// messages can be routed to it. Sent by the Host after the audit-ingest
/// singleton proxy is created. Lives here (not in Commons) because
/// <c>ZB.MOM.WW.ScadaBridge.Commons</c> has no Akka package reference and cannot hold an
/// <see cref="IActorRef"/> field.
/// </summary>
public sealed record RegisterAuditIngest(IActorRef AuditIngestActor);
// NOTE: there is deliberately no RegisterAuditIngest counterpart. The audit-log-ingest
// singleton proxy goes STRAIGHT to the two gRPC servers that need it
// (CentralControlGrpcService.SetAuditIngestActor / SiteStreamGrpcServer.SetAuditIngestActor);
// routing audit batches through this actor was a redundant hop and was removed.
src/ZB.MOM.WW.ScadaBridge.Communication/CentralControlGrpc/CentralControl.cs
+75 -28
View File
@@ -114,33 +114,34 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
"c0R0bzoCOAEaQgogU3RvcmVBbmRGb3J3YXJkQnVmZmVyRGVwdGhzRW50cnkS",
"CwoDa2V5GAEgASgJEg0KBXZhbHVlGAIgASgFOgI4ASJjChZTaXRlSGVhbHRo",
"UmVwb3J0QWNrRHRvEg8KB3NpdGVfaWQYASABKAkSFwoPc2VxdWVuY2VfbnVt",
"YmVyGAIgASgDEhAKCGFjY2VwdGVkGAMgASgIEg0KBWVycm9yGAQgASgJIngK",
"DEhlYXJ0YmVhdER0bxIPCgdzaXRlX2lkGAEgASgJEhUKDW5vZGVfaG9zdG5h",
"bWUYAiABKAkSEQoJaXNfYWN0aXZlGAMgASgIEi0KCXRpbWVzdGFtcBgEIAEo",
"CzIaLmdvb2dsZS5wcm90b2J1Zi5UaW1lc3RhbXAqvQEKFENvbm5lY3Rpb25I",
"ZWFsdGhFbnVtEiEKHUNPTk5FQ1RJT05fSEVBTFRIX1VOU1BFQ0lGSUVEEAAS",
"HwobQ09OTkVDVElPTl9IRUFMVEhfQ09OTkVDVEVEEAESIgoeQ09OTkVDVElP",
"Tl9IRUFMVEhfRElTQ09OTkVDVEVEEAISIAocQ09OTkVDVElPTl9IRUFMVEhf",
"Q09OTkVDVElORxADEhsKF0NPTk5FQ1RJT05fSEVBTFRIX0VSUk9SEAQyoQYK",
"FUNlbnRyYWxDb250cm9sU2VydmljZRKDAQoSU3VibWl0Tm90aWZpY2F0aW9u",
"EjQuc2NhZGFicmlkZ2UuY2VudHJhbGNvbnRyb2wudjEuTm90aWZpY2F0aW9u",
"U3VibWl0RHRvGjcuc2NhZGFicmlkZ2UuY2VudHJhbGNvbnRyb2wudjEuTm90",
"aWZpY2F0aW9uU3VibWl0QWNrRHRvEpIBChdRdWVyeU5vdGlmaWNhdGlvblN0",
"YXR1cxI5LnNjYWRhYnJpZGdlLmNlbnRyYWxjb250cm9sLnYxLk5vdGlmaWNh",
"dGlvblN0YXR1c1F1ZXJ5RHRvGjwuc2NhZGFicmlkZ2UuY2VudHJhbGNvbnRy",
"b2wudjEuTm90aWZpY2F0aW9uU3RhdHVzUmVzcG9uc2VEdG8SRwoRSW5nZXN0",
"QXVkaXRFdmVudHMSGy5zaXRlc3RyZWFtLkF1ZGl0RXZlbnRCYXRjaBoVLnNp",
"dGVzdHJlYW0uSW5nZXN0QWNrElAKFUluZ2VzdENhY2hlZFRlbGVtZXRyeRIg",
"LnNpdGVzdHJlYW0uQ2FjaGVkVGVsZW1ldHJ5QmF0Y2gaFS5zaXRlc3RyZWFt",
"LkluZ2VzdEFjaxKAAQoNUmVjb25jaWxlU2l0ZRI2LnNjYWRhYnJpZGdlLmNl",
"bnRyYWxjb250cm9sLnYxLlJlY29uY2lsZVNpdGVSZXF1ZXN0RHRvGjcuc2Nh",
"ZGFicmlkZ2UuY2VudHJhbGNvbnRyb2wudjEuUmVjb25jaWxlU2l0ZVJlc3Bv",
"bnNlRHRvEn0KEFJlcG9ydFNpdGVIZWFsdGgSMi5zY2FkYWJyaWRnZS5jZW50",
"cmFsY29udHJvbC52MS5TaXRlSGVhbHRoUmVwb3J0RHRvGjUuc2NhZGFicmlk",
"Z2UuY2VudHJhbGNvbnRyb2wudjEuU2l0ZUhlYWx0aFJlcG9ydEFja0R0bxJQ",
"CglIZWFydGJlYXQSKy5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5I",
"ZWFydGJlYXREdG8aFi5nb29nbGUucHJvdG9idWYuRW1wdHlCK6oCKFpCLk1P",
"TS5XVy5TY2FkYUJyaWRnZS5Db21tdW5pY2F0aW9uLkdycGNiBnByb3RvMw=="));
"YmVyGAIgASgDEhAKCGFjY2VwdGVkGAMgASgIEg0KBWVycm9yGAQgASgJIosB",
"CgxIZWFydGJlYXREdG8SDwoHc2l0ZV9pZBgBIAEoCRIVCg1ub2RlX2hvc3Ru",
"YW1lGAIgASgJEhEKCWlzX2FjdGl2ZRgDIAEoCBItCgl0aW1lc3RhbXAYBCAB",
"KAsyGi5nb29nbGUucHJvdG9idWYuVGltZXN0YW1wEhEKCXN5bnRoZXRpYxgF",
"IAEoCCq9AQoUQ29ubmVjdGlvbkhlYWx0aEVudW0SIQodQ09OTkVDVElPTl9I",
"RUFMVEhfVU5TUEVDSUZJRUQQABIfChtDT05ORUNUSU9OX0hFQUxUSF9DT05O",
"RUNURUQQARIiCh5DT05ORUNUSU9OX0hFQUxUSF9ESVNDT05ORUNURUQQAhIg",
"ChxDT05ORUNUSU9OX0hFQUxUSF9DT05ORUNUSU5HEAMSGwoXQ09OTkVDVElP",
"Tl9IRUFMVEhfRVJST1IQBDKhBgoVQ2VudHJhbENvbnRyb2xTZXJ2aWNlEoMB",
"ChJTdWJtaXROb3RpZmljYXRpb24SNC5zY2FkYWJyaWRnZS5jZW50cmFsY29u",
"dHJvbC52MS5Ob3RpZmljYXRpb25TdWJtaXREdG8aNy5zY2FkYWJyaWRnZS5j",
"ZW50cmFsY29udHJvbC52MS5Ob3RpZmljYXRpb25TdWJtaXRBY2tEdG8SkgEK",
"F1F1ZXJ5Tm90aWZpY2F0aW9uU3RhdHVzEjkuc2NhZGFicmlkZ2UuY2VudHJh",
"bGNvbnRyb2wudjEuTm90aWZpY2F0aW9uU3RhdHVzUXVlcnlEdG8aPC5zY2Fk",
"YWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5Ob3RpZmljYXRpb25TdGF0dXNS",
"ZXNwb25zZUR0bxJHChFJbmdlc3RBdWRpdEV2ZW50cxIbLnNpdGVzdHJlYW0u",
"QXVkaXRFdmVudEJhdGNoGhUuc2l0ZXN0cmVhbS5Jbmdlc3RBY2sSUAoVSW5n",
"ZXN0Q2FjaGVkVGVsZW1ldHJ5EiAuc2l0ZXN0cmVhbS5DYWNoZWRUZWxlbWV0",
"cnlCYXRjaBoVLnNpdGVzdHJlYW0uSW5nZXN0QWNrEoABCg1SZWNvbmNpbGVT",
"aXRlEjYuc2NhZGFicmlkZ2UuY2VudHJhbGNvbnRyb2wudjEuUmVjb25jaWxl",
"U2l0ZVJlcXVlc3REdG8aNy5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52",
"MS5SZWNvbmNpbGVTaXRlUmVzcG9uc2VEdG8SfQoQUmVwb3J0U2l0ZUhlYWx0",
"aBIyLnNjYWRhYnJpZGdlLmNlbnRyYWxjb250cm9sLnYxLlNpdGVIZWFsdGhS",
"ZXBvcnREdG8aNS5zY2FkYWJyaWRnZS5jZW50cmFsY29udHJvbC52MS5TaXRl",
"SGVhbHRoUmVwb3J0QWNrRHRvElAKCUhlYXJ0YmVhdBIrLnNjYWRhYnJpZGdl",
"LmNlbnRyYWxjb250cm9sLnYxLkhlYXJ0YmVhdER0bxoWLmdvb2dsZS5wcm90",
"b2J1Zi5FbXB0eUIrqgIoWkIuTU9NLldXLlNjYWRhQnJpZGdlLkNvbW11bmlj",
"YXRpb24uR3JwY2IGcHJvdG8z"));
descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData,
new pbr::FileDescriptor[] { global::Google.Protobuf.WellKnownTypes.EmptyReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.TimestampReflection.Descriptor, global::Google.Protobuf.WellKnownTypes.WrappersReflection.Descriptor, global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SitestreamReflection.Descriptor, },
new pbr::GeneratedClrTypeInfo(new[] {typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.ConnectionHealthEnum), }, null, new pbr::GeneratedClrTypeInfo[] {
@@ -160,7 +161,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.NodeStatusListDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.NodeStatusListDto.Parser, new[]{ "Nodes" }, null, null, null, null),
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportDto.Parser, new[]{ "SiteId", "SequenceNumber", "ReportTimestamp", "DataConnectionStatuses", "TagResolutionCounts", "ScriptErrorCount", "AlarmEvaluationErrorCount", "StoreAndForwardBufferDepths", "DeadLetterCount", "DeployedInstanceCount", "EnabledInstanceCount", "DisabledInstanceCount", "NodeRole", "NodeHostname", "DataConnectionEndpoints", "DataConnectionTagQuality", "ParkedMessageCount", "ClusterNodes", "SiteAuditWriteFailures", "AuditRedactionFailure", "SiteAuditBacklog", "SiteEventLogWriteFailures", "OldestParkedMessageAgeSeconds", "ScriptQueueDepth", "ScriptBusyThreads", "ScriptOldestBusyAgeSeconds", "LocalDbReplicationConnected", "LocalDbOplogBacklog" }, null, null, null, new pbr::GeneratedClrTypeInfo[] { null, null, null, }),
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportAckDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteHealthReportAckDto.Parser, new[]{ "SiteId", "SequenceNumber", "Accepted", "Error" }, null, null, null, null),
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.HeartbeatDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.HeartbeatDto.Parser, new[]{ "SiteId", "NodeHostname", "IsActive", "Timestamp" }, null, null, null, null)
new pbr::GeneratedClrTypeInfo(typeof(global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.HeartbeatDto), global::ZB.MOM.WW.ScadaBridge.Communication.Grpc.HeartbeatDto.Parser, new[]{ "SiteId", "NodeHostname", "IsActive", "Timestamp", "Synthetic" }, null, null, null, null)
}));
}
#endregion
@@ -5826,6 +5827,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
nodeHostname_ = other.nodeHostname_;
isActive_ = other.isActive_;
timestamp_ = other.timestamp_ != null ? other.timestamp_.Clone() : null;
synthetic_ = other.synthetic_;
_unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields);
}
@@ -5883,6 +5885,27 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
}
}
/// <summary>Field number for the "synthetic" field.</summary>
public const int SyntheticFieldNumber = 5;
private bool synthetic_;
/// <summary>
/// Additive (field 5): true marks a SYNTHETIC heartbeat that carries no liveness
/// meaning — today only CentralChannelProvider's failback probe, which reuses this
/// RPC purely to test whether the preferred central endpoint answers again. Central
/// must skip ALL liveness/health bookkeeping for such a heartbeat: it does not come
/// from a site node's heartbeat timer, and treating it as one would keep a site
/// looking alive on the health dashboard from the probe alone. proto3 defaults it to
/// false, so a pre-existing site's heartbeat is a real one, as before.
/// </summary>
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
public bool Synthetic {
get { return synthetic_; }
set {
synthetic_ = value;
}
}
[global::System.Diagnostics.DebuggerNonUserCodeAttribute]
[global::System.CodeDom.Compiler.GeneratedCode("protoc", null)]
public override bool Equals(object other) {
@@ -5902,6 +5925,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
if (NodeHostname != other.NodeHostname) return false;
if (IsActive != other.IsActive) return false;
if (!object.Equals(Timestamp, other.Timestamp)) return false;
if (Synthetic != other.Synthetic) return false;
return Equals(_unknownFields, other._unknownFields);
}
@@ -5913,6 +5937,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
if (NodeHostname.Length != 0) hash ^= NodeHostname.GetHashCode();
if (IsActive != false) hash ^= IsActive.GetHashCode();
if (timestamp_ != null) hash ^= Timestamp.GetHashCode();
if (Synthetic != false) hash ^= Synthetic.GetHashCode();
if (_unknownFields != null) {
hash ^= _unknownFields.GetHashCode();
}
@@ -5947,6 +5972,10 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
output.WriteRawTag(34);
output.WriteMessage(Timestamp);
}
if (Synthetic != false) {
output.WriteRawTag(40);
output.WriteBool(Synthetic);
}
if (_unknownFields != null) {
_unknownFields.WriteTo(output);
}
@@ -5973,6 +6002,10 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
output.WriteRawTag(34);
output.WriteMessage(Timestamp);
}
if (Synthetic != false) {
output.WriteRawTag(40);
output.WriteBool(Synthetic);
}
if (_unknownFields != null) {
_unknownFields.WriteTo(ref output);
}
@@ -5995,6 +6028,9 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
if (timestamp_ != null) {
size += 1 + pb::CodedOutputStream.ComputeMessageSize(Timestamp);
}
if (Synthetic != false) {
size += 1 + 1;
}
if (_unknownFields != null) {
size += _unknownFields.CalculateSize();
}
@@ -6022,6 +6058,9 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
}
Timestamp.MergeFrom(other.Timestamp);
}
if (other.Synthetic != false) {
Synthetic = other.Synthetic;
}
_unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields);
}
@@ -6060,6 +6099,10 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
input.ReadMessage(Timestamp);
break;
}
case 40: {
Synthetic = input.ReadBool();
break;
}
}
}
#endif
@@ -6098,6 +6141,10 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc {
input.ReadMessage(Timestamp);
break;
}
case 40: {
Synthetic = input.ReadBool();
break;
}
}
}
}
src/ZB.MOM.WW.ScadaBridge.Communication/CommunicationOptions.cs
+30
View File
@@ -47,6 +47,36 @@ public class CommunicationOptions
/// </summary>
public TimeSpan NotificationForwardTimeout { get; set; } = TimeSpan.FromSeconds(30);
/// <summary>
/// Audit Log: timeout for the SITE-side Ask that forwards one audit-telemetry batch
/// (<c>IngestAuditEventsCommand</c> / <c>IngestCachedTelemetryCommand</c>) through the site's
/// <c>SiteCommunicationActor</c> and awaits central's ack. Deliberately the LONGEST rung of the
/// ingest timeout ladder.
/// </summary>
/// <remarks>
/// <para>
/// The ladder is strictly monotonic, outermost first:
/// <c>AuditForwardTimeout</c> (35 s, this option) &gt;
/// <c>SiteStreamGrpcServer.AuditIngestAskTimeout</c> (30 s — both the gRPC call deadline and
/// central's own Ask of the ingest singleton) &gt;
/// <c>AuditLogIngestActor.IngestBudget</c> (20 s) &gt;
/// <c>AuditLogIngestActor.IngestSqlCommandTimeout</c> (15 s).
/// </para>
/// <para>
/// Strictness matters: it used to reuse <see cref="NotificationForwardTimeout"/> (30 s), the
/// same value as the rung below it, so a slow-but-succeeding central write could be acked to a
/// caller whose Ask had already expired — the drain loop would treat the batch as unsent and
/// re-ship it. Central dedups on <c>EventId</c>, so the duplicate was harmless but the retry
/// traffic and the "stalled" telemetry signal were not. A timeout here is still transient: the
/// rows stay <c>Pending</c> and drain on the next tick.
/// </para>
/// <para>
/// <see cref="CommunicationOptionsValidator"/> enforces the outermost inequality — this value
/// must be strictly greater than the 30 s gRPC/central Ask rung.
/// </para>
/// </remarks>
public TimeSpan AuditForwardTimeout { get; set; } = TimeSpan.FromSeconds(35);
/// <summary>
/// Preshared key authenticating this node's gRPC control plane — the site↔central
/// boundary. On a site node this is the key its inbound gate
src/ZB.MOM.WW.ScadaBridge.Communication/CommunicationOptionsValidator.cs
+9
View File
@@ -39,6 +39,15 @@ public sealed class CommunicationOptionsValidator : OptionsValidatorBase<Communi
builder.RequireThat(options.NotificationForwardTimeout > TimeSpan.Zero,
$"ScadaBridge:Communication:NotificationForwardTimeout must be a positive duration (was {options.NotificationForwardTimeout}).");
// Outermost rung of the audit-ingest timeout ladder. It must be strictly greater than the
// gRPC deadline / central Ask beneath it, or a slow-but-succeeding central write gets acked
// to a caller that already gave up and re-sent the batch. See CommunicationOptions.
builder.RequireThat(
options.AuditForwardTimeout > Grpc.SiteStreamGrpcServer.AuditIngestAskTimeout,
$"ScadaBridge:Communication:AuditForwardTimeout must be strictly greater than the "
+ $"central audit-ingest Ask timeout ({Grpc.SiteStreamGrpcServer.AuditIngestAskTimeout}) "
+ $"so the ingest timeout ladder stays monotonic (was {options.AuditForwardTimeout}).");
builder.RequireThat(options.GrpcKeepAlivePingDelay > TimeSpan.Zero,
$"ScadaBridge:Communication:GrpcKeepAlivePingDelay must be a positive duration (was {options.GrpcKeepAlivePingDelay}).");
src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/CentralChannelProvider.cs
+17 -1
View File
@@ -38,6 +38,15 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc;
/// </remarks>
public sealed class CentralChannelProvider : IDisposable
{
/// <summary>
/// Hostname stamped on the failback probe's heartbeat, so a central-side log line names it
/// for what it is. It is a LABEL only — the machine-readable contract is
/// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Messages.Health.HeartbeatMessage.Synthetic"/>
/// (<c>HeartbeatDto.synthetic</c>, proto field 5), which central keys its skip on. Never
/// filter on this string.
/// </summary>
public const string SyntheticProbeHostname = "failback-probe";
private static readonly TimeSpan DefaultBackoffBase = TimeSpan.FromSeconds(1);
private static readonly TimeSpan DefaultBackoffCap = TimeSpan.FromSeconds(60);
private static readonly TimeSpan DefaultProbeDeadline = TimeSpan.FromSeconds(5);
@@ -202,9 +211,16 @@ public sealed class CentralChannelProvider : IDisposable
new HeartbeatDto
{
SiteId = _siteId,
NodeHostname = "failback-probe",
NodeHostname = SyntheticProbeHostname,
IsActive = false,
Timestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
// Explicit contract, not a hostname convention: central skips ALL
// liveness/health bookkeeping for a synthetic heartbeat. Without it the
// probe would stamp the site's last-seen time on the central health
// aggregator every backoff tick, so a site whose real heartbeats had
// stopped could still look alive purely because its transport layer was
// probing for failback.
Synthetic = true,
},
deadline: DateTime.UtcNow.Add(_probeDeadline)).ConfigureAwait(false);
src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/CentralControlDtoMapper.cs
+3 -1
View File
@@ -695,6 +695,7 @@ public static class CentralControlDtoMapper
NodeHostname = msg.NodeHostname,
IsActive = msg.IsActive,
Timestamp = Timestamp.FromDateTimeOffset(msg.Timestamp),
Synthetic = msg.Synthetic,
};
}
@@ -709,7 +710,8 @@ public static class CentralControlDtoMapper
SiteId: dto.SiteId,
NodeHostname: dto.NodeHostname,
IsActive: dto.IsActive,
Timestamp: dto.Timestamp.ToDateTimeOffset());
Timestamp: dto.Timestamp.ToDateTimeOffset(),
Synthetic: dto.Synthetic);
}
/// <summary>Projects a <see cref="ConnectionHealth"/> onto its wire enum.</summary>
src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/CentralControlGrpcService.cs
+85 -7
View File
@@ -33,13 +33,23 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc;
/// host's Akka bootstrap, not by the container.
/// </para>
/// <para>
/// <b>The two audit-ingest RPCs are the exception: they Ask the ingest singleton DIRECTLY.</b>
/// <c>CentralCommunicationActor</c> used to relay them — an extra hop that re-<c>Ask</c>ed the
/// same <c>audit-log-ingest</c> proxy with the same 30 s
/// <see cref="SiteStreamGrpcServer.AuditIngestAskTimeout"/>, so the inner Ask expired at the same
/// instant as the outer one and could only add latency plus a
/// <see cref="Akka.Actor.Status.Failure"/> repackaging. The proxy now arrives through
/// <see cref="SetAuditIngestActor"/>, exactly as <see cref="SiteStreamGrpcServer"/> has always
/// taken it, and the relay handlers were deleted.
/// </para>
/// <para>
/// <b>Fault semantics deliberately differ from <see cref="SiteStreamGrpcServer"/>'s ingest
/// RPCs.</b> That server answers a failed audit ingest with an EMPTY <c>IngestAck</c>; this one
/// fails the call with a non-OK status. Both leave the site's rows <c>Pending</c> for the next
/// drain, so the outcome is the same — but this service replaced the ClusterClient path, whose
/// documented behaviour was to propagate the fault (<c>CentralCommunicationActor</c>'s
/// <c>HandleIngestAuditEvents</c> pipes a <c>Status.Failure</c> back), and preserving that keeps
/// a lost batch visible as a failure rather than as a successful call that acked nothing.
/// documented behaviour was to propagate the fault, and preserving that keeps a lost batch
/// visible as a failure rather than as a successful call that acked nothing. A proxy that has
/// not been handed over yet is NOT a fault and still answers with an empty ack.
/// </para>
/// <para>
/// <b>Status mapping, and why it is not uniform.</b> A site transport may safely re-send a call
@@ -66,6 +76,14 @@ public sealed class CentralControlGrpcService : CentralControlService.CentralCon
// Kestrel's thread pool.
private volatile IActorRef? _central;
// Central-side audit-ingest singleton proxy (audit-log-ingest). Handed over by the host
// AFTER the singleton starts, so it arrives strictly later than _central. The two ingest
// RPCs Ask THIS proxy directly rather than relaying through CentralCommunicationActor —
// see the class remarks. Null until registration completes; an ingest arriving before then
// gets an empty ack (leave the site's rows Pending, retry next drain), which is exactly
// what the removed relay replied in the same window.
private volatile IActorRef? _auditIngest;
/// <summary>
/// Creates the service. <b>This must remain the only public constructor</b> — see
/// <see cref="SetReady"/> for how the actor arrives, and the Host's
@@ -105,9 +123,32 @@ public sealed class CentralControlGrpcService : CentralControlService.CentralCon
_central = centralCommunicationActor;
}
/// <summary>
/// Hands the central <c>AuditLogIngestActor</c> singleton proxy to the service so the two
/// ingest RPCs can Ask it DIRECTLY. Mirrors
/// <see cref="SiteStreamGrpcServer.SetAuditIngestActor"/> — the site-stream server has always
/// dispatched straight to the singleton, and this service now matches it.
/// </summary>
/// <remarks>
/// Removing the <c>CentralCommunicationActor</c> relay removes an actor hop, a second Ask,
/// and a second timeout from every audit batch: the relay's own Ask was armed with the same
/// 30 s <see cref="SiteStreamGrpcServer.AuditIngestAskTimeout"/> as this service's, so the two
/// expired together and the inner one could only ever add latency and a
/// <see cref="Akka.Actor.Status.Failure"/> repackaging on the way out.
/// </remarks>
/// <param name="auditIngestProxy">The audit-log-ingest singleton proxy.</param>
public void SetAuditIngestActor(IActorRef auditIngestProxy)
{
ArgumentNullException.ThrowIfNull(auditIngestProxy);
_auditIngest = auditIngestProxy;
}
/// <summary>Exposed for wiring assertions in tests.</summary>
internal bool IsReady => _central is not null;
/// <summary>Exposed for wiring assertions in tests.</summary>
internal bool IsAuditIngestBound => _auditIngest is not null;
/// <inheritdoc />
public override async Task<NotificationSubmitAckDto> SubmitNotification(
NotificationSubmitDto request, ServerCallContext context)
@@ -147,9 +188,16 @@ public sealed class CentralControlGrpcService : CentralControlService.CentralCon
return new IngestAck();
}
var central = RequireReady(context);
// Node-readiness gate first (the actor system must be up), then the ingest proxy.
RequireReady(context);
var ingest = RequireAuditIngest(request.Events.Count, nameof(IngestAuditEvents));
if (ingest is null)
{
return new IngestAck();
}
var reply = await AskAsync<IngestAuditEventsReply>(
central,
ingest,
CentralControlDtoMapper.FromDto(request),
SiteStreamGrpcServer.AuditIngestAskTimeout,
context).ConfigureAwait(false);
@@ -166,9 +214,15 @@ public sealed class CentralControlGrpcService : CentralControlService.CentralCon
return new IngestAck();
}
var central = RequireReady(context);
RequireReady(context);
var ingest = RequireAuditIngest(request.Packets.Count, nameof(IngestCachedTelemetry));
if (ingest is null)
{
return new IngestAck();
}
var reply = await AskAsync<IngestCachedTelemetryReply>(
central,
ingest,
CentralControlDtoMapper.FromDto(request),
SiteStreamGrpcServer.AuditIngestAskTimeout,
context).ConfigureAwait(false);
@@ -176,6 +230,30 @@ public sealed class CentralControlGrpcService : CentralControlService.CentralCon
return CentralControlDtoMapper.ToIngestAck(reply.AcceptedEventIds);
}
/// <summary>
/// Returns the audit-ingest singleton proxy, or <c>null</c> when the host has not handed it
/// over yet (it starts moments after <see cref="SetReady"/>). A null answer means the caller
/// must reply with an EMPTY <see cref="IngestAck"/> — nothing accepted, rows stay
/// <c>Pending</c> on the site and drain on the next tick. That is deliberately NOT
/// <see cref="StatusCode.Unavailable"/>: readiness here is intentionally narrow (see
/// <see cref="SetReady"/>), and this is the exact reply the removed
/// <c>CentralCommunicationActor</c> relay produced in the same window.
/// </summary>
private IActorRef? RequireAuditIngest(int itemCount, string rpc)
{
var ingest = _auditIngest;
if (ingest is not null)
{
return ingest;
}
_logger.LogWarning(
"{Rpc} received {Count} item(s) before SetAuditIngestActor was called; returning an "
+ "empty ack so the site keeps its rows Pending and retries.",
rpc, itemCount);
return null;
}
/// <inheritdoc />
public override async Task<ReconcileSiteResponseDto> ReconcileSite(
ReconcileSiteRequestDto request, ServerCallContext context)
src/ZB.MOM.WW.ScadaBridge.Communication/ISiteAlarmLiveCache.cs
+25
View File
@@ -62,4 +62,29 @@ public interface ISiteAlarmLiveCache
/// <param name="siteId">The numeric site id.</param>
/// <returns><c>true</c> while a <b>living</b> aggregator has seeded and published; reverts to false if the aggregator terminates (R2 N6) — a frozen snapshot is never reported as live.</returns>
bool IsLive(int siteId);
/// <summary>
/// The Enabled instances on the site that failed to answer the aggregator's most recent
/// snapshot fan-out (seed or reconcile) — i.e. that reported <c>InstanceNotFound</c>, timed
/// out, or faulted. Deterministically ordered by instance name, and empty when the site has
/// no aggregator or has not yet completed a fan-out.
/// </summary>
/// <remarks>
/// <para>
/// Published alongside the alarm snapshot on every fan-out, because the aggregator already
/// knows it: the seed/reconcile is the SAME per-instance debug-snapshot fan-out the Alarm
/// Summary poll used to run a second time purely to compute this list. Exposing it here lets
/// the UI-side poll be skipped entirely while <see cref="IsLive"/> is true, which is the
/// whole point — one fan-out per site per reconcile window, not two.
/// </para>
/// <para>
/// It is refreshed at the aggregator's fan-out cadence (seed, then each reconcile), NOT on
/// every stream delta: an instance going silent is precisely what the stream cannot tell you.
/// Callers must therefore treat it as reconcile-fresh, not delta-fresh — which matches the
/// poll's own 15 s freshness, so nothing on screen changes.
/// </para>
/// </remarks>
/// <param name="siteId">The numeric site id.</param>
/// <returns>The not-reporting instance unique names (possibly empty), never <c>null</c>.</returns>
IReadOnlyList<string> GetNotReportingInstances(int siteId);
}
src/ZB.MOM.WW.ScadaBridge.Communication/Protos/central_control.proto
+8
View File
@@ -244,4 +244,12 @@ message HeartbeatDto {
string node_hostname = 2;
bool is_active = 3;
google.protobuf.Timestamp timestamp = 4;
// Additive (field 5): true marks a SYNTHETIC heartbeat that carries no liveness
// meaning — today only CentralChannelProvider's failback probe, which reuses this
// RPC purely to test whether the preferred central endpoint answers again. Central
// must skip ALL liveness/health bookkeeping for such a heartbeat: it does not come
// from a site node's heartbeat timer, and treating it as one would keep a site
// looking alive on the health dashboard from the probe alone. proto3 defaults it to
// false, so a pre-existing site's heartbeat is a real one, as before.
bool synthetic = 5;
}
src/ZB.MOM.WW.ScadaBridge.Communication/SiteAlarmLiveCacheService.cs
+60 -2
View File
@@ -30,6 +30,7 @@ namespace ZB.MOM.WW.ScadaBridge.Communication;
public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
{
private static readonly IReadOnlyList<AlarmStateChanged> Empty = Array.Empty<AlarmStateChanged>();
private static readonly IReadOnlyList<string> EmptyNames = Array.Empty<string>();
private readonly IServiceProvider _serviceProvider;
private readonly CommunicationService _communicationService;
@@ -135,6 +136,13 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
return _sites.TryGetValue(siteId, out var entry) && entry.HasPublished && entry.StreamLive;
}
/// <inheritdoc />
public IReadOnlyList<string> GetNotReportingInstances(int siteId)
{
lock (_lock)
return _sites.TryGetValue(siteId, out var entry) ? entry.NotReporting : EmptyNames;
}
// ── Subscriber teardown ─────────────────────────────────────────────────────
private void Unsubscribe(Subscription subscription)
@@ -332,6 +340,15 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
/// <see cref="DebugViewSnapshot.InstanceNotFound"/> contributes no rows rather than
/// failing the whole seed (mirrors <c>AlarmSummaryService</c>). Re-enumerates instances
/// each call so a reconcile picks up enable/disable/deploy/delete drift.
/// <para>
/// It also records WHICH instances failed to answer into
/// <see cref="SiteEntry.NotReporting"/>, published through
/// <see cref="ISiteAlarmLiveCache.GetNotReportingInstances"/>. That set was previously
/// computed and thrown away here, forcing the Alarm Summary page to run a second, identical
/// fan-out through <c>AlarmSummaryService</c> purely to rebuild it. Same classification as
/// that service (<c>InstanceNotFound</c> or any non-cancellation fault), same
/// ordinal-ignore-case ordering, so the page renders identically from either source.
/// </para>
/// </summary>
private async Task<IReadOnlyList<AlarmStateChanged>> FanOutSnapshotsAsync(int siteId, CancellationToken ct)
{
@@ -343,7 +360,10 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
var siteRepo = scope.ServiceProvider.GetRequiredService<ISiteRepository>();
var site = await siteRepo.GetSiteByIdAsync(siteId, ct);
if (site is null)
{
PublishNotReporting(siteId, EmptyNames);
return Empty;
}
siteIdentifier = site.SiteIdentifier;
var instanceRepo = scope.ServiceProvider.GetRequiredService<ITemplateEngineRepository>();
@@ -355,25 +375,48 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
}
if (enabledInstanceNames.Count == 0)
{
PublishNotReporting(siteId, EmptyNames);
return Empty;
}
var rows = new ConcurrentBag<AlarmStateChanged>();
var notReporting = new ConcurrentBag<string>();
using var gate = new SemaphoreSlim(
Math.Max(1, _options.LiveAlarmCacheSeedConcurrency),
Math.Max(1, _options.LiveAlarmCacheSeedConcurrency));
var fetches = enabledInstanceNames.Select(name =>
FetchInstanceSnapshotAsync(siteIdentifier, name, gate, rows, ct));
FetchInstanceSnapshotAsync(siteIdentifier, name, gate, rows, notReporting, ct));
await Task.WhenAll(fetches);
PublishNotReporting(
siteId,
notReporting.OrderBy(n => n, StringComparer.OrdinalIgnoreCase).ToList());
return rows.ToList();
}
/// <summary>
/// Stores the fan-out's not-reporting set on the site entry. Silently no-ops when the entry
/// has been torn down mid-fan-out (last viewer left), matching every other late-write path
/// in this service.
/// </summary>
private void PublishNotReporting(int siteId, IReadOnlyList<string> notReporting)
{
lock (_lock)
{
if (_sites.TryGetValue(siteId, out var entry))
entry.NotReporting = notReporting;
}
}
private async Task FetchInstanceSnapshotAsync(
string siteIdentifier,
string instanceUniqueName,
SemaphoreSlim gate,
ConcurrentBag<AlarmStateChanged> rows,
ConcurrentBag<string> notReporting,
CancellationToken ct)
{
await gate.WaitAsync(ct);
@@ -387,7 +430,10 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
instanceUniqueName, Guid.NewGuid().ToString("N"), AlarmsOnly: true);
var snapshot = await _communicationService.RequestDebugSnapshotAsync(siteIdentifier, request, ct);
if (snapshot.InstanceNotFound)
{
notReporting.Add(instanceUniqueName);
return;
}
foreach (var alarm in snapshot.AlarmStates)
rows.Add(alarm);
@@ -398,7 +444,9 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
}
catch
{
// Degrade this one instance to "no rows" rather than failing the whole seed.
// Degrade this one instance to "no rows, not reporting" rather than failing the
// whole seed. Same rule as AlarmSummaryService.FetchInstanceAsync.
notReporting.Add(instanceUniqueName);
}
finally
{
@@ -474,6 +522,9 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
entry.HasPublished = false;
entry.StreamLive = false;
entry.Current = Empty;
// A dead aggregator's last fan-out result is stale too — clear it so the page's
// poll fallback (now authoritative again) owns the not-reporting list outright.
entry.NotReporting = EmptyNames;
if (entry.Subscribers.Count > 0 && !entry.Starting)
{
@@ -509,6 +560,13 @@ public sealed class SiteAlarmLiveCacheService : ISiteAlarmLiveCache
/// <summary>True once the aggregator has seeded and published at least once.</summary>
public bool HasPublished { get; set; }
/// <summary>
/// Enabled instances that failed to answer the most recent snapshot fan-out, ordered by
/// name. Refreshed on every seed/reconcile (see <c>FanOutSnapshotsAsync</c>) and cleared
/// when liveness resets, so a dead aggregator never leaves a stale list behind.
/// </summary>
public IReadOnlyList<string> NotReporting { get; set; } = EmptyNames;
/// <summary>
/// Liveness of the aggregator's site-wide gRPC stream as of the last publish. Both
/// this and <see cref="HasPublished"/> must hold for <c>IsLive</c>: a published cache