dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

perf(comms+audit): close phase-2 residuals — direct ingest path, monotonic timeouts, synthetic probe, not-reporting set, cursor-exact audit pull

Browse Source
This commit is contained in:
Joseph Doherty
2026-08-14 21:38:23 -04:00
parent 4cd1441984
commit a5882753dd
38 changed files with 1254 additions and 443 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ZB.MOM.WW.ScadaBridge.Communication/Actors/CentralCommunicationActor.cs
+36 -113
View File
@@ -67,36 +67,6 @@ public class CentralCommunicationActor : ReceiveActor
/// </summary>
private IActorRef? _notificationOutboxProxy;
/// <summary>
/// Proxy <see cref="IActorRef"/> for the central AuditLogIngestActor cluster
/// singleton. Set via <see cref="RegisterAuditIngest"/> — the Host creates the
/// singleton proxy after this actor and registers it (mirrors
/// <see cref="_notificationOutboxProxy"/>). Null until registration completes;
/// an audit ingest command arriving before then is answered with an empty
/// reply so the site keeps its rows Pending and retries.
///
/// Once registered, the handler Asks this proxy and pipes the reply straight
/// back to the caller. On an Ask timeout or a faulted reply, PipeTo forwards a
/// <see cref="Status.Failure"/> to the caller — the fault propagates rather
/// than being swallowed. This differs from the gRPC handler
/// (<c>SiteStreamGrpcServer</c>), which catches the exception and returns an
/// empty ack; here the faulted Ask is the transient signal the site relies on
/// (see <see cref="HandleIngestAuditEvents"/>).
/// </summary>
private IActorRef? _auditIngestProxy;
/// <summary>
/// Default Ask timeout for routing audit ingest commands to the
/// Effective Ask timeout for audit ingest routing. Defaults to
/// <see cref="Grpc.SiteStreamGrpcServer.AuditIngestAskTimeout"/> (30 s) — the two
/// audit-ingest entry points (the site stream server and the control plane) share one source of truth
/// for the timeout. Overridable via the constructor so tests can exercise the
/// timeout/fault path without waiting 30 s. When the window is exceeded the Ask
/// faults and that fault is piped back to the caller as a
/// <see cref="Status.Failure"/> (see <see cref="HandleIngestAuditEvents"/>).
/// </summary>
private readonly TimeSpan _auditIngestAskTimeout;
/// <summary>
/// DistributedPubSub topic used to fan health reports out to the peer
/// central node so both per-node aggregators stay in sync. See
@@ -111,12 +81,10 @@ public class CentralCommunicationActor : ReceiveActor
/// </summary>
/// <param name="serviceProvider">DI service provider for scoped repository and aggregator access.</param>
/// <param name="transport">The central→site command transport to route every <see cref="SiteEnvelope"/> through.</param>
/// <param name="auditIngestAskTimeout">Optional override for the audit-ingest Ask timeout (test hook).</param>
public CentralCommunicationActor(
IServiceProvider serviceProvider,
ISiteCommandTransport transport,
TimeSpan? auditIngestAskTimeout = null)
: this(serviceProvider, auditIngestAskTimeout)
ISiteCommandTransport transport)
: this(serviceProvider)
{
_transport = transport ?? throw new ArgumentNullException(nameof(transport));
}
@@ -125,13 +93,9 @@ public class CentralCommunicationActor : ReceiveActor
/// <see cref="_transport"/> is assigned by the delegating public constructor before any message
/// is dispatched.</summary>
/// <param name="serviceProvider">DI service provider.</param>
/// <param name="auditIngestAskTimeout">Optional audit-ingest Ask timeout override.</param>
private CentralCommunicationActor(
IServiceProvider serviceProvider,
TimeSpan? auditIngestAskTimeout)
private CentralCommunicationActor(IServiceProvider serviceProvider)
{
_serviceProvider = serviceProvider;
_auditIngestAskTimeout = auditIngestAskTimeout ?? Grpc.SiteStreamGrpcServer.AuditIngestAskTimeout;
// Site address cache loaded from database
Receive<SiteAddressCacheLoaded>(HandleSiteAddressCacheLoaded);
@@ -176,24 +140,11 @@ public class CentralCommunicationActor : ReceiveActor
// so the NotificationStatusResponse routes back to the querying site.
Receive<NotificationStatusQuery>(HandleNotificationStatusQuery);
// Audit Log: the Host registers the AuditLogIngestActor singleton
// proxy after this actor is created (the proxy cannot exist before this
// actor's construction).
Receive<RegisterAuditIngest>(msg =>
{
_auditIngestProxy = msg.AuditIngestActor;
_log.Info("Registered audit ingest proxy");
});
// Audit Log site→central ingest: a site forwards a batch of audit
// events to the central cluster. Ask the ingest proxy
// and pipe the IngestAuditEventsReply back to the original Sender (the
// site's transport path) so the site can flip its rows to Forwarded.
Receive<IngestAuditEventsCommand>(HandleIngestAuditEvents);
// Audit Log combined-telemetry ingest: routes to the same proxy
// the same way; the proxy replies with an IngestCachedTelemetryReply.
Receive<IngestCachedTelemetryCommand>(HandleIngestCachedTelemetry);
// Audit Log site→central ingest is NOT relayed here. The central-hosted
// CentralControlGrpcService Asks the audit-log-ingest singleton proxy directly
// (as SiteStreamGrpcServer always has), so this actor no longer sits in that
// path — the relay was a second hop with an identical 30 s Ask timeout that
// could only add latency. Do not reintroduce it.
// Startup reconciliation: a site node forwards its local deployed inventory on
// startup. Resolve the scoped ReconcileService, diff the
@@ -239,51 +190,6 @@ public class CentralCommunicationActor : ReceiveActor
_notificationOutboxProxy.Forward(msg);
}
private void HandleIngestAuditEvents(IngestAuditEventsCommand msg)
{
if (_auditIngestProxy == null)
{
// No ingest proxy registered yet (host startup race). Reply with an
// empty IngestAuditEventsReply so the site keeps its rows Pending and
// retries — the same behaviour as the gRPC handler's wiring-race path.
_log.Warning(
"Cannot route IngestAuditEventsCommand ({0} events) — audit ingest not available",
msg.Events.Count);
Sender.Tell(new IngestAuditEventsReply(Array.Empty<Guid>()));
return;
}
// Capture Sender before the async/PipeTo — Akka resets Sender between
// dispatches. The reply is piped straight back to the calling site node.
// On an Ask timeout or a faulted reply, PipeTo delivers a Status.Failure to
// replyTo: the fault propagates to the caller rather than being swallowed.
// The site's own Ask through this path then faults, and the site drain loop
// treats that as a transient failure — rows stay Pending and are retried on
// the next tick. (The gRPC handler instead returns an empty ack on fault;
// propagating the fault here is the cleaner transient signal.)
var replyTo = Sender;
_log.Debug("Routing IngestAuditEventsCommand ({0} events) to the audit ingest actor", msg.Events.Count);
_auditIngestProxy.Ask<IngestAuditEventsReply>(msg, _auditIngestAskTimeout)
.PipeTo(replyTo);
}
private void HandleIngestCachedTelemetry(IngestCachedTelemetryCommand msg)
{
if (_auditIngestProxy == null)
{
_log.Warning(
"Cannot route IngestCachedTelemetryCommand ({0} entries) — audit ingest not available",
msg.Entries.Count);
Sender.Tell(new IngestCachedTelemetryReply(Array.Empty<Guid>()));
return;
}
var replyTo = Sender;
_log.Debug("Routing IngestCachedTelemetryCommand ({0} entries) to the audit ingest actor", msg.Entries.Count);
_auditIngestProxy.Ask<IngestCachedTelemetryReply>(msg, _auditIngestAskTimeout)
.PipeTo(replyTo);
}
/// <summary>
/// Startup reconciliation (site→central): resolve the scoped
/// <see cref="ReconcileService"/> in a DI scope, diff the node's reported inventory
@@ -291,7 +197,7 @@ public class CentralCommunicationActor : ReceiveActor
/// back to the site node's transport path. The actor stays thin — all the diff
/// and staging logic lives in the service. Mirrors the DB-access pattern used by
/// <see cref="LoadSiteAddressesFromDb"/> (Task.Run + CreateScope + PipeTo) and the
/// Sender-preservation pattern of <see cref="HandleIngestAuditEvents"/>.
/// Sender-preservation pattern of <see cref="HandleNotificationSubmit"/>.
///
/// On a faulted task PipeTo delivers a <see cref="Status.Failure"/> to the node; its
/// Ask faults and it simply retries reconcile on the next startup — reconcile is
@@ -328,6 +234,21 @@ public class CentralCommunicationActor : ReceiveActor
private void HandleHeartbeat(HeartbeatMessage heartbeat)
{
// Synthetic heartbeats carry no liveness meaning — today only
// CentralChannelProvider's failback probe, which reuses this RPC to ask whether the
// preferred central endpoint answers again. It is emitted by a site's TRANSPORT layer,
// not by any node's heartbeat timer, so stamping it would let a site whose real
// heartbeats had stopped keep looking alive on the health dashboard for as long as its
// transport kept probing. Dropped before the local mark AND before the peer fan-out,
// so neither central node's aggregator ever sees it.
if (heartbeat.Synthetic)
{
_log.Debug(
"Ignoring synthetic heartbeat from site {0} (node '{1}') — probe traffic, not liveness",
heartbeat.SiteId, heartbeat.NodeHostname);
return;
}
MarkHeartbeatLocally(heartbeat);
// Fan the heartbeat out to the peer central node so BOTH aggregators mark
@@ -350,9 +271,17 @@ public class CentralCommunicationActor : ReceiveActor
/// <summary>
/// Marks a site heartbeat on the local aggregator without re-broadcasting.
/// Used for both site-originated heartbeats and peer-replicated ones.
/// A synthetic heartbeat is never marked — <see cref="HandleHeartbeat"/> already drops it
/// before the fan-out, so a replica can only carry one if a peer node predates the flag;
/// the guard here is belt-and-braces on the last hop before the aggregator.
/// </summary>
private void MarkHeartbeatLocally(HeartbeatMessage heartbeat)
{
if (heartbeat.Synthetic)
{
return;
}
var aggregator = _serviceProvider.GetService<ICentralHealthAggregator>();
aggregator?.MarkHeartbeat(heartbeat.SiteId, heartbeat.Timestamp);
}
@@ -623,13 +552,7 @@ public record DebugStreamTerminated(string SiteId, string CorrelationId);
/// </summary>
public record RegisterNotificationOutbox(IActorRef OutboxProxy);
/// <summary>
/// Registers the central AuditLogIngestActor singleton proxy with the
/// <see cref="CentralCommunicationActor"/> so site-forwarded
/// <see cref="IngestAuditEventsCommand"/> and <see cref="IngestCachedTelemetryCommand"/>
/// messages can be routed to it. Sent by the Host after the audit-ingest
/// singleton proxy is created. Lives here (not in Commons) because
/// <c>ZB.MOM.WW.ScadaBridge.Commons</c> has no Akka package reference and cannot hold an
/// <see cref="IActorRef"/> field.
/// </summary>
public sealed record RegisterAuditIngest(IActorRef AuditIngestActor);
// NOTE: there is deliberately no RegisterAuditIngest counterpart. The audit-log-ingest
// singleton proxy goes STRAIGHT to the two gRPC servers that need it
// (CentralControlGrpcService.SetAuditIngestActor / SiteStreamGrpcServer.SetAuditIngestActor);
// routing audit batches through this actor was a redundant hop and was removed.