perf(comms+audit): close phase-2 residuals — direct ingest path, monotonic timeouts, synthetic probe, not-reporting set, cursor-exact audit pull

src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/CentralControlGrpcService.cs

@@ -33,13 +33,23 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc;
 /// host's Akka bootstrap, not by the container.
 /// </para>
 /// <para>
+/// <b>The two audit-ingest RPCs are the exception: they Ask the ingest singleton DIRECTLY.</b>
+/// <c>CentralCommunicationActor</c> used to relay them — an extra hop that re-<c>Ask</c>ed the
+/// same <c>audit-log-ingest</c> proxy with the same 30 s
+/// <see cref="SiteStreamGrpcServer.AuditIngestAskTimeout"/>, so the inner Ask expired at the same
+/// instant as the outer one and could only add latency plus a
+/// <see cref="Akka.Actor.Status.Failure"/> repackaging. The proxy now arrives through
+/// <see cref="SetAuditIngestActor"/>, exactly as <see cref="SiteStreamGrpcServer"/> has always
+/// taken it, and the relay handlers were deleted.
+/// </para>
+/// <para>
 /// <b>Fault semantics deliberately differ from <see cref="SiteStreamGrpcServer"/>'s ingest
 /// RPCs.</b> That server answers a failed audit ingest with an EMPTY <c>IngestAck</c>; this one
 /// fails the call with a non-OK status. Both leave the site's rows <c>Pending</c> for the next
 /// drain, so the outcome is the same — but this service replaced the ClusterClient path, whose
-/// documented behaviour was to propagate the fault (<c>CentralCommunicationActor</c>'s
-/// <c>HandleIngestAuditEvents</c> pipes a <c>Status.Failure</c> back), and preserving that keeps
-/// a lost batch visible as a failure rather than as a successful call that acked nothing.
+/// documented behaviour was to propagate the fault, and preserving that keeps a lost batch
+/// visible as a failure rather than as a successful call that acked nothing. A proxy that has
+/// not been handed over yet is NOT a fault and still answers with an empty ack.
 /// </para>
 /// <para>
 /// <b>Status mapping, and why it is not uniform.</b> A site transport may safely re-send a call
@@ -66,6 +76,14 @@ public sealed class CentralControlGrpcService : CentralControlService.CentralCon
     // Kestrel's thread pool.
     private volatile IActorRef? _central;
 
+    // Central-side audit-ingest singleton proxy (audit-log-ingest). Handed over by the host
+    // AFTER the singleton starts, so it arrives strictly later than _central. The two ingest
+    // RPCs Ask THIS proxy directly rather than relaying through CentralCommunicationActor —
+    // see the class remarks. Null until registration completes; an ingest arriving before then
+    // gets an empty ack (leave the site's rows Pending, retry next drain), which is exactly
+    // what the removed relay replied in the same window.
+    private volatile IActorRef? _auditIngest;
+
     /// <summary>
     /// Creates the service. <b>This must remain the only public constructor</b> — see
     /// <see cref="SetReady"/> for how the actor arrives, and the Host's
@@ -105,9 +123,32 @@ public sealed class CentralControlGrpcService : CentralControlService.CentralCon
         _central = centralCommunicationActor;
     }
 
+    /// <summary>
+    /// Hands the central <c>AuditLogIngestActor</c> singleton proxy to the service so the two
+    /// ingest RPCs can Ask it DIRECTLY. Mirrors
+    /// <see cref="SiteStreamGrpcServer.SetAuditIngestActor"/> — the site-stream server has always
+    /// dispatched straight to the singleton, and this service now matches it.
+    /// </summary>
+    /// <remarks>
+    /// Removing the <c>CentralCommunicationActor</c> relay removes an actor hop, a second Ask,
+    /// and a second timeout from every audit batch: the relay's own Ask was armed with the same
+    /// 30 s <see cref="SiteStreamGrpcServer.AuditIngestAskTimeout"/> as this service's, so the two
+    /// expired together and the inner one could only ever add latency and a
+    /// <see cref="Akka.Actor.Status.Failure"/> repackaging on the way out.
+    /// </remarks>
+    /// <param name="auditIngestProxy">The audit-log-ingest singleton proxy.</param>
+    public void SetAuditIngestActor(IActorRef auditIngestProxy)
+    {
+        ArgumentNullException.ThrowIfNull(auditIngestProxy);
+        _auditIngest = auditIngestProxy;
+    }
+
     /// <summary>Exposed for wiring assertions in tests.</summary>
     internal bool IsReady => _central is not null;
 
+    /// <summary>Exposed for wiring assertions in tests.</summary>
+    internal bool IsAuditIngestBound => _auditIngest is not null;
+
     /// <inheritdoc />
     public override async Task<NotificationSubmitAckDto> SubmitNotification(
         NotificationSubmitDto request, ServerCallContext context)
@@ -147,9 +188,16 @@ public sealed class CentralControlGrpcService : CentralControlService.CentralCon
             return new IngestAck();
         }
 
-        var central = RequireReady(context);
+        // Node-readiness gate first (the actor system must be up), then the ingest proxy.
+        RequireReady(context);
+        var ingest = RequireAuditIngest(request.Events.Count, nameof(IngestAuditEvents));
+        if (ingest is null)
+        {
+            return new IngestAck();
+        }
+
         var reply = await AskAsync<IngestAuditEventsReply>(
-            central,
+            ingest,
             CentralControlDtoMapper.FromDto(request),
             SiteStreamGrpcServer.AuditIngestAskTimeout,
             context).ConfigureAwait(false);
@@ -166,9 +214,15 @@ public sealed class CentralControlGrpcService : CentralControlService.CentralCon
             return new IngestAck();
         }
 
-        var central = RequireReady(context);
+        RequireReady(context);
+        var ingest = RequireAuditIngest(request.Packets.Count, nameof(IngestCachedTelemetry));
+        if (ingest is null)
+        {
+            return new IngestAck();
+        }
+
         var reply = await AskAsync<IngestCachedTelemetryReply>(
-            central,
+            ingest,
             CentralControlDtoMapper.FromDto(request),
             SiteStreamGrpcServer.AuditIngestAskTimeout,
             context).ConfigureAwait(false);
@@ -176,6 +230,30 @@ public sealed class CentralControlGrpcService : CentralControlService.CentralCon
         return CentralControlDtoMapper.ToIngestAck(reply.AcceptedEventIds);
     }
 
+    /// <summary>
+    /// Returns the audit-ingest singleton proxy, or <c>null</c> when the host has not handed it
+    /// over yet (it starts moments after <see cref="SetReady"/>). A null answer means the caller
+    /// must reply with an EMPTY <see cref="IngestAck"/> — nothing accepted, rows stay
+    /// <c>Pending</c> on the site and drain on the next tick. That is deliberately NOT
+    /// <see cref="StatusCode.Unavailable"/>: readiness here is intentionally narrow (see
+    /// <see cref="SetReady"/>), and this is the exact reply the removed
+    /// <c>CentralCommunicationActor</c> relay produced in the same window.
+    /// </summary>
+    private IActorRef? RequireAuditIngest(int itemCount, string rpc)
+    {
+        var ingest = _auditIngest;
+        if (ingest is not null)
+        {
+            return ingest;
+        }
+
+        _logger.LogWarning(
+            "{Rpc} received {Count} item(s) before SetAuditIngestActor was called; returning an "
+            + "empty ack so the site keeps its rows Pending and retries.",
+            rpc, itemCount);
+        return null;
+    }
+
     /// <inheritdoc />
     public override async Task<ReconcileSiteResponseDto> ReconcileSite(
         ReconcileSiteRequestDto request, ServerCallContext context)