perf(comms+audit): close phase-2 residuals — direct ingest path, monotonic timeouts, synthetic probe, not-reporting set, cursor-exact audit pull

src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/CentralChannelProvider.cs

@@ -38,6 +38,15 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc;
 /// </remarks>
 public sealed class CentralChannelProvider : IDisposable
 {
+    /// <summary>
+    /// Hostname stamped on the failback probe's heartbeat, so a central-side log line names it
+    /// for what it is. It is a LABEL only — the machine-readable contract is
+    /// <see cref="ZB.MOM.WW.ScadaBridge.Commons.Messages.Health.HeartbeatMessage.Synthetic"/>
+    /// (<c>HeartbeatDto.synthetic</c>, proto field 5), which central keys its skip on. Never
+    /// filter on this string.
+    /// </summary>
+    public const string SyntheticProbeHostname = "failback-probe";
+
     private static readonly TimeSpan DefaultBackoffBase = TimeSpan.FromSeconds(1);
     private static readonly TimeSpan DefaultBackoffCap = TimeSpan.FromSeconds(60);
     private static readonly TimeSpan DefaultProbeDeadline = TimeSpan.FromSeconds(5);
@@ -202,9 +211,16 @@ public sealed class CentralChannelProvider : IDisposable
                 new HeartbeatDto
                 {
                     SiteId = _siteId,
-                    NodeHostname = "failback-probe",
+                    NodeHostname = SyntheticProbeHostname,
                     IsActive = false,
                     Timestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
+                    // Explicit contract, not a hostname convention: central skips ALL
+                    // liveness/health bookkeeping for a synthetic heartbeat. Without it the
+                    // probe would stamp the site's last-seen time on the central health
+                    // aggregator every backoff tick, so a site whose real heartbeats had
+                    // stopped could still look alive purely because its transport layer was
+                    // probing for failback.
+                    Synthetic = true,
                 },
                 deadline: DateTime.UtcNow.Add(_probeDeadline)).ConfigureAwait(false);
 

src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/CentralControlDtoMapper.cs

@@ -695,6 +695,7 @@ public static class CentralControlDtoMapper
             NodeHostname = msg.NodeHostname,
             IsActive = msg.IsActive,
             Timestamp = Timestamp.FromDateTimeOffset(msg.Timestamp),
+            Synthetic = msg.Synthetic,
         };
     }
 
@@ -709,7 +710,8 @@ public static class CentralControlDtoMapper
             SiteId: dto.SiteId,
             NodeHostname: dto.NodeHostname,
             IsActive: dto.IsActive,
-            Timestamp: dto.Timestamp.ToDateTimeOffset());
+            Timestamp: dto.Timestamp.ToDateTimeOffset(),
+            Synthetic: dto.Synthetic);
     }
 
     /// <summary>Projects a <see cref="ConnectionHealth"/> onto its wire enum.</summary>

src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/CentralControlGrpcService.cs

@@ -33,13 +33,23 @@ namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc;
 /// host's Akka bootstrap, not by the container.
 /// </para>
 /// <para>
+/// <b>The two audit-ingest RPCs are the exception: they Ask the ingest singleton DIRECTLY.</b>
+/// <c>CentralCommunicationActor</c> used to relay them — an extra hop that re-<c>Ask</c>ed the
+/// same <c>audit-log-ingest</c> proxy with the same 30 s
+/// <see cref="SiteStreamGrpcServer.AuditIngestAskTimeout"/>, so the inner Ask expired at the same
+/// instant as the outer one and could only add latency plus a
+/// <see cref="Akka.Actor.Status.Failure"/> repackaging. The proxy now arrives through
+/// <see cref="SetAuditIngestActor"/>, exactly as <see cref="SiteStreamGrpcServer"/> has always
+/// taken it, and the relay handlers were deleted.
+/// </para>
+/// <para>
 /// <b>Fault semantics deliberately differ from <see cref="SiteStreamGrpcServer"/>'s ingest
 /// RPCs.</b> That server answers a failed audit ingest with an EMPTY <c>IngestAck</c>; this one
 /// fails the call with a non-OK status. Both leave the site's rows <c>Pending</c> for the next
 /// drain, so the outcome is the same — but this service replaced the ClusterClient path, whose
-/// documented behaviour was to propagate the fault (<c>CentralCommunicationActor</c>'s
-/// <c>HandleIngestAuditEvents</c> pipes a <c>Status.Failure</c> back), and preserving that keeps
-/// a lost batch visible as a failure rather than as a successful call that acked nothing.
+/// documented behaviour was to propagate the fault, and preserving that keeps a lost batch
+/// visible as a failure rather than as a successful call that acked nothing. A proxy that has
+/// not been handed over yet is NOT a fault and still answers with an empty ack.
 /// </para>
 /// <para>
 /// <b>Status mapping, and why it is not uniform.</b> A site transport may safely re-send a call
@@ -66,6 +76,14 @@ public sealed class CentralControlGrpcService : CentralControlService.CentralCon
     // Kestrel's thread pool.
     private volatile IActorRef? _central;
 
+    // Central-side audit-ingest singleton proxy (audit-log-ingest). Handed over by the host
+    // AFTER the singleton starts, so it arrives strictly later than _central. The two ingest
+    // RPCs Ask THIS proxy directly rather than relaying through CentralCommunicationActor —
+    // see the class remarks. Null until registration completes; an ingest arriving before then
+    // gets an empty ack (leave the site's rows Pending, retry next drain), which is exactly
+    // what the removed relay replied in the same window.
+    private volatile IActorRef? _auditIngest;
+
     /// <summary>
     /// Creates the service. <b>This must remain the only public constructor</b> — see
     /// <see cref="SetReady"/> for how the actor arrives, and the Host's
@@ -105,9 +123,32 @@ public sealed class CentralControlGrpcService : CentralControlService.CentralCon
         _central = centralCommunicationActor;
     }
 
+    /// <summary>
+    /// Hands the central <c>AuditLogIngestActor</c> singleton proxy to the service so the two
+    /// ingest RPCs can Ask it DIRECTLY. Mirrors
+    /// <see cref="SiteStreamGrpcServer.SetAuditIngestActor"/> — the site-stream server has always
+    /// dispatched straight to the singleton, and this service now matches it.
+    /// </summary>
+    /// <remarks>
+    /// Removing the <c>CentralCommunicationActor</c> relay removes an actor hop, a second Ask,
+    /// and a second timeout from every audit batch: the relay's own Ask was armed with the same
+    /// 30 s <see cref="SiteStreamGrpcServer.AuditIngestAskTimeout"/> as this service's, so the two
+    /// expired together and the inner one could only ever add latency and a
+    /// <see cref="Akka.Actor.Status.Failure"/> repackaging on the way out.
+    /// </remarks>
+    /// <param name="auditIngestProxy">The audit-log-ingest singleton proxy.</param>
+    public void SetAuditIngestActor(IActorRef auditIngestProxy)
+    {
+        ArgumentNullException.ThrowIfNull(auditIngestProxy);
+        _auditIngest = auditIngestProxy;
+    }
+
     /// <summary>Exposed for wiring assertions in tests.</summary>
     internal bool IsReady => _central is not null;
 
+    /// <summary>Exposed for wiring assertions in tests.</summary>
+    internal bool IsAuditIngestBound => _auditIngest is not null;
+
     /// <inheritdoc />
     public override async Task<NotificationSubmitAckDto> SubmitNotification(
         NotificationSubmitDto request, ServerCallContext context)
@@ -147,9 +188,16 @@ public sealed class CentralControlGrpcService : CentralControlService.CentralCon
             return new IngestAck();
         }
 
-        var central = RequireReady(context);
+        // Node-readiness gate first (the actor system must be up), then the ingest proxy.
+        RequireReady(context);
+        var ingest = RequireAuditIngest(request.Events.Count, nameof(IngestAuditEvents));
+        if (ingest is null)
+        {
+            return new IngestAck();
+        }
+
         var reply = await AskAsync<IngestAuditEventsReply>(
-            central,
+            ingest,
             CentralControlDtoMapper.FromDto(request),
             SiteStreamGrpcServer.AuditIngestAskTimeout,
             context).ConfigureAwait(false);
@@ -166,9 +214,15 @@ public sealed class CentralControlGrpcService : CentralControlService.CentralCon
             return new IngestAck();
         }
 
-        var central = RequireReady(context);
+        RequireReady(context);
+        var ingest = RequireAuditIngest(request.Packets.Count, nameof(IngestCachedTelemetry));
+        if (ingest is null)
+        {
+            return new IngestAck();
+        }
+
         var reply = await AskAsync<IngestCachedTelemetryReply>(
-            central,
+            ingest,
             CentralControlDtoMapper.FromDto(request),
             SiteStreamGrpcServer.AuditIngestAskTimeout,
             context).ConfigureAwait(false);
@@ -176,6 +230,30 @@ public sealed class CentralControlGrpcService : CentralControlService.CentralCon
         return CentralControlDtoMapper.ToIngestAck(reply.AcceptedEventIds);
     }
 
+    /// <summary>
+    /// Returns the audit-ingest singleton proxy, or <c>null</c> when the host has not handed it
+    /// over yet (it starts moments after <see cref="SetReady"/>). A null answer means the caller
+    /// must reply with an EMPTY <see cref="IngestAck"/> — nothing accepted, rows stay
+    /// <c>Pending</c> on the site and drain on the next tick. That is deliberately NOT
+    /// <see cref="StatusCode.Unavailable"/>: readiness here is intentionally narrow (see
+    /// <see cref="SetReady"/>), and this is the exact reply the removed
+    /// <c>CentralCommunicationActor</c> relay produced in the same window.
+    /// </summary>
+    private IActorRef? RequireAuditIngest(int itemCount, string rpc)
+    {
+        var ingest = _auditIngest;
+        if (ingest is not null)
+        {
+            return ingest;
+        }
+
+        _logger.LogWarning(
+            "{Rpc} received {Count} item(s) before SetAuditIngestActor was called; returning an "
+            + "empty ack so the site keeps its rows Pending and retries.",
+            rpc, itemCount);
+        return null;
+    }
+
     /// <inheritdoc />
     public override async Task<ReconcileSiteResponseDto> ReconcileSite(
         ReconcileSiteRequestDto request, ServerCallContext context)