perf(comms+audit): close phase-2 residuals — direct ingest path, monotonic timeouts, synthetic probe, not-reporting set, cursor-exact audit pull

src/ZB.MOM.WW.ScadaBridge.Host/Actors/AkkaHostedService.cs

@@ -534,12 +534,12 @@ akka {{
         _logger.LogInformation("NotificationOutbox singleton created and registered with CentralCommunicationActor");
 
         // Audit Log — central singleton mirrors the Notification Outbox
-        // pattern. The IngestAuditEvents gRPC handler lives on SiteStreamGrpcServer
-        // (Communication.Grpc); a central node hosting that server (reconciliation
-        // path) hands the proxy in via SetAuditIngestActor below. When the gRPC
-        // server is not registered (current central topology), the host still
-        // brings the singleton up so an in-process test (or a future
-        // direct caller) can Ask the proxy without further wiring.
+        // pattern. TWO gRPC servers can carry an IngestAuditEvents call and both Ask this
+        // proxy directly: the central-hosted CentralControlGrpcService (the production
+        // site→central path) and SiteStreamGrpcServer (Communication.Grpc). Each is handed
+        // the proxy below; either may be absent on a given node and the wiring no-ops. Even
+        // with neither bound the host still brings the singleton up so an in-process test
+        // (or a future direct caller) can Ask the proxy without further wiring.
         // IAuditLogRepository is a SCOPED EF Core service, so the singleton
         // actor takes the root IServiceProvider and creates a fresh scope per
         // message (mirroring NotificationOutboxActor). Pre-resolving the
@@ -555,10 +555,11 @@ akka {{
                 auditIngestLogger)),
             _logger);
 
-        // Hand the audit-ingest proxy to the CentralCommunicationActor so audit
-        // ingest commands forwarded by sites are routed to the
-        // singleton. Mirrors the RegisterNotificationOutbox wiring above.
-        centralCommActor.Tell(new RegisterAuditIngest(auditIngest.Proxy));
+        // Hand the audit-ingest proxy to the central-hosted gRPC control plane so its two
+        // ingest RPCs Ask the singleton DIRECTLY. There is deliberately no
+        // CentralCommunicationActor relay any more: it re-Asked this same proxy with the same
+        // 30 s timeout, so it could only add a hop and latency to every audit batch.
+        centralControlGrpc?.SetAuditIngestActor(auditIngest.Proxy);
 
         // Hand the proxy to the SiteStreamGrpcServer (if registered on this node)
         // so the IngestAuditEvents RPC routes incoming site batches to the singleton.
@@ -568,7 +569,9 @@ akka {{
         var grpcServer = _serviceProvider.GetService<ZB.MOM.WW.ScadaBridge.Communication.Grpc.SiteStreamGrpcServer>();
         grpcServer?.SetAuditIngestActor(auditIngest.Proxy);
         _logger.LogInformation(
-            "AuditLogIngestActor singleton created (gRPC server bound: {GrpcBound})",
+            "AuditLogIngestActor singleton created (control-plane bound: {ControlBound}, "
+            + "site-stream server bound: {GrpcBound})",
+            centralControlGrpc is not null,
             grpcServer is not null);
 
         // Subscribe the per-site stalled
@@ -1066,13 +1069,16 @@ akka {{
         // not the DI-resolved NoOpSiteStreamAuditClient. The NoOp default stays
         // correct for central/test composition roots (no SiteCommunicationActor);
         // a site role wires the real client here so the
-        // SQLite Pending backlog actually drains to central. The forward Ask
-        // reuses NotificationForwardTimeout — the same site→central command
-        // forward bound notifications already use over this transport.
+        // SQLite Pending backlog actually drains to central. The forward Ask uses
+        // AuditForwardTimeout (35 s) — the OUTERMOST rung of the ingest timeout
+        // ladder (35 > 30 gRPC/central Ask > 20 actor budget > 15 SQL), so a
+        // slow-but-succeeding central write is never acked to a caller that has
+        // already given up and re-sent. It used to reuse NotificationForwardTimeout
+        // (30 s), which tied it with the rung below.
         ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry.ISiteStreamAuditClient siteAuditClient =
             new ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry.SiteCommunicationAuditClient(
                 siteCommActor,
-                _communicationOptions.NotificationForwardTimeout);
+                _communicationOptions.AuditForwardTimeout);
         var siteAuditLogger = _serviceProvider.GetRequiredService<ILoggerFactory>()
             .CreateLogger<ZB.MOM.WW.ScadaBridge.AuditLog.Site.Telemetry.SiteAuditTelemetryActor>();