using Akka.Actor; using Grpc.Core; using Microsoft.Extensions.Logging; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health; using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification; using ZB.MOM.WW.ScadaBridge.Communication.Actors; using AkkaStatus = Akka.Actor.Status; namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc; /// /// The that carries the seven site→central sends over gRPC — the /// migration target for the Akka ClusterClient path. Each method encodes the message with /// , dials CentralControlService through the sticky /// , and delivers the decoded reply (or a transient-failure /// signal) to the waiting Ask. /// /// /// /// The actor handlers are synchronous; each method here kicks off the RPC as a detached task and /// Tells the result to when it completes — IActorRef.Tell /// is thread-safe, so the reply lands at the Ask exactly as central's ClusterClient reply did. On /// any non-OK status the reply is , which the S&F / audit / health /// layers already treat as transient. /// /// /// Cross-node retry only on provably-unsent failures. An /// (connection refused / node not ready) flips the channel pair and retries once on the peer. /// A is NEVER retried across nodes — a deploy / write / /// failover may already have executed, and duplicating it is worse than surfacing a transient /// failure the layer above tolerates. /// /// /// Per-call deadlines mirror today's Ask timeouts. Notification submit/status → /// NotificationForwardTimeout (30 s, the value the S&F forwarder and the central service /// both use); health → HealthReportTimeout (10 s); reconcile → QueryTimeout (30 s); /// both ingest RPCs → (the one shared 30 s /// constant). The heartbeat, fire-and-forget with no server-side Ask, is merely bounded by /// HealthReportTimeout and its failures are swallowed. /// /// public sealed class GrpcCentralTransport : ICentralTransport { private readonly CentralChannelProvider _channels; private readonly CommunicationOptions _options; private readonly ILogger _logger; /// Creates the transport over a channel pair. /// The sticky central channel pair. /// Communication options supplying the per-call deadlines. /// Logger for failover/fault diagnostics. public GrpcCentralTransport( CentralChannelProvider channels, CommunicationOptions options, ILogger logger) { ArgumentNullException.ThrowIfNull(channels); ArgumentNullException.ThrowIfNull(options); ArgumentNullException.ThrowIfNull(logger); _channels = channels; _options = options; _logger = logger; } /// public void SubmitNotification(NotificationSubmit message, IActorRef replyTo) { var dto = CentralControlDtoMapper.ToDto(message); Dispatch(replyTo, _options.NotificationForwardTimeout, (c, o) => c.SubmitNotificationAsync(dto, o), ack => CentralControlDtoMapper.FromDto(ack)); } /// public void QueryNotificationStatus(NotificationStatusQuery message, IActorRef replyTo) { var dto = CentralControlDtoMapper.ToDto(message); Dispatch(replyTo, _options.NotificationForwardTimeout, (c, o) => c.QueryNotificationStatusAsync(dto, o), response => CentralControlDtoMapper.FromDto(response)); } /// public void IngestAuditEvents(IngestAuditEventsCommand message, IActorRef replyTo) { var batch = CentralControlDtoMapper.ToDto(message); Dispatch(replyTo, SiteStreamGrpcServer.AuditIngestAskTimeout, (c, o) => c.IngestAuditEventsAsync(batch, o), ack => new IngestAuditEventsReply(CentralControlDtoMapper.FromIngestAck(ack))); } /// public void IngestCachedTelemetry(IngestCachedTelemetryCommand message, IActorRef replyTo) { var batch = CentralControlDtoMapper.ToDto(message); Dispatch(replyTo, SiteStreamGrpcServer.AuditIngestAskTimeout, (c, o) => c.IngestCachedTelemetryAsync(batch, o), ack => new IngestCachedTelemetryReply(CentralControlDtoMapper.FromIngestAck(ack))); } /// public void ReconcileSite(ReconcileSiteRequest message, IActorRef replyTo) { var dto = CentralControlDtoMapper.ToDto(message); Dispatch(replyTo, _options.QueryTimeout, (c, o) => c.ReconcileSiteAsync(dto, o), response => CentralControlDtoMapper.FromDto(response)); } /// public void ReportSiteHealth(SiteHealthReport message, IActorRef replyTo) { var dto = CentralControlDtoMapper.ToDto(message); Dispatch(replyTo, _options.HealthReportTimeout, (c, o) => c.ReportSiteHealthAsync(dto, o), ack => CentralControlDtoMapper.FromDto(ack)); } /// public void SendHeartbeat(HeartbeatMessage message, IActorRef self) { _ = SendHeartbeatAsync(message); } private async Task SendHeartbeatAsync(HeartbeatMessage message) { var (index, client) = _channels.Current(); var dto = CentralControlDtoMapper.ToDto(message); try { var options = new CallOptions(deadline: DateTime.UtcNow.Add(_options.HealthReportTimeout)); using var call = client.HeartbeatAsync(dto, options); await call.ResponseAsync.ConfigureAwait(false); } catch (RpcException ex) when (IsConnectFailure(ex)) { // Nudge the pair so the next call tries the peer, but never fault: a heartbeat // failure must not surface on the site's heartbeat timer path. _channels.ReportUnavailable(index); _logger.LogDebug(ex, "Heartbeat to central endpoint {Endpoint} was unavailable.", CurrentEndpointSafe()); } catch (Exception ex) { _logger.LogDebug(ex, "Heartbeat to central failed (swallowed — heartbeats are fire-and-forget)."); } } /// /// Runs a unary RPC on the current channel, delivers the decoded reply to /// , and applies the sticky-failover / no-retry-on-deadline policy. /// private void Dispatch( IActorRef replyTo, TimeSpan timeout, Func> call, Func decode) { _ = DispatchAsync(replyTo, timeout, call, decode); } private async Task DispatchAsync( IActorRef replyTo, TimeSpan timeout, Func> call, Func decode) { var (index, client) = _channels.Current(); try { var reply = await InvokeAsync(client, timeout, call).ConfigureAwait(false); replyTo.Tell(decode(reply)); } catch (RpcException ex) when (IsConnectFailure(ex)) { // Provably unsent: the connection was refused / the node was not ready. Fail over // to the peer and retry ONCE. This is the only status we retry across nodes. _channels.ReportUnavailable(index); var (retryIndex, retryClient) = _channels.Current(); if (retryIndex != index) { try { var reply = await InvokeAsync(retryClient, timeout, call).ConfigureAwait(false); replyTo.Tell(decode(reply)); return; } catch (Exception retryEx) { _logger.LogWarning(retryEx, "Central control-plane call failed on both endpoints; surfacing as transient."); replyTo.Tell(new AkkaStatus.Failure(retryEx)); return; } } replyTo.Tell(new AkkaStatus.Failure(ex)); } catch (Exception ex) { // DeadlineExceeded / Internal / PermissionDenied / a PSK-resolution throw — do NOT // retry across nodes (the call may have run). Surface as the transient failure the // layer above already tolerates. replyTo.Tell(new AkkaStatus.Failure(ex)); } } private static async Task InvokeAsync( CentralControlService.CentralControlServiceClient client, TimeSpan timeout, Func> call) { var options = new CallOptions(deadline: DateTime.UtcNow.Add(timeout)); using var asyncCall = call(client, options); return await asyncCall.ResponseAsync.ConfigureAwait(false); } /// /// A failure that provably never reached a server — the only class safe to retry on the peer. /// is deliberately excluded (the call may have run). /// /// /// Two shapes qualify: a server-signalled (e.g. a node /// that returns Unavailable while it is still starting), and a client-side failure to even /// start the call — Grpc.Net surfaces a refused/failed connection as /// "Error starting gRPC call" with the transport exception /// attached, and there the request never left the client. Anything else — including a deadline, /// a permission denial, or a generic server-side Internal after the call reached the server — /// is NOT retried across nodes. /// private static bool IsConnectFailure(RpcException ex) { if (ex.StatusCode == StatusCode.Unavailable) { return true; } // "Error starting gRPC call" is Grpc.Net's marker for a call that could not be sent — a // refused/failed connection carrying an HttpRequestException. Provably unsent. return ex.StatusCode == StatusCode.Internal && (ex.Status.DebugException is HttpRequestException || ex.Status.Detail.StartsWith("Error starting gRPC call", StringComparison.Ordinal)); } private string CurrentEndpointSafe() { try { return _channels.CurrentEndpoint; } catch { return "(unknown)"; } } }