using Akka.Actor;
using Grpc.Core;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
using AkkaStatus = Akka.Actor.Status;
namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc;
///
/// The that carries the seven site→central sends over gRPC — the
/// migration target for the Akka ClusterClient path. Each method encodes the message with
/// , dials CentralControlService through the sticky
/// , and delivers the decoded reply (or a transient-failure
/// signal) to the waiting Ask.
///
///
///
/// The actor handlers are synchronous; each method here kicks off the RPC as a detached task and
/// Tells the result to when it completes — IActorRef.Tell
/// is thread-safe, so the reply lands at the Ask exactly as central's ClusterClient reply did. On
/// any non-OK status the reply is , which the S&F / audit / health
/// layers already treat as transient.
///
///
/// Cross-node retry only on provably-unsent failures. An
/// (connection refused / node not ready) flips the channel pair and retries once on the peer.
/// A is NEVER retried across nodes — a deploy / write /
/// failover may already have executed, and duplicating it is worse than surfacing a transient
/// failure the layer above tolerates.
///
///
/// Per-call deadlines mirror today's Ask timeouts. Notification submit/status →
/// NotificationForwardTimeout (30 s, the value the S&F forwarder and the central service
/// both use); health → HealthReportTimeout (10 s); reconcile → QueryTimeout (30 s);
/// both ingest RPCs → (the one shared 30 s
/// constant). The heartbeat, fire-and-forget with no server-side Ask, is merely bounded by
/// HealthReportTimeout and its failures are swallowed.
///
///
public sealed class GrpcCentralTransport : ICentralTransport
{
private readonly CentralChannelProvider _channels;
private readonly CommunicationOptions _options;
private readonly ILogger _logger;
/// Creates the transport over a channel pair.
/// The sticky central channel pair.
/// Communication options supplying the per-call deadlines.
/// Logger for failover/fault diagnostics.
public GrpcCentralTransport(
CentralChannelProvider channels,
CommunicationOptions options,
ILogger logger)
{
ArgumentNullException.ThrowIfNull(channels);
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(logger);
_channels = channels;
_options = options;
_logger = logger;
}
///
public void SubmitNotification(NotificationSubmit message, IActorRef replyTo)
{
var dto = CentralControlDtoMapper.ToDto(message);
Dispatch(replyTo, _options.NotificationForwardTimeout,
(c, o) => c.SubmitNotificationAsync(dto, o),
ack => CentralControlDtoMapper.FromDto(ack));
}
///
public void QueryNotificationStatus(NotificationStatusQuery message, IActorRef replyTo)
{
var dto = CentralControlDtoMapper.ToDto(message);
Dispatch(replyTo, _options.NotificationForwardTimeout,
(c, o) => c.QueryNotificationStatusAsync(dto, o),
response => CentralControlDtoMapper.FromDto(response));
}
///
public void IngestAuditEvents(IngestAuditEventsCommand message, IActorRef replyTo)
{
var batch = CentralControlDtoMapper.ToDto(message);
Dispatch(replyTo, SiteStreamGrpcServer.AuditIngestAskTimeout,
(c, o) => c.IngestAuditEventsAsync(batch, o),
ack => new IngestAuditEventsReply(CentralControlDtoMapper.FromIngestAck(ack)));
}
///
public void IngestCachedTelemetry(IngestCachedTelemetryCommand message, IActorRef replyTo)
{
var batch = CentralControlDtoMapper.ToDto(message);
Dispatch(replyTo, SiteStreamGrpcServer.AuditIngestAskTimeout,
(c, o) => c.IngestCachedTelemetryAsync(batch, o),
ack => new IngestCachedTelemetryReply(CentralControlDtoMapper.FromIngestAck(ack)));
}
///
public void ReconcileSite(ReconcileSiteRequest message, IActorRef replyTo)
{
var dto = CentralControlDtoMapper.ToDto(message);
Dispatch(replyTo, _options.QueryTimeout,
(c, o) => c.ReconcileSiteAsync(dto, o),
response => CentralControlDtoMapper.FromDto(response));
}
///
public void ReportSiteHealth(SiteHealthReport message, IActorRef replyTo)
{
var dto = CentralControlDtoMapper.ToDto(message);
Dispatch(replyTo, _options.HealthReportTimeout,
(c, o) => c.ReportSiteHealthAsync(dto, o),
ack => CentralControlDtoMapper.FromDto(ack));
}
///
public void SendHeartbeat(HeartbeatMessage message, IActorRef self)
{
_ = SendHeartbeatAsync(message);
}
private async Task SendHeartbeatAsync(HeartbeatMessage message)
{
var (index, client) = _channels.Current();
var dto = CentralControlDtoMapper.ToDto(message);
try
{
var options = new CallOptions(deadline: DateTime.UtcNow.Add(_options.HealthReportTimeout));
using var call = client.HeartbeatAsync(dto, options);
await call.ResponseAsync.ConfigureAwait(false);
}
catch (RpcException ex) when (IsConnectFailure(ex))
{
// Nudge the pair so the next call tries the peer, but never fault: a heartbeat
// failure must not surface on the site's heartbeat timer path.
_channels.ReportUnavailable(index);
_logger.LogDebug(ex, "Heartbeat to central endpoint {Endpoint} was unavailable.", CurrentEndpointSafe());
}
catch (Exception ex)
{
_logger.LogDebug(ex, "Heartbeat to central failed (swallowed — heartbeats are fire-and-forget).");
}
}
///
/// Runs a unary RPC on the current channel, delivers the decoded reply to
/// , and applies the sticky-failover / no-retry-on-deadline policy.
///
private void Dispatch(
IActorRef replyTo,
TimeSpan timeout,
Func> call,
Func decode)
{
_ = DispatchAsync(replyTo, timeout, call, decode);
}
private async Task DispatchAsync(
IActorRef replyTo,
TimeSpan timeout,
Func> call,
Func decode)
{
var (index, client) = _channels.Current();
try
{
var reply = await InvokeAsync(client, timeout, call).ConfigureAwait(false);
replyTo.Tell(decode(reply));
}
catch (RpcException ex) when (IsConnectFailure(ex))
{
// Provably unsent: the connection was refused / the node was not ready. Fail over
// to the peer and retry ONCE. This is the only status we retry across nodes.
_channels.ReportUnavailable(index);
var (retryIndex, retryClient) = _channels.Current();
if (retryIndex != index)
{
try
{
var reply = await InvokeAsync(retryClient, timeout, call).ConfigureAwait(false);
replyTo.Tell(decode(reply));
return;
}
catch (Exception retryEx)
{
_logger.LogWarning(retryEx,
"Central control-plane call failed on both endpoints; surfacing as transient.");
replyTo.Tell(new AkkaStatus.Failure(retryEx));
return;
}
}
replyTo.Tell(new AkkaStatus.Failure(ex));
}
catch (Exception ex)
{
// DeadlineExceeded / Internal / PermissionDenied / a PSK-resolution throw — do NOT
// retry across nodes (the call may have run). Surface as the transient failure the
// layer above already tolerates.
replyTo.Tell(new AkkaStatus.Failure(ex));
}
}
private static async Task InvokeAsync(
CentralControlService.CentralControlServiceClient client,
TimeSpan timeout,
Func> call)
{
var options = new CallOptions(deadline: DateTime.UtcNow.Add(timeout));
using var asyncCall = call(client, options);
return await asyncCall.ResponseAsync.ConfigureAwait(false);
}
///
/// A failure that provably never reached a server — the only class safe to retry on the peer.
/// is deliberately excluded (the call may have run).
///
///
/// Two shapes qualify: a server-signalled (e.g. a node
/// that returns Unavailable while it is still starting), and a client-side failure to even
/// start the call — Grpc.Net surfaces a refused/failed connection as
/// "Error starting gRPC call" with the transport exception
/// attached, and there the request never left the client. Anything else — including a deadline,
/// a permission denial, or a generic server-side Internal after the call reached the server —
/// is NOT retried across nodes.
///
private static bool IsConnectFailure(RpcException ex)
{
if (ex.StatusCode == StatusCode.Unavailable)
{
return true;
}
// "Error starting gRPC call" is Grpc.Net's marker for a call that could not be sent — a
// refused/failed connection carrying an HttpRequestException. Provably unsent.
return ex.StatusCode == StatusCode.Internal
&& (ex.Status.DebugException is HttpRequestException
|| ex.Status.Detail.StartsWith("Error starting gRPC call", StringComparison.Ordinal));
}
private string CurrentEndpointSafe()
{
try
{
return _channels.CurrentEndpoint;
}
catch
{
return "(unknown)";
}
}
}