feat(grpc): site-side ICentralTransport seam + gRPC transport (T1A.3)
Introduce ICentralTransport as the site->central choke point inside SiteCommunicationActor. The seven site->central sends (notification submit/ status, audit + cached-telemetry ingest, reconcile, health, heartbeat) now delegate to an injected transport instead of owning ClusterClient.Send inline. - AkkaCentralTransport: verbatim extraction of today's ClusterClient.Send path, including the exact sender-forwarding that routes central's reply straight back to the waiting Ask. Default when no transport is injected -> behaviour unchanged, existing SiteCommunicationActorTests pass as-is. - GrpcCentralTransport + CentralChannelProvider: dial CentralControlService with sticky failover + background failback (1s-doubling-cap-60s), PSK + x-scadabridge-site via ControlPlaneCredentials, per-call deadlines mirroring today's Ask timeouts. Cross-node retry ONLY on provably-unsent connect failures; never on DeadlineExceeded. Heartbeat stays fire-and-forget. - StaticSitePskProvider: site's single own-key provider (fail-closed). - CommunicationOptions: CentralTransport flag (default Akka) + CentralGrpcEndpoints (validator: required when transport=Grpc). Host selects the impl; the ClusterClient is created only on the Akka path. Tests: actor-with-fake-transport (7 delegations + fault routing + heartbeat no-fault), AkkaCentralTransport sender-forwarding, GrpcCentralTransport over in-process TestServer (failover flip, sticky, failback, PSK+header, deadline, no-retry-on-deadline), validator. Communication.Tests 371 green, Host.Tests 391 green; the three above-seam suites pass unmodified.
This commit is contained in:
Joseph Doherty
@@ -0,0 +1,274 @@
|
||||
using Google.Protobuf.WellKnownTypes;
|
||||
using Grpc.Core;
|
||||
using Grpc.Net.Client;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// A pair (or more) of gRPC channels to the central cluster's control-plane nodes, with the
|
||||
/// sticky-failover + background-failback policy of the design (§3.5). The site holds one channel
|
||||
/// per central endpoint, prefers the first, and stays on it until a call proves it unreachable —
|
||||
/// only then flipping to the next, and only <see cref="StatusCode.Unavailable"/> / connect
|
||||
/// failures count (a <see cref="StatusCode.DeadlineExceeded"/> never flips or retries, because the
|
||||
/// call may have run).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>Sticky.</b> All calls go to the current channel; a healthy preferred endpoint never
|
||||
/// ping-pongs. <see cref="ReportUnavailable"/> flips to the next endpoint (round-robin) when the
|
||||
/// caller sees the current one refuse a connection.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Failback.</b> While off the preferred endpoint a background probe (a cheap <c>Heartbeat</c>
|
||||
/// ping — always answered, even by a not-yet-ready node) re-checks the preferred one. On the first
|
||||
/// success new calls return to it; in-flight calls finish where they are. The probe is
|
||||
/// event-driven: it arms on a flip and stops the moment we are back on the preferred endpoint, so
|
||||
/// a steady healthy pair spends no cycles. Its cadence backs off exponentially — 1 s, doubling,
|
||||
/// capped at 60 s — while the preferred endpoint stays down, so a genuinely dead node is not
|
||||
/// probed hard.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Auth.</b> Every channel carries the site's own preshared key and its
|
||||
/// <c>x-scadabridge-site</c> identity via <see cref="ControlPlaneCredentials"/> — the same
|
||||
/// insecure-h2c call-credentials shape the streaming client uses. The <paramref name="handlerFactory"/>
|
||||
/// seam lets a test point a channel at an in-process <c>TestServer</c>; production uses a
|
||||
/// keepalive-configured <see cref="SocketsHttpHandler"/>.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class CentralChannelProvider : IDisposable
|
||||
{
|
||||
private static readonly TimeSpan DefaultBackoffBase = TimeSpan.FromSeconds(1);
|
||||
private static readonly TimeSpan DefaultBackoffCap = TimeSpan.FromSeconds(60);
|
||||
private static readonly TimeSpan DefaultProbeDeadline = TimeSpan.FromSeconds(5);
|
||||
|
||||
private readonly IReadOnlyList<string> _endpoints;
|
||||
private readonly GrpcChannel[] _channels;
|
||||
private readonly CentralControlService.CentralControlServiceClient[] _clients;
|
||||
private readonly ILogger _logger;
|
||||
private readonly string _siteId;
|
||||
private readonly TimeSpan _backoffBase;
|
||||
private readonly TimeSpan _backoffCap;
|
||||
private readonly TimeSpan _probeDeadline;
|
||||
private readonly Timer? _failbackTimer;
|
||||
private readonly object _gate = new();
|
||||
|
||||
private volatile int _current; // preferred == 0
|
||||
private int _consecutiveProbeFailures;
|
||||
private bool _disposed;
|
||||
|
||||
/// <summary>Creates the provider and opens one channel per endpoint.</summary>
|
||||
/// <param name="endpoints">Central control-plane endpoints, preferred first (index 0). Must be non-empty.</param>
|
||||
/// <param name="pskProvider">Resolves this site's preshared key (site-side: a single-key provider).</param>
|
||||
/// <param name="siteId">This site's identity, sent as the <c>x-scadabridge-site</c> header.</param>
|
||||
/// <param name="options">Communication options supplying gRPC keepalive settings.</param>
|
||||
/// <param name="logger">Logger for flip/failback diagnostics.</param>
|
||||
/// <param name="handlerFactory">Test seam: per-endpoint <see cref="HttpMessageHandler"/>; null uses a production socket handler.</param>
|
||||
/// <param name="probeDeadline">Deadline for a failback probe. Null uses 5 s.</param>
|
||||
/// <param name="backoffBase">Initial failback-probe backoff. Null uses 1 s.</param>
|
||||
/// <param name="backoffCap">Maximum failback-probe backoff. Null uses 60 s.</param>
|
||||
public CentralChannelProvider(
|
||||
IReadOnlyList<string> endpoints,
|
||||
ISitePskProvider pskProvider,
|
||||
string siteId,
|
||||
CommunicationOptions options,
|
||||
ILogger logger,
|
||||
Func<string, HttpMessageHandler>? handlerFactory = null,
|
||||
TimeSpan? probeDeadline = null,
|
||||
TimeSpan? backoffBase = null,
|
||||
TimeSpan? backoffCap = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(endpoints);
|
||||
ArgumentNullException.ThrowIfNull(pskProvider);
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(siteId);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
if (endpoints.Count == 0)
|
||||
{
|
||||
throw new ArgumentException("At least one central gRPC endpoint is required.", nameof(endpoints));
|
||||
}
|
||||
|
||||
_endpoints = endpoints;
|
||||
_logger = logger;
|
||||
_siteId = siteId;
|
||||
_backoffBase = backoffBase ?? DefaultBackoffBase;
|
||||
_backoffCap = backoffCap ?? DefaultBackoffCap;
|
||||
_probeDeadline = probeDeadline ?? DefaultProbeDeadline;
|
||||
|
||||
_channels = new GrpcChannel[endpoints.Count];
|
||||
_clients = new CentralControlService.CentralControlServiceClient[endpoints.Count];
|
||||
for (var i = 0; i < endpoints.Count; i++)
|
||||
{
|
||||
var channelOptions = new GrpcChannelOptions
|
||||
{
|
||||
HttpHandler = handlerFactory?.Invoke(endpoints[i]) ?? new SocketsHttpHandler
|
||||
{
|
||||
KeepAlivePingDelay = options.GrpcKeepAlivePingDelay,
|
||||
KeepAlivePingTimeout = options.GrpcKeepAlivePingTimeout,
|
||||
KeepAlivePingPolicy = HttpKeepAlivePingPolicy.Always,
|
||||
EnableMultipleHttp2Connections = true,
|
||||
},
|
||||
}.WithSiteCredentials(pskProvider, siteId);
|
||||
|
||||
_channels[i] = GrpcChannel.ForAddress(endpoints[i], channelOptions);
|
||||
_clients[i] = new CentralControlService.CentralControlServiceClient(_channels[i]);
|
||||
}
|
||||
|
||||
// Only a multi-endpoint pair can ever fail over, so a lone endpoint needs no probe.
|
||||
if (endpoints.Count > 1)
|
||||
{
|
||||
_failbackTimer = new Timer(_ => _ = FailbackTickAsync(), null, Timeout.Infinite, Timeout.Infinite);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>The number of endpoints in the pair.</summary>
|
||||
public int EndpointCount => _endpoints.Count;
|
||||
|
||||
/// <summary>The index of the endpoint calls are currently routed to (preferred == 0).</summary>
|
||||
public int CurrentIndex => _current;
|
||||
|
||||
/// <summary>The endpoint address calls are currently routed to.</summary>
|
||||
public string CurrentEndpoint => _endpoints[_current];
|
||||
|
||||
/// <summary>
|
||||
/// The endpoint index and client calls should use right now. Captured together so a caller can
|
||||
/// tell <see cref="ReportUnavailable"/> exactly which endpoint failed even if a concurrent flip
|
||||
/// has already moved <see cref="CurrentIndex"/>.
|
||||
/// </summary>
|
||||
/// <returns>The current endpoint index and its client.</returns>
|
||||
public (int Index, CentralControlService.CentralControlServiceClient Client) Current()
|
||||
{
|
||||
var idx = _current;
|
||||
return (idx, _clients[idx]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reports that the endpoint at <paramref name="failedIndex"/> refused a connection (an
|
||||
/// <see cref="StatusCode.Unavailable"/> / connect failure). If it is still the current endpoint
|
||||
/// and another exists, flips to the next one and — when now off the preferred endpoint — arms
|
||||
/// the failback probe. Idempotent under a concurrent flip: a stale index is ignored.
|
||||
/// </summary>
|
||||
/// <param name="failedIndex">The endpoint index the caller's failed call used.</param>
|
||||
public void ReportUnavailable(int failedIndex)
|
||||
{
|
||||
if (_endpoints.Count < 2)
|
||||
{
|
||||
return; // nothing to fail over to
|
||||
}
|
||||
|
||||
lock (_gate)
|
||||
{
|
||||
if (_disposed || failedIndex != _current)
|
||||
{
|
||||
return; // a concurrent flip already moved us; do not double-flip
|
||||
}
|
||||
|
||||
var next = (failedIndex + 1) % _endpoints.Count;
|
||||
_current = next;
|
||||
_logger.LogWarning(
|
||||
"Central control-plane endpoint {Failed} is unavailable; site {SiteId} failed over to {Next}.",
|
||||
_endpoints[failedIndex], _siteId, _endpoints[next]);
|
||||
|
||||
if (_current != 0)
|
||||
{
|
||||
_consecutiveProbeFailures = 0;
|
||||
ArmFailback(_backoffBase);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void ArmFailback(TimeSpan due)
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_failbackTimer?.Change(due, Timeout.InfiniteTimeSpan);
|
||||
}
|
||||
|
||||
private async Task FailbackTickAsync()
|
||||
{
|
||||
int currentAtTick = _current;
|
||||
if (_disposed || currentAtTick == 0)
|
||||
{
|
||||
return; // already back on the preferred endpoint (or shutting down)
|
||||
}
|
||||
|
||||
var preferred = _clients[0];
|
||||
try
|
||||
{
|
||||
await preferred.HeartbeatAsync(
|
||||
new HeartbeatDto
|
||||
{
|
||||
SiteId = _siteId,
|
||||
NodeHostname = "failback-probe",
|
||||
IsActive = false,
|
||||
Timestamp = Timestamp.FromDateTimeOffset(DateTimeOffset.UtcNow),
|
||||
},
|
||||
deadline: DateTime.UtcNow.Add(_probeDeadline)).ConfigureAwait(false);
|
||||
|
||||
// The preferred endpoint answered — return new calls to it.
|
||||
lock (_gate)
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_current = 0;
|
||||
_consecutiveProbeFailures = 0;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Central control-plane preferred endpoint {Preferred} is reachable again; site {SiteId} failed back.",
|
||||
_endpoints[0], _siteId);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_disposed || _current == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_consecutiveProbeFailures++;
|
||||
var backoff = NextBackoff(_consecutiveProbeFailures);
|
||||
_logger.LogDebug(ex,
|
||||
"Failback probe of preferred central endpoint {Preferred} failed; re-probing in {Backoff}.",
|
||||
_endpoints[0], backoff);
|
||||
ArmFailback(backoff);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private TimeSpan NextBackoff(int failures)
|
||||
{
|
||||
// 1 s, doubling, capped at 60 s. Guard the shift against overflow for a long outage.
|
||||
var exponent = Math.Min(failures - 1, 20);
|
||||
var scaled = _backoffBase.Ticks * (1L << exponent);
|
||||
var cap = _backoffCap.Ticks;
|
||||
return TimeSpan.FromTicks(scaled >= cap || scaled < 0 ? cap : scaled);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Dispose()
|
||||
{
|
||||
lock (_gate)
|
||||
{
|
||||
if (_disposed)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
_disposed = true;
|
||||
}
|
||||
|
||||
_failbackTimer?.Dispose();
|
||||
foreach (var channel in _channels)
|
||||
{
|
||||
channel.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,258 @@
|
||||
using Akka.Actor;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Audit;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Health;
|
||||
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Notification;
|
||||
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
||||
using AkkaStatus = Akka.Actor.Status;
|
||||
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// The <see cref="ICentralTransport"/> that carries the seven site→central sends over gRPC — the
|
||||
/// migration target for the Akka <c>ClusterClient</c> path. Each method encodes the message with
|
||||
/// <see cref="CentralControlDtoMapper"/>, dials <c>CentralControlService</c> through the sticky
|
||||
/// <see cref="CentralChannelProvider"/>, and delivers the decoded reply (or a transient-failure
|
||||
/// signal) to the waiting Ask.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// The actor handlers are synchronous; each method here kicks off the RPC as a detached task and
|
||||
/// <c>Tell</c>s the result to <paramref name="replyTo"/> when it completes — <c>IActorRef.Tell</c>
|
||||
/// is thread-safe, so the reply lands at the Ask exactly as central's ClusterClient reply did. On
|
||||
/// any non-OK status the reply is <see cref="Status.Failure"/>, which the S&F / audit / health
|
||||
/// layers already treat as transient.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Cross-node retry only on provably-unsent failures.</b> An <see cref="StatusCode.Unavailable"/>
|
||||
/// (connection refused / node not ready) flips the channel pair and retries once on the peer.
|
||||
/// A <see cref="StatusCode.DeadlineExceeded"/> is NEVER retried across nodes — a deploy / write /
|
||||
/// failover may already have executed, and duplicating it is worse than surfacing a transient
|
||||
/// failure the layer above tolerates.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Per-call deadlines mirror today's Ask timeouts.</b> Notification submit/status →
|
||||
/// <c>NotificationForwardTimeout</c> (30 s, the value the S&F forwarder and the central service
|
||||
/// both use); health → <c>HealthReportTimeout</c> (10 s); reconcile → <c>QueryTimeout</c> (30 s);
|
||||
/// both ingest RPCs → <see cref="SiteStreamGrpcServer.AuditIngestAskTimeout"/> (the one shared 30 s
|
||||
/// constant). The heartbeat, fire-and-forget with no server-side Ask, is merely bounded by
|
||||
/// <c>HealthReportTimeout</c> and its failures are swallowed.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class GrpcCentralTransport : ICentralTransport
|
||||
{
|
||||
private readonly CentralChannelProvider _channels;
|
||||
private readonly CommunicationOptions _options;
|
||||
private readonly ILogger<GrpcCentralTransport> _logger;
|
||||
|
||||
/// <summary>Creates the transport over a channel pair.</summary>
|
||||
/// <param name="channels">The sticky central channel pair.</param>
|
||||
/// <param name="options">Communication options supplying the per-call deadlines.</param>
|
||||
/// <param name="logger">Logger for failover/fault diagnostics.</param>
|
||||
public GrpcCentralTransport(
|
||||
CentralChannelProvider channels,
|
||||
CommunicationOptions options,
|
||||
ILogger<GrpcCentralTransport> logger)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(channels);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
ArgumentNullException.ThrowIfNull(logger);
|
||||
|
||||
_channels = channels;
|
||||
_options = options;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SubmitNotification(NotificationSubmit message, IActorRef replyTo)
|
||||
{
|
||||
var dto = CentralControlDtoMapper.ToDto(message);
|
||||
Dispatch(replyTo, _options.NotificationForwardTimeout,
|
||||
(c, o) => c.SubmitNotificationAsync(dto, o),
|
||||
ack => CentralControlDtoMapper.FromDto(ack));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void QueryNotificationStatus(NotificationStatusQuery message, IActorRef replyTo)
|
||||
{
|
||||
var dto = CentralControlDtoMapper.ToDto(message);
|
||||
Dispatch(replyTo, _options.NotificationForwardTimeout,
|
||||
(c, o) => c.QueryNotificationStatusAsync(dto, o),
|
||||
response => CentralControlDtoMapper.FromDto(response));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void IngestAuditEvents(IngestAuditEventsCommand message, IActorRef replyTo)
|
||||
{
|
||||
var batch = CentralControlDtoMapper.ToDto(message);
|
||||
Dispatch(replyTo, SiteStreamGrpcServer.AuditIngestAskTimeout,
|
||||
(c, o) => c.IngestAuditEventsAsync(batch, o),
|
||||
ack => new IngestAuditEventsReply(CentralControlDtoMapper.FromIngestAck(ack)));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void IngestCachedTelemetry(IngestCachedTelemetryCommand message, IActorRef replyTo)
|
||||
{
|
||||
var batch = CentralControlDtoMapper.ToDto(message);
|
||||
Dispatch(replyTo, SiteStreamGrpcServer.AuditIngestAskTimeout,
|
||||
(c, o) => c.IngestCachedTelemetryAsync(batch, o),
|
||||
ack => new IngestCachedTelemetryReply(CentralControlDtoMapper.FromIngestAck(ack)));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void ReconcileSite(ReconcileSiteRequest message, IActorRef replyTo)
|
||||
{
|
||||
var dto = CentralControlDtoMapper.ToDto(message);
|
||||
Dispatch(replyTo, _options.QueryTimeout,
|
||||
(c, o) => c.ReconcileSiteAsync(dto, o),
|
||||
response => CentralControlDtoMapper.FromDto(response));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void ReportSiteHealth(SiteHealthReport message, IActorRef replyTo)
|
||||
{
|
||||
var dto = CentralControlDtoMapper.ToDto(message);
|
||||
Dispatch(replyTo, _options.HealthReportTimeout,
|
||||
(c, o) => c.ReportSiteHealthAsync(dto, o),
|
||||
ack => CentralControlDtoMapper.FromDto(ack));
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void SendHeartbeat(HeartbeatMessage message, IActorRef self)
|
||||
{
|
||||
_ = SendHeartbeatAsync(message);
|
||||
}
|
||||
|
||||
private async Task SendHeartbeatAsync(HeartbeatMessage message)
|
||||
{
|
||||
var (index, client) = _channels.Current();
|
||||
var dto = CentralControlDtoMapper.ToDto(message);
|
||||
try
|
||||
{
|
||||
var options = new CallOptions(deadline: DateTime.UtcNow.Add(_options.HealthReportTimeout));
|
||||
using var call = client.HeartbeatAsync(dto, options);
|
||||
await call.ResponseAsync.ConfigureAwait(false);
|
||||
}
|
||||
catch (RpcException ex) when (IsConnectFailure(ex))
|
||||
{
|
||||
// Nudge the pair so the next call tries the peer, but never fault: a heartbeat
|
||||
// failure must not surface on the site's heartbeat timer path.
|
||||
_channels.ReportUnavailable(index);
|
||||
_logger.LogDebug(ex, "Heartbeat to central endpoint {Endpoint} was unavailable.", CurrentEndpointSafe());
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogDebug(ex, "Heartbeat to central failed (swallowed — heartbeats are fire-and-forget).");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Runs a unary RPC on the current channel, delivers the decoded reply to
|
||||
/// <paramref name="replyTo"/>, and applies the sticky-failover / no-retry-on-deadline policy.
|
||||
/// </summary>
|
||||
private void Dispatch<TWire>(
|
||||
IActorRef replyTo,
|
||||
TimeSpan timeout,
|
||||
Func<CentralControlService.CentralControlServiceClient, CallOptions, AsyncUnaryCall<TWire>> call,
|
||||
Func<TWire, object> decode)
|
||||
{
|
||||
_ = DispatchAsync(replyTo, timeout, call, decode);
|
||||
}
|
||||
|
||||
private async Task DispatchAsync<TWire>(
|
||||
IActorRef replyTo,
|
||||
TimeSpan timeout,
|
||||
Func<CentralControlService.CentralControlServiceClient, CallOptions, AsyncUnaryCall<TWire>> call,
|
||||
Func<TWire, object> decode)
|
||||
{
|
||||
var (index, client) = _channels.Current();
|
||||
try
|
||||
{
|
||||
var reply = await InvokeAsync(client, timeout, call).ConfigureAwait(false);
|
||||
replyTo.Tell(decode(reply));
|
||||
}
|
||||
catch (RpcException ex) when (IsConnectFailure(ex))
|
||||
{
|
||||
// Provably unsent: the connection was refused / the node was not ready. Fail over
|
||||
// to the peer and retry ONCE. This is the only status we retry across nodes.
|
||||
_channels.ReportUnavailable(index);
|
||||
var (retryIndex, retryClient) = _channels.Current();
|
||||
if (retryIndex != index)
|
||||
{
|
||||
try
|
||||
{
|
||||
var reply = await InvokeAsync(retryClient, timeout, call).ConfigureAwait(false);
|
||||
replyTo.Tell(decode(reply));
|
||||
return;
|
||||
}
|
||||
catch (Exception retryEx)
|
||||
{
|
||||
_logger.LogWarning(retryEx,
|
||||
"Central control-plane call failed on both endpoints; surfacing as transient.");
|
||||
replyTo.Tell(new AkkaStatus.Failure(retryEx));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
replyTo.Tell(new AkkaStatus.Failure(ex));
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
// DeadlineExceeded / Internal / PermissionDenied / a PSK-resolution throw — do NOT
|
||||
// retry across nodes (the call may have run). Surface as the transient failure the
|
||||
// layer above already tolerates.
|
||||
replyTo.Tell(new AkkaStatus.Failure(ex));
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<TWire> InvokeAsync<TWire>(
|
||||
CentralControlService.CentralControlServiceClient client,
|
||||
TimeSpan timeout,
|
||||
Func<CentralControlService.CentralControlServiceClient, CallOptions, AsyncUnaryCall<TWire>> call)
|
||||
{
|
||||
var options = new CallOptions(deadline: DateTime.UtcNow.Add(timeout));
|
||||
using var asyncCall = call(client, options);
|
||||
return await asyncCall.ResponseAsync.ConfigureAwait(false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A failure that provably never reached a server — the only class safe to retry on the peer.
|
||||
/// <see cref="StatusCode.DeadlineExceeded"/> is deliberately excluded (the call may have run).
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Two shapes qualify: a server-signalled <see cref="StatusCode.Unavailable"/> (e.g. a node
|
||||
/// that returns Unavailable while it is still starting), and a client-side failure to even
|
||||
/// start the call — Grpc.Net surfaces a refused/failed connection as
|
||||
/// <see cref="StatusCode.Internal"/> "Error starting gRPC call" with the transport exception
|
||||
/// attached, and there the request never left the client. Anything else — including a deadline,
|
||||
/// a permission denial, or a generic server-side Internal after the call reached the server —
|
||||
/// is NOT retried across nodes.
|
||||
/// </remarks>
|
||||
private static bool IsConnectFailure(RpcException ex)
|
||||
{
|
||||
if (ex.StatusCode == StatusCode.Unavailable)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// "Error starting gRPC call" is Grpc.Net's marker for a call that could not be sent — a
|
||||
// refused/failed connection carrying an HttpRequestException. Provably unsent.
|
||||
return ex.StatusCode == StatusCode.Internal
|
||||
&& (ex.Status.DebugException is HttpRequestException
|
||||
|| ex.Status.Detail.StartsWith("Error starting gRPC call", StringComparison.Ordinal));
|
||||
}
|
||||
|
||||
private string CurrentEndpointSafe()
|
||||
{
|
||||
try
|
||||
{
|
||||
return _channels.CurrentEndpoint;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return "(unknown)";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// Site-side <see cref="ISitePskProvider"/> over a single fixed key — the one key a site node
|
||||
/// presents on every control-plane call it makes to central (<c>CommunicationOptions.GrpcPsk</c>).
|
||||
/// Central's provider resolves a key <em>per site</em>; a site has exactly one, so it ignores the
|
||||
/// requested <c>siteId</c> and returns its own key.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <b>Fail-closed.</b> An empty key throws, matching the contract on <see cref="ISitePskProvider"/>
|
||||
/// and the interceptor's own posture: a node shipped without a key must not degrade to an
|
||||
/// unauthenticated dial.
|
||||
/// </remarks>
|
||||
public sealed class StaticSitePskProvider : ISitePskProvider
|
||||
{
|
||||
private readonly string _key;
|
||||
|
||||
/// <summary>Creates the provider bound to a site's own preshared key.</summary>
|
||||
/// <param name="key">The site's <c>GrpcPsk</c>. Empty is permitted at construction but throws on use.</param>
|
||||
public StaticSitePskProvider(string key)
|
||||
{
|
||||
_key = key ?? string.Empty;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public ValueTask<string> GetAsync(string siteId, CancellationToken ct)
|
||||
{
|
||||
if (string.IsNullOrEmpty(_key))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"No gRPC preshared key is configured for this site (ScadaBridge:Communication:GrpcPsk). "
|
||||
+ "The control plane is fail-closed: an unauthenticated dial does not happen.");
|
||||
}
|
||||
|
||||
return new ValueTask<string>(_key);
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public void Invalidate(string siteId)
|
||||
{
|
||||
// A single static key never changes for the process lifetime; nothing to drop.
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user