Files
ScadaBridge/src/ZB.MOM.WW.ScadaBridge.Communication/Grpc/GrpcSiteTransport.cs
T
Joseph Doherty 86ad4d5c8e feat(comm): T1B.3 — central-side gRPC site-command transport seam (default Akka)
Extract the central→site send path in CentralCommunicationActor behind a new
ISiteCommandTransport, selected by ScadaBridge:Communication:SiteTransport
(Akka | Grpc, default Akka — rollback = flip the flag). CommunicationService's
27 commands, SiteCallAuditActor's 2 parked relays and DebugStreamBridgeActor's
subscribe/unsubscribe are untouched; the seam sits below SiteEnvelope.

- AkkaSiteTransport: today's per-site ClusterClient path extracted verbatim
  (the _siteClients lookup + ClusterClient.Send with the reply-to sender
  preserved, and the "no client ⇒ warn + drop, caller's Ask times out" path).
- GrpcSiteTransport: dials the site SiteCommandService (T1B.1 proto client) via
  SiteCommandDtoMapper, PSK + x-scadabridge-site on the channel through
  ControlPlaneCredentials, per-command deadlines set EQUAL to today's
  CommunicationService Ask timeouts (per-command, not per-group: DeploymentState
  query and TriggerSiteFailover use QueryTimeout; the two parked relays map to
  QueryTimeout so SiteCallAudit's inner RelayTimeout 10s < 30s ordering holds;
  WaitForAttribute keeps its dynamic Timeout + IntegrationTimeout).
- SitePairChannelProvider: per-site A/B channel pair with sticky failover
  (flip only on Unavailable — NEVER on DeadlineExceeded, a write/deploy/failover
  may have run), background failback probe to the preferred node with 1s→60s
  doubling backoff, PSK invalidation on site removal. Fed by the SAME DB refresh
  loop (extended to carry GrpcNodeA/GrpcNodeBAddress) — no second poll.

Tests: actor-with-substitute-transport (routing, Ask-reply plumbing, per-site
lifecycle across refreshes), ResolveDeadline pinned to each command's current
Ask timeout, and GrpcSiteTransport/SitePairChannelProvider over dual in-process
TestServers (PSK+header+deadline attached, Unavailable failover + stickiness,
failback to preferred, no-retry-on-DeadlineExceeded). Proto csproj untouched
(no active <Protobuf> item). Full solution builds 0 warnings; Communication.Tests
607 green with Akka default.
2026-07-22 20:09:43 -04:00

245 lines
11 KiB
C#

using Akka.Actor;
using Grpc.Net.Client;
using Microsoft.Extensions.Logging;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.InboundApi;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Management;
using ZB.MOM.WW.ScadaBridge.Commons.Messages.RemoteQuery;
using ZB.MOM.WW.ScadaBridge.Communication.Actors;
namespace ZB.MOM.WW.ScadaBridge.Communication.Grpc;
/// <summary>
/// The gRPC central→site command transport: encodes each <see cref="SiteEnvelope"/> command with
/// <see cref="SiteCommandDtoMapper"/>, dials the site's <c>SiteCommandService</c> through the sticky
/// <see cref="SitePairChannelProvider"/> (PSK + <c>x-scadabridge-site</c> already on the channel),
/// and routes the decoded reply back to the waiting <c>Ask</c> (or the debug-bridge actor).
/// </summary>
/// <remarks>
/// <para>
/// <b>Per-call deadlines match today's Ask timeouts exactly</b> — see <see cref="ResolveDeadline"/>.
/// Behaviour is otherwise unchanged from the Akka path: an unknown/unconfigured site is warned and
/// dropped (the caller's Ask times out), and a transport fault surfaces to the caller as a
/// <see cref="Status.Failure"/>, which the S&amp;F/audit layers already treat as transient.
/// </para>
/// <para>
/// <b>Cross-node retry is the channel provider's job</b> and happens only on
/// <c>Unavailable</c> — never on <c>DeadlineExceeded</c> (a write/deploy/failover may have run).
/// </para>
/// </remarks>
public sealed class GrpcSiteTransport : ISiteCommandTransport
{
private readonly SitePairChannelProvider _channels;
private readonly CommunicationOptions _options;
private readonly ILogger<GrpcSiteTransport> _logger;
// Sites this transport has pushed into the channel provider — used to diff removals per refresh,
// the gRPC analogue of the Akka transport's _siteClients key set. Touched only on the actor thread.
private readonly HashSet<string> _knownSites = new(StringComparer.Ordinal);
/// <summary>Creates the gRPC transport and ensures the provider's failback loop is running.</summary>
/// <param name="channels">The shared per-site channel-pair provider.</param>
/// <param name="options">Communication options supplying the per-command deadlines.</param>
/// <param name="logger">Logger for drop/fault diagnostics.</param>
public GrpcSiteTransport(
SitePairChannelProvider channels,
CommunicationOptions options,
ILogger<GrpcSiteTransport> logger)
{
_channels = channels ?? throw new ArgumentNullException(nameof(channels));
_options = options ?? throw new ArgumentNullException(nameof(options));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_channels.EnsureFailbackLoop();
}
/// <inheritdoc />
public void Send(SiteEnvelope envelope, IActorRef replyTo)
{
ArgumentNullException.ThrowIfNull(envelope);
// UnsubscribeDebugView is the one fire-and-forget command: the site never acks it over Akka
// (the debug bridge Tells and expects nothing), so we run the RPC but do not deliver its
// synthetic ack — mirroring today's no-reply behaviour and avoiding a dead-lettered ack.
var fireAndForget = envelope.Message is UnsubscribeDebugViewRequest;
_ = RunAsync(envelope, replyTo, fireAndForget);
}
private async Task RunAsync(SiteEnvelope envelope, IActorRef replyTo, bool fireAndForget)
{
try
{
var reply = await SendCoreAsync(envelope, CancellationToken.None).ConfigureAwait(false);
if (!fireAndForget && !replyTo.IsNobody())
{
replyTo.Tell(reply, ActorRefs.NoSender);
}
}
catch (SiteChannelUnavailableException)
{
// Parity with the Akka "no ClusterClient for site" path: warn and drop, so the caller's
// Ask times out. Central never buffers.
_logger.LogWarning(
"No gRPC channel for site {SiteId}; dropping {Message} (caller's Ask will time out)",
envelope.SiteId, envelope.Message.GetType().Name);
}
catch (Exception ex)
{
if (!fireAndForget && !replyTo.IsNobody())
{
// A timeout or non-OK status faults the caller's Ask exactly as an Akka Ask timeout
// did — the S&F/audit layers already treat that as transient.
replyTo.Tell(new Status.Failure(ex), ActorRefs.NoSender);
}
else
{
_logger.LogWarning(ex,
"Fire-and-forget {Message} to site {SiteId} faulted",
envelope.Message.GetType().Name, envelope.SiteId);
}
}
}
private Task<object> SendCoreAsync(SiteEnvelope envelope, CancellationToken ct)
{
var message = envelope.Message;
// Compute the absolute deadline ONCE so a cross-node failover retry shares the overall
// budget rather than restarting it.
var deadline = DateTime.UtcNow + ResolveDeadline(message);
var group = SiteCommandDtoMapper.GroupOf(message);
return _channels.ExecuteAsync(
envelope.SiteId,
(channel, callCt) => InvokeAsync(channel, group, message, deadline, callCt),
ct);
}
private static async Task<object> InvokeAsync(
GrpcChannel channel, SiteCommandGroup group, object message, DateTime deadline, CancellationToken ct)
{
var client = new SiteCommandService.SiteCommandServiceClient(channel);
switch (group)
{
case SiteCommandGroup.Lifecycle:
{
var reply = await client.ExecuteLifecycleAsync(
SiteCommandDtoMapper.ToLifecycleRequest(message),
deadline: deadline, cancellationToken: ct);
return SiteCommandDtoMapper.FromLifecycleReply(reply);
}
case SiteCommandGroup.OpcUa:
{
var reply = await client.ExecuteOpcUaAsync(
SiteCommandDtoMapper.ToOpcUaRequest(message),
deadline: deadline, cancellationToken: ct);
return SiteCommandDtoMapper.FromOpcUaReply(reply);
}
case SiteCommandGroup.Query:
{
var reply = await client.ExecuteQueryAsync(
SiteCommandDtoMapper.ToQueryRequest(message),
deadline: deadline, cancellationToken: ct);
return SiteCommandDtoMapper.FromQueryReply(reply);
}
case SiteCommandGroup.Parked:
{
var reply = await client.ExecuteParkedAsync(
SiteCommandDtoMapper.ToParkedRequest(message),
deadline: deadline, cancellationToken: ct);
return SiteCommandDtoMapper.FromParkedReply(reply);
}
case SiteCommandGroup.Route:
{
var reply = await client.ExecuteRouteAsync(
SiteCommandDtoMapper.ToRouteRequest(message),
deadline: deadline, cancellationToken: ct);
return SiteCommandDtoMapper.FromRouteReply(reply);
}
case SiteCommandGroup.Failover:
{
var ack = await client.TriggerFailoverAsync(
SiteCommandDtoMapper.ToProto((TriggerSiteFailover)message),
deadline: deadline, cancellationToken: ct);
return SiteCommandDtoMapper.FromProto(ack);
}
default:
throw new ArgumentOutOfRangeException(nameof(group), group, "Unknown site command group.");
}
}
/// <summary>
/// The per-command deadline, set EQUAL to the Ask timeout <c>CommunicationService</c> uses for
/// that command today, so flipping the transport changes nothing about how long a call waits.
/// Note the group is NOT a uniform deadline class: within Lifecycle, <c>DeploymentStateQuery</c>
/// uses <c>QueryTimeout</c> (not <c>LifecycleTimeout</c>), and <c>TriggerSiteFailover</c> uses
/// <c>QueryTimeout</c> (not <c>LifecycleTimeout</c>) — matching the real Ask sites, not the
/// plan's per-group table. The two parked relays (<c>RetryParkedOperation</c>/
/// <c>DiscardParkedOperation</c>) map to <c>QueryTimeout</c> (30s), preserving the
/// <c>SiteCallAuditActor</c> inner <c>RelayTimeout</c> (10s) &lt; 30s ordering. WaitForAttribute
/// keeps its dynamic <c>request.Timeout + IntegrationTimeout</c> budget.
/// </summary>
/// <param name="message">The command being sent.</param>
/// <returns>The deadline duration for that command.</returns>
internal TimeSpan ResolveDeadline(object message) => message switch
{
RefreshDeploymentCommand => _options.DeploymentTimeout,
EnableInstanceCommand or DisableInstanceCommand or DeleteInstanceCommand => _options.LifecycleTimeout,
DeploymentStateQueryRequest => _options.QueryTimeout,
DeployArtifactsCommand => _options.ArtifactDeploymentTimeout,
BrowseNodeCommand or SearchAddressSpaceCommand or ReadTagValuesCommand or VerifyEndpointCommand
or TrustServerCertCommand or ListServerCertsCommand or RemoveServerCertCommand or WriteTagRequest
=> _options.QueryTimeout,
EventLogQueryRequest or DebugSnapshotRequest => _options.QueryTimeout,
SubscribeDebugViewRequest or UnsubscribeDebugViewRequest => _options.DebugViewTimeout,
ParkedMessageQueryRequest or ParkedMessageRetryRequest or ParkedMessageDiscardRequest
or RetryParkedOperation or DiscardParkedOperation
=> _options.QueryTimeout,
RouteToCallRequest or RouteToGetAttributesRequest or RouteToSetAttributesRequest
=> _options.IntegrationTimeout,
RouteToWaitForAttributeRequest r => r.Timeout + _options.IntegrationTimeout,
TriggerSiteFailover => _options.QueryTimeout,
_ => throw new ArgumentException(
$"'{message.GetType().Name}' is not a migrated site command.", nameof(message))
};
/// <inheritdoc />
public void ReconcileSites(SiteAddressCacheLoaded cache)
{
ArgumentNullException.ThrowIfNull(cache);
var desired = cache.GrpcContacts;
// Drop sites that lost their gRPC endpoints (or were deleted).
foreach (var removed in _knownSites.Where(s => !desired.ContainsKey(s)).ToList())
{
_channels.RemoveSite(removed);
_knownSites.Remove(removed);
}
// Create/refresh the rest.
foreach (var (siteId, endpoints) in desired)
{
_channels.UpdateSite(siteId, endpoints.NodeA, endpoints.NodeB);
_knownSites.Add(siteId);
}
}
}