using System.Collections.Concurrent;
using Grpc.Core;
using Grpc.Net.Client;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1;
namespace ZB.MOM.WW.OtOpcUa.Runtime.Communication;
///
/// Posts this node's to central's CentralCommandService as a unary
/// gRPC call (mesh gRPC command plane, Task 1B.2), the MeshTransport:Mode = Grpc
/// cross-mesh ack transport.
///
///
///
/// One ordered pass, stopping at the first success.
/// is an availability list, not a
/// fan-out list — the single-target semantic this leg owes (one ack per node per dispatch,
/// deliberately not SendToAll).
///
///
/// Delivery is at-most-once per SUCCESSFUL attempt, but the walk as a whole is
/// at-least-once, and that is fine. An endpoint that processes the ack and then exceeds
/// its deadline before answering is indistinguishable here from one that never received it, so
/// the next endpoint is tried and central may see the ack twice.
/// ConfigPublishCoordinator.HandleAck absorbs that idempotently: acks land in a
/// NodeId-keyed dictionary behind an expected-set membership guard and a
/// stale-deployment guard, so a duplicate overwrites this node's own entry rather than
/// inflating the count, and the row upsert behind it is idempotent too. Duplicates therefore
/// cannot seal a deployment other nodes never applied — the single-endpoint rule is defence in
/// depth and a bound on wasted work, not a correctness dependency.
///
///
/// No retry beyond that single pass, and no queue. The transport buffers nothing
/// and retried nothing, and this is a like-for-like replacement, not an upgrade to delivery
/// guarantees. The deploy path's own apply deadline stays the only recovery mechanism; a
/// retry loop here would instead pile acks up behind an unreachable central and deliver a
/// burst of stale ones when it returned.
///
///
/// All endpoints failing is a DROP — but a loud one. That is the same outcome the
/// node has when it cannot reach central; what changes is that the operator
/// now gets the node, the deployment, the correlation id and the status each endpoint
/// answered with, which is what distinguishes "central is down" from "central rejected this
/// node's key".
///
///
/// The bearer never reaches a log. It travels in the call headers on every attempt and
/// the failure log names endpoints and status codes only — a per-attempt dump of the
/// would write the cluster's pre-shared key into the node log of every
/// deployment that could not reach central.
///
///
/// The cluster id comes from this node's own cluster-<id> Akka role, read
/// through off the same bound Cluster:Roles that
/// ClusterRoleInfo reads (the OTOPCUA_ROLES fallback is already applied to those
/// options by AddOtOpcUaCluster's PostConfigure, so nothing is re-resolved
/// here). The two must agree: central grades the bearer against the key of the cluster named
/// in the header, so a node that authenticated as a different cluster than it gossips would be
/// rejected on every ack.
///
///
public sealed class GrpcApplyAckSender : IApplyAckSender, IDisposable
{
/// The cluster the caller claims to be; central grades the bearer against ITS key.
private const string ClusterHeader = "x-otopcua-cluster";
private readonly MeshTransportOptions _options;
private readonly string? _clusterId;
private readonly ILogger _log;
private readonly Func _clientFactory;
private readonly ConcurrentDictionary _channels = new(StringComparer.Ordinal);
/// Initializes a new instance of the class.
/// Mesh-transport options; supplies the endpoints, the PSK and the deadline.
///
/// Cluster options, read only for this node's roles — the cluster-<id> one names
/// the cluster this node authenticates as.
///
/// Logger.
///
/// Seam for tests: builds the generated client for an endpoint. Defaults to a real
/// (one cached per endpoint).
///
public GrpcApplyAckSender(
IOptions options,
IOptions clusterOptions,
ILogger log,
Func? clientFactory = null)
{
ArgumentNullException.ThrowIfNull(options);
ArgumentNullException.ThrowIfNull(clusterOptions);
ArgumentNullException.ThrowIfNull(log);
_options = options.Value;
_log = log;
_clusterId = RoleParser.ClusterIdFromRoles(clusterOptions.Value.Roles);
_clientFactory = clientFactory ?? DefaultClientFactory;
if (_clusterId is null)
{
// Said once, at construction, rather than on every ack. Central cannot grade a bearer it
// has no cluster for, so every ack this node posts will come back PermissionDenied — and
// without this line that reads as a key mismatch rather than a missing role.
_log.LogWarning(
"This node carries no '{Prefix}' role, so its deployment acks will be "
+ "posted without the {Header} header and central will reject every one of them. Add "
+ "the cluster-scoped role to Cluster:Roles (or OTOPCUA_ROLES).",
RoleParser.ClusterRolePrefix, ClusterHeader);
}
}
///
public async Task SendAsync(ApplyAck ack, CancellationToken cancellationToken = default)
{
try
{
// INSIDE the try, not before it. This is an async method, so a guard clause above the try
// does not throw to the caller — it faults the returned task, which the caller discards.
// That would break this method's "never faults" promise in the one case a caller could
// never see.
ArgumentNullException.ThrowIfNull(ack);
await PostAsync(ack, cancellationToken);
}
catch (Exception ex)
{
// Deliberately unbounded, and deliberately not a rethrow. The caller discards this task
// (see IApplyAckSender), so anything escaping would be swallowed by the runtime and
// resurface as an unobserved exception at GC — a drop with no line in the node log, which
// is precisely the failure mode this transport replaces. Encoding faults and non-RpcException
// transport faults both land here.
//
// The ack is null-conditional because the guard above is one of the things this catches.
_log.LogWarning(
ex,
"Dropping the ApplyAck for {DeploymentId} (correlation {CorrelationId}) from {NodeId}: "
+ "posting it to central failed unexpectedly. It is not buffered and will not be "
+ "retried, so the deployment will time out at central naming this node even though it "
+ "applied successfully.",
ack?.DeploymentId, ack?.CorrelationId, ack?.NodeId);
}
}
private async Task PostAsync(ApplyAck ack, CancellationToken cancellationToken)
{
var endpoints = _options.CentralGrpcEndpoints;
if (endpoints.Length == 0)
{
_log.LogWarning(
"Dropping the ApplyAck for {DeploymentId} from {NodeId}: no "
+ "MeshTransport:CentralGrpcEndpoints are configured, so there is nowhere to post it. "
+ "The deployment will time out at central naming this node even though it applied.",
ack.DeploymentId, ack.NodeId);
return;
}
var message = MeshTransportDtoMapper.ToProto(ack);
// Hoisted: read-only, and the walk is strictly sequential, so one Metadata is shared safely
// across the attempts. The DEADLINE emphatically is not hoisted — see below.
var headers = new Metadata { { "authorization", $"Bearer {_options.GrpcPsk}" } };
if (_clusterId is not null)
headers.Add(ClusterHeader, _clusterId);
var failures = new List(endpoints.Length);
foreach (var endpoint in endpoints)
{
try
{
// Computed FRESH per endpoint, which is what MeshTransport:AckTimeoutSeconds documents
// ("applied per endpoint, not per ack"). A single absolute deadline hoisted out of this
// loop defeats failover against the characteristic WAN failure: a central that HANGS
// (SYN black-hole, frozen host) rather than refusing. The first endpoint burns the whole
// timeout, and the second is then dialled with a deadline already in the past — it fails
// instantly as DeadlineExceeded without a packet leaving the node, so the surviving
// central is never actually tried.
var deadline = _options.AckTimeoutSeconds > 0
? DateTime.UtcNow.AddSeconds(_options.AckTimeoutSeconds)
: (DateTime?)null;
var client = _clientFactory(endpoint);
using var call = client.ReportApplyAsync(message, headers, deadline, cancellationToken);
await call;
if (failures.Count == 0)
{
_log.LogDebug(
"Posted the ApplyAck for {DeploymentId} ({Outcome}) to {Endpoint}.",
ack.DeploymentId, ack.Outcome, endpoint);
}
else
{
// Information, not Debug, because this is a FAILOVER: the ack landed, but only
// after one or more endpoints cost their full deadline. The deploy simply took
// that much longer, and without this line the only evidence is the absence of the
// all-endpoints-failed Warning — which reads identically to a healthy first-try.
_log.LogInformation(
"Posted the ApplyAck for {DeploymentId} ({Outcome}) to {Endpoint} after "
+ "{FailedCount} endpoint(s) failed ({Failures}).",
ack.DeploymentId, ack.Outcome, endpoint, failures.Count, string.Join("; ", failures));
}
// The single-endpoint rule, and it is this `return`: the walk stops at the first
// endpoint that accepted the ack and never goes back to one that did not.
return;
}
catch (RpcException ex)
{
// Status code only — never the exception's own message and never the headers, either
// of which could echo credential material back into the log.
failures.Add($"{endpoint} => {ex.StatusCode}");
}
}
_log.LogWarning(
"Dropping the ApplyAck for {DeploymentId} (correlation {CorrelationId}) from {NodeId}: all "
+ "{Count} central endpoint(s) refused it ({Failures}). It is not buffered and will not be "
+ "retried, so the deployment will time out at central naming this node even though it "
+ "applied successfully.",
ack.DeploymentId, ack.CorrelationId, ack.NodeId, endpoints.Length, string.Join("; ", failures));
}
private CentralCommandService.CentralCommandServiceClient DefaultClientFactory(string endpoint)
{
// h2c: GrpcChannel over an http:// address negotiates prior-knowledge HTTP/2. One channel per
// endpoint, cached and reused (creating a channel per ack would leak a socket per deployment).
var channel = _channels.GetOrAdd(endpoint, ep => GrpcChannel.ForAddress(ep));
return new CentralCommandService.CentralCommandServiceClient(channel);
}
///
public void Dispose()
{
foreach (var channel in _channels.Values)
channel.Dispose();
_channels.Clear();
}
}