feat(mesh): CentralCommandService on the ConfigServe listener
This commit is contained in:
Joseph Doherty
@@ -94,9 +94,10 @@ public sealed class MeshTransportOptionsValidator : IValidateOptions<MeshTranspo
|
||||
}
|
||||
}
|
||||
|
||||
// The Grpc surface below is node-side: the PSK a driver presents (and requires), the central
|
||||
// endpoints it posts its ack to, and the two call deadlines. Scoped to the driver role for the
|
||||
// same reason the ClusterClient block above is scoped to its mode — an admin-only central node
|
||||
// The first Grpc block below is node-side: the PSK a driver presents (and requires), the
|
||||
// central endpoints it posts its ack to, and the two call deadlines (the admin-side rule that
|
||||
// follows it is a separate scope). Scoped to the driver role for the same reason the
|
||||
// ClusterClient block above is scoped to its mode — an admin-only central node
|
||||
// dials nodes rather than being dialled and resolves per-cluster keys at runtime through
|
||||
// MeshTransportOptions.ClusterPsks / the Secrets store, so requiring a single GrpcPsk and an
|
||||
// endpoint list of it would demand configuration it never reads. Roles come from
|
||||
@@ -107,6 +108,7 @@ public sealed class MeshTransportOptionsValidator : IValidateOptions<MeshTranspo
|
||||
var roles = RoleParser.ResolveEffectiveRoles(
|
||||
_configuration.GetSection("Cluster:Roles").Get<string[]>());
|
||||
var isDriver = Array.IndexOf(roles, RoleParser.Driver) >= 0;
|
||||
var isAdmin = Array.IndexOf(roles, RoleParser.Admin) >= 0;
|
||||
|
||||
if (isGrpc && isDriver)
|
||||
{
|
||||
@@ -166,6 +168,37 @@ public sealed class MeshTransportOptionsValidator : IValidateOptions<MeshTranspo
|
||||
}
|
||||
}
|
||||
|
||||
// The one admin-side obligation, and it is cross-read rather than a member of this options
|
||||
// class: central hosts CentralCommandService on the SAME dedicated h2c listener as the
|
||||
// config-serve endpoint (ConfigServe:GrpcListenPort) — a second cleartext HTTP/2 port would
|
||||
// buy nothing but another firewall rule. With that port unset the service is never mapped, so
|
||||
// every node's deployment ack has nowhere to land, and nothing errors: the deployment simply
|
||||
// times out naming nodes that applied perfectly. Owned here rather than by ConfigServe's own
|
||||
// validation because the requirement comes from MeshTransport:Mode, not from config-serve — a
|
||||
// Dps or ClusterClient central node is legitimately portless.
|
||||
if (isGrpc && isAdmin)
|
||||
{
|
||||
// Read as a string, the way SplitTopologyTransportValidator reads the modes:
|
||||
// GetValue<int> THROWS on an unparseable value, and a validator whose whole job is to
|
||||
// produce a readable failure must not be the thing that crashes on a typo. Missing,
|
||||
// unparseable and non-positive all mean the same thing — nothing is bound — so they share
|
||||
// one message.
|
||||
var configServePort = _configuration[
|
||||
$"{ConfigServeOptions.SectionName}:{nameof(ConfigServeOptions.GrpcListenPort)}"];
|
||||
if (!int.TryParse(configServePort, out var port) || port <= 0)
|
||||
{
|
||||
errors.Add(
|
||||
$"{ConfigServeOptions.SectionName}:"
|
||||
+ $"{nameof(ConfigServeOptions.GrpcListenPort)} is not set to a positive port. "
|
||||
+ $"Under {MeshTransportOptions.ModeGrpc} an admin node serves the central "
|
||||
+ "command endpoint on that listener, and without it every node's deployment ack "
|
||||
+ "has nowhere to land — the deploy would time out naming nodes that applied "
|
||||
+ "perfectly. Supply it via the environment "
|
||||
+ $"({ConfigServeOptions.SectionName}__"
|
||||
+ $"{nameof(ConfigServeOptions.GrpcListenPort)}).");
|
||||
}
|
||||
}
|
||||
|
||||
return errors.Count == 0
|
||||
? ValidateOptionsResult.Success
|
||||
: ValidateOptionsResult.Fail(string.Join(" ", errors));
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
using Akka.Actor;
|
||||
using Akka.Hosting;
|
||||
using Grpc.Core;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
|
||||
using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1;
|
||||
using ZB.MOM.WW.OtOpcUa.ControlPlane;
|
||||
// The generated service container is itself named CentralCommandService; alias its nested server
|
||||
// base so this impl can keep the natural name without colliding with the generated type.
|
||||
using GeneratedServiceBase =
|
||||
ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1.CentralCommandService.CentralCommandServiceBase;
|
||||
// Akka.Actor (needed for the single-argument Tell) also defines a Status. Every Status in this file
|
||||
// is the gRPC one, so bind the name rather than qualifying each use.
|
||||
using Status = Grpc.Core.Status;
|
||||
|
||||
namespace ZB.MOM.WW.OtOpcUa.AdminUI.Grpc;
|
||||
|
||||
/// <summary>
|
||||
/// Central-side ingress for a node's deployment acknowledgment (ClusterClient→gRPC migration
|
||||
/// Phase 1B). Decodes an <see cref="ApplyAckMessage"/> and hands it to the node-local
|
||||
/// <c>CentralCommunicationActor</c> — the same actor an Akka ClusterClient receptionist delivered
|
||||
/// to, whose forward-to-the-coordinator path is untouched. Registered only on admin-role nodes
|
||||
/// and only when <c>ConfigServe:GrpcListenPort > 0</c>.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// <para>
|
||||
/// <b>The actor is resolved per call, not in the constructor.</b> Kestrel answers before
|
||||
/// <c>Akka.Hosting</c> has finished spawning the actor graph, so a ctor lookup would throw
|
||||
/// during the boot window and surface as an opaque <c>Unknown</c>. Resolving per call turns
|
||||
/// that window into <see cref="StatusCode.Unavailable"/> — the retryable status — so the
|
||||
/// node's next attempt lands rather than the deploy recording a permanent failure against a
|
||||
/// node that applied fine.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>Decoding is graded before the registry.</b> A message that cannot decode is
|
||||
/// <see cref="StatusCode.InvalidArgument"/> even when the actor is also missing: Unavailable
|
||||
/// invites a retry, and retrying a message that can never decode is a loop. The decode itself
|
||||
/// never guesses (see <see cref="MeshTransportDtoMapper"/>) — an unspecified outcome, an
|
||||
/// unparseable id or a blank node name is refused rather than resolved to a plausible value,
|
||||
/// because the deploy path seals a revision on <c>Applied</c> and a guessed ack seals a
|
||||
/// deployment nobody applied.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>The rejection log carries the wire correlation id verbatim.</b> The inner ids are
|
||||
/// exactly what may have failed to parse, so the only value guaranteed readable at that point
|
||||
/// is the raw string — which is what an operator has to correlate the rejection against the
|
||||
/// node's own log.
|
||||
/// </para>
|
||||
/// <para>
|
||||
/// <b>One-way semantics, preserved.</b> <c>Tell</c>, not <c>Ask</c>: the returned
|
||||
/// <see cref="ReportAck"/> means "handed to the comm actor", never "the deployment is
|
||||
/// complete" — the same promise the receptionist made. The comm actor forwards to the
|
||||
/// coordinator singleton proxy, so an ack landing on either central node routes identically.
|
||||
/// </para>
|
||||
/// </remarks>
|
||||
public sealed class CentralCommandGrpcService : GeneratedServiceBase
|
||||
{
|
||||
private readonly ActorRegistry _registry;
|
||||
private readonly ILogger<CentralCommandGrpcService> _log;
|
||||
|
||||
/// <summary>Initializes a new instance of the <see cref="CentralCommandGrpcService"/> class.</summary>
|
||||
/// <param name="registry">The actor registry holding the node's <c>CentralCommunicationActor</c>.</param>
|
||||
/// <param name="log">Logger.</param>
|
||||
public CentralCommandGrpcService(ActorRegistry registry, ILogger<CentralCommandGrpcService> log)
|
||||
{
|
||||
_registry = registry;
|
||||
_log = log;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<ReportAck> ReportApply(ApplyAckMessage request, ServerCallContext context)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
ApplyAck ack;
|
||||
try
|
||||
{
|
||||
// Exactly the three the mapper documents: an unspecified/unknown outcome
|
||||
// (InvalidOperationException), an unparseable id (FormatException) and a blank node id
|
||||
// (ArgumentException). Deliberately not a blanket catch — anything else escaping the
|
||||
// mapper is a bug on this side, and reporting it to the caller as their InvalidArgument
|
||||
// would send an operator to debug a node that sent a perfectly good ack.
|
||||
ack = MeshTransportDtoMapper.FromProto(request);
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidOperationException or FormatException or ArgumentException)
|
||||
{
|
||||
// Logged with the RAW correlation id: the decode may have failed on the ids themselves.
|
||||
_log.LogWarning(
|
||||
ex,
|
||||
"Rejected an ApplyAck (correlation '{CorrelationId}') from {Peer}: it could not be "
|
||||
+ "decoded, so the deployment it names is NOT acknowledged on this node",
|
||||
request.CorrelationId, context.Peer);
|
||||
throw new RpcException(new Status(StatusCode.InvalidArgument, "undecodable ApplyAck"));
|
||||
}
|
||||
|
||||
if (!_registry.TryGet<CentralCommunicationActorKey>(out var actor))
|
||||
{
|
||||
_log.LogWarning(
|
||||
"Received an ApplyAck for {DeploymentId} from {NodeId} before the central "
|
||||
+ "communication actor was registered on this node; refusing with Unavailable so the "
|
||||
+ "caller retries rather than recording a failure against a node that applied",
|
||||
ack.DeploymentId, ack.NodeId);
|
||||
throw new RpcException(new Status(StatusCode.Unavailable, "central command actor not ready"));
|
||||
}
|
||||
|
||||
// Tell, not Ask: the RPC's own response is the acknowledgment, and the comm actor's existing
|
||||
// Forward-to-coordinator path is what it always was.
|
||||
actor.Tell(ack);
|
||||
|
||||
_log.LogDebug(
|
||||
"ApplyAck for {DeploymentId} from {NodeId} ({Outcome}) handed to the central communication actor",
|
||||
ack.DeploymentId, ack.NodeId, ack.Outcome);
|
||||
|
||||
return Task.FromResult(new ReportAck());
|
||||
}
|
||||
}
|
||||
@@ -744,6 +744,16 @@ if (hasDriver && syncListenPort > 0)
|
||||
if (hasAdmin && configServeGrpcPort > 0)
|
||||
{
|
||||
app.MapGrpcService<DeploymentArtifactGrpcService>();
|
||||
|
||||
// Central command ingress (ClusterClient→gRPC migration): the node→central deployment ack. It
|
||||
// rides the SAME listener rather than claiming a port of its own — a second cleartext h2c port
|
||||
// would buy nothing but another firewall rule, and both endpoints are admin-role and dialled by
|
||||
// the same driver nodes. Deliberately NOT gated on MeshTransport:Mode: mapping it under Dps
|
||||
// costs nothing (nothing dials it) whereas gating it would mean flipping the mode on a running
|
||||
// fleet requires a central restart before any node can ack — the same reasoning that registers
|
||||
// CentralCommunicationActor in both modes. CentralCommandAuthInterceptor (added at AddGrpc
|
||||
// above) is the ONLY inbound auth and fail-closes per cluster.
|
||||
app.MapGrpcService<CentralCommandGrpcService>();
|
||||
}
|
||||
|
||||
// Driver-node live-telemetry stream (per-cluster mesh Phase 5). Central dials in; gated by
|
||||
|
||||
Reference in New Issue
Block a user