518c699b90
Refactor SiteCommunicationActor's central→site routing table into one SiteCommandDispatcher — the single routing truth for the 28 migrated commands (IntegrationCallRequest, the dead 29th, stays on the actor and out of the dispatcher). The Akka actor and the new SiteCommandGrpcService both route through one dispatcher instance so the two transports can never drift on where a command goes. Server-side only: nothing central flips to gRPC yet (that is T1B.3); ClusterClient remains the live path. Decisions worth recording: - Targets preserved byte-for-byte. Lifecycle/OPC UA/query/route → the Deployment Manager singleton proxy; DeployArtifacts/EventLog/parked → their null-guarded handlers with the exact same "handler not available" replies; the parked handler stays NODE-LOCAL (per-node replicated-store owner), never the singleton proxy — pinned by a dispatcher test that asserts the target is the parked probe and NOT the dm proxy. - Sender preservation intact. The actor's command handlers became thin DispatchCommand delegations that still Forward (central Ask → reply routes straight back); the existing SiteCommunicationActorTests pass unchanged, which is the regression guard for that plumbing. UnsubscribeDebugView keeps its fire-and-forget shape: the actor Forwards, the gRPC service Tells + returns the synthetic UnsubscribeDebugViewAck so a unary RPC still answers. - Ack-before-Leave on failover. The dispatcher's PrepareFailover resolves the standby with a DRY-RUN (no leave) to build the ack, and hands back a deferred CommitLeave; the gRPC service returns the ack, then schedules the real Cluster.Leave — so a caller reaching the very node about to leave still gets its ack instead of a broken stream. The actor path keeps today's coupled resolve-and-leave (over ClusterClient the ack Tell only enqueues, so order is immaterial). Proven at both levels: a dispatcher test asserts the ack is built before CommitLeave runs, and a TestServer test asserts the recorded seam order is resolve-then-leave. - ControlPlaneAuthInterceptor gates SiteCommandService by EXTENDING DefaultGatedPrefixes (descriptor-derived), not by adding a constructor — the one-public-ctor invariant and its test stay green. Tests: SiteCommandDispatcherTests (28-command routing incl. parked node-locality and both failover paths) and SiteCommandGrpcService TestServer tests (auth, readiness→Unavailable, one command per oneof group, failover ordering). Full solution build 0/0; Communication.Tests 574 and Host.Tests 377 green. No active <Protobuf> item.
304 lines
15 KiB
C#
304 lines
15 KiB
C#
using Akka.Actor;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Artifacts;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DataConnection;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.DebugView;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Deployment;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.InboundApi;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Lifecycle;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.Management;
|
|
using ZB.MOM.WW.ScadaBridge.Commons.Messages.RemoteQuery;
|
|
using ZB.MOM.WW.ScadaBridge.Communication.Grpc;
|
|
|
|
namespace ZB.MOM.WW.ScadaBridge.Communication.Actors;
|
|
|
|
/// <summary>
|
|
/// The single routing truth for the 28 migrated central→site commands. Given one
|
|
/// of those command records it decides which local target answers it —
|
|
/// the Deployment Manager singleton proxy, the artifact/event-log/parked-message
|
|
/// handlers, or the node-local failover path — preserving EXACTLY the targets and
|
|
/// null-guard semantics <see cref="SiteCommunicationActor"/> used when this routing
|
|
/// lived inline.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// <para>
|
|
/// Two transports call this one unit: the Akka <see cref="SiteCommunicationActor"/>
|
|
/// (ClusterClient) and the new <c>SiteCommandGrpcService</c> (gRPC). Centralising the
|
|
/// table means the two can never drift on where a command goes. Each transport keeps
|
|
/// its own send mechanics — the actor <c>Forward</c>s (preserving the Ask sender), the
|
|
/// gRPC service <c>Ask</c>s and encodes the reply — but both read the same
|
|
/// <see cref="Route"/> here.
|
|
/// </para>
|
|
/// <para>
|
|
/// <b>The parked-message handler stays node-local on purpose.</b> A parked retry or
|
|
/// discard must run on the node that holds the replicated store row, so parked
|
|
/// commands route to the per-node <c>_parkedMessageHandler</c> — never onto the
|
|
/// singleton proxy (design §7.3). This is the same target the actor used; the
|
|
/// extraction does not "fix" it.
|
|
/// </para>
|
|
/// <para>
|
|
/// <b>Excluded by design:</b> <c>IntegrationCallRequest</c> — the 29th command, dead at
|
|
/// both ends. It never enters this dispatcher; the actor keeps its own vestigial handler
|
|
/// for it (28 of 29 migrate).
|
|
/// </para>
|
|
/// </remarks>
|
|
public sealed class SiteCommandDispatcher
|
|
{
|
|
/// <summary>How a resolved command is delivered to its target.</summary>
|
|
public enum RouteDisposition
|
|
{
|
|
/// <summary>Forward/Ask <see cref="Route.Target"/> and route its reply back.</summary>
|
|
Forward,
|
|
|
|
/// <summary>
|
|
/// Tell <see cref="Route.Target"/> and answer immediately with
|
|
/// <see cref="Route.Reply"/> — the fire-and-forget path (only
|
|
/// <see cref="UnsubscribeDebugViewRequest"/>, which the downstream never acks).
|
|
/// The actor <c>Forward</c>s it and sends nothing back; the gRPC transport Tells and
|
|
/// returns the synthetic ack so a unary RPC still answers.
|
|
/// </summary>
|
|
TellFireAndForget,
|
|
|
|
/// <summary>
|
|
/// No target is available (a null-guarded handler is unregistered); answer with the
|
|
/// synthetic <see cref="Route.Reply"/> — the "handler not available" reply the actor
|
|
/// produced inline.
|
|
/// </summary>
|
|
ImmediateReply
|
|
}
|
|
|
|
/// <summary>A resolved routing decision for one command.</summary>
|
|
/// <param name="Disposition">How the command is delivered.</param>
|
|
/// <param name="Target">The target actor for <see cref="RouteDisposition.Forward"/>/<see cref="RouteDisposition.TellFireAndForget"/>; <c>null</c> for an immediate reply.</param>
|
|
/// <param name="Reply">The synthetic reply for <see cref="RouteDisposition.ImmediateReply"/>/<see cref="RouteDisposition.TellFireAndForget"/>; <c>null</c> for a forward.</param>
|
|
public readonly record struct Route(RouteDisposition Disposition, IActorRef? Target, object? Reply);
|
|
|
|
/// <summary>The outcome of preparing a failover: the ack to send now, plus the leave to run after.</summary>
|
|
/// <param name="Ack">The ack the caller must send back before the node leaves.</param>
|
|
/// <param name="CommitLeave">
|
|
/// When non-<c>null</c> (accepted only), invoking it performs the real graceful
|
|
/// <c>Cluster.Leave</c>. It is deliberately deferred so the transport can flush the
|
|
/// <see cref="Ack"/> first — a caller reaching the very node that is about to leave still
|
|
/// receives the ack rather than a broken stream.
|
|
/// </param>
|
|
public readonly record struct FailoverOutcome(SiteFailoverAck Ack, Action? CommitLeave);
|
|
|
|
private readonly string _siteId;
|
|
private readonly IActorRef _deploymentManagerProxy;
|
|
|
|
// (role, dryRun) -> leaving node address, or null when there is no standby.
|
|
// dryRun:true resolves the target WITHOUT leaving; dryRun:false performs the Leave.
|
|
private readonly Func<string, bool, string?> _resolveFailover;
|
|
|
|
// Registered at runtime via the actor's RegisterLocalHandler flow. Written on the actor
|
|
// thread, read by both the actor and the gRPC service (a Kestrel thread), so volatile.
|
|
private volatile IActorRef? _eventLogHandler;
|
|
private volatile IActorRef? _parkedMessageHandler;
|
|
private volatile IActorRef? _artifactHandler;
|
|
|
|
/// <summary>Creates the dispatcher.</summary>
|
|
/// <param name="siteId">This site's identifier, stamped into synthetic replies and matched by the failover guard.</param>
|
|
/// <param name="deploymentManagerProxy">The local Deployment Manager singleton proxy — the target for all lifecycle/OPC UA/query/route commands.</param>
|
|
/// <param name="resolveFailover">
|
|
/// Resolves (and, when <c>dryRun</c> is false, performs) the graceful leave of the oldest
|
|
/// Up member in a role scope. Injected so tests need no real cluster.
|
|
/// </param>
|
|
public SiteCommandDispatcher(
|
|
string siteId,
|
|
IActorRef deploymentManagerProxy,
|
|
Func<string, bool, string?> resolveFailover)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(siteId);
|
|
ArgumentNullException.ThrowIfNull(deploymentManagerProxy);
|
|
ArgumentNullException.ThrowIfNull(resolveFailover);
|
|
|
|
_siteId = siteId;
|
|
_deploymentManagerProxy = deploymentManagerProxy;
|
|
_resolveFailover = resolveFailover;
|
|
}
|
|
|
|
/// <summary>Registers the site event-log query handler (a cluster singleton proxy).</summary>
|
|
/// <param name="handler">The event-log handler.</param>
|
|
public void RegisterEventLogHandler(IActorRef handler) => _eventLogHandler = handler;
|
|
|
|
/// <summary>Registers the node-local parked-message handler (the replicated-store owner on this node).</summary>
|
|
/// <param name="handler">The parked-message handler.</param>
|
|
public void RegisterParkedMessageHandler(IActorRef handler) => _parkedMessageHandler = handler;
|
|
|
|
/// <summary>Registers the artifact-deployment handler.</summary>
|
|
/// <param name="handler">The artifact handler.</param>
|
|
public void RegisterArtifactHandler(IActorRef handler) => _artifactHandler = handler;
|
|
|
|
/// <summary>
|
|
/// Resolves how one of the 27 non-failover commands is delivered. <see cref="TriggerSiteFailover"/>
|
|
/// is handled separately via <see cref="PrepareFailover"/>/<see cref="HandleFailover"/> because its
|
|
/// leave is deferred; passing it here throws.
|
|
/// </summary>
|
|
/// <param name="command">The command to route.</param>
|
|
/// <returns>The routing decision.</returns>
|
|
/// <exception cref="ArgumentException">The command is not a migrated site command (or is failover).</exception>
|
|
public Route ResolveRoute(object command)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(command);
|
|
|
|
return command switch
|
|
{
|
|
// ── Deployment + instance lifecycle → Deployment Manager singleton proxy ──
|
|
RefreshDeploymentCommand => ToProxy(),
|
|
EnableInstanceCommand => ToProxy(),
|
|
DisableInstanceCommand => ToProxy(),
|
|
DeleteInstanceCommand => ToProxy(),
|
|
DeploymentStateQueryRequest => ToProxy(),
|
|
|
|
// ── Artifact deployment → artifact handler (null-guarded) ──
|
|
DeployArtifactsCommand c => _artifactHandler is { } h
|
|
? Forwarded(h)
|
|
: Immediate(new ArtifactDeploymentResponse(
|
|
c.DeploymentId, _siteId, false, "Artifact handler not available", DateTimeOffset.UtcNow)),
|
|
|
|
// ── Interactive OPC UA / MxGateway → Deployment Manager singleton proxy ──
|
|
// The singleton always lands on the active node, which owns the live sessions.
|
|
BrowseNodeCommand => ToProxy(),
|
|
SearchAddressSpaceCommand => ToProxy(),
|
|
ReadTagValuesCommand => ToProxy(),
|
|
VerifyEndpointCommand => ToProxy(),
|
|
TrustServerCertCommand => ToProxy(),
|
|
ListServerCertsCommand => ToProxy(),
|
|
RemoveServerCertCommand => ToProxy(),
|
|
WriteTagRequest => ToProxy(),
|
|
|
|
// ── Remote queries: event log (null-guarded) + debug view (proxy) ──
|
|
EventLogQueryRequest r => _eventLogHandler is { } h
|
|
? Forwarded(h)
|
|
: Immediate(new EventLogQueryResponse(
|
|
r.CorrelationId, _siteId, [], null, false, false,
|
|
"Event log handler not available", DateTimeOffset.UtcNow)),
|
|
DebugSnapshotRequest => ToProxy(),
|
|
SubscribeDebugViewRequest => ToProxy(),
|
|
// Fire-and-forget: the Deployment Manager never acks an unsubscribe, so the gRPC
|
|
// transport Tells it and returns the synthetic ack; the actor just Forwards.
|
|
UnsubscribeDebugViewRequest => new Route(
|
|
RouteDisposition.TellFireAndForget, _deploymentManagerProxy, UnsubscribeDebugViewAck.Instance),
|
|
|
|
// ── Parked store-and-forward actions → node-local parked handler (null-guarded) ──
|
|
ParkedMessageQueryRequest r => _parkedMessageHandler is { } h
|
|
? Forwarded(h)
|
|
: Immediate(new ParkedMessageQueryResponse(
|
|
r.CorrelationId, _siteId, [], 0, r.PageNumber, r.PageSize, false,
|
|
"Parked message handler not available", DateTimeOffset.UtcNow)),
|
|
ParkedMessageRetryRequest r => _parkedMessageHandler is { } h
|
|
? Forwarded(h)
|
|
: Immediate(new ParkedMessageRetryResponse(
|
|
r.CorrelationId, false, "Parked message handler not available")),
|
|
ParkedMessageDiscardRequest r => _parkedMessageHandler is { } h
|
|
? Forwarded(h)
|
|
: Immediate(new ParkedMessageDiscardResponse(
|
|
r.CorrelationId, false, "Parked message handler not available")),
|
|
RetryParkedOperation r => _parkedMessageHandler is { } h
|
|
? Forwarded(h)
|
|
: Immediate(new ParkedOperationActionAck(
|
|
r.CorrelationId, Applied: false, "Parked message handler not available")),
|
|
DiscardParkedOperation r => _parkedMessageHandler is { } h
|
|
? Forwarded(h)
|
|
: Immediate(new ParkedOperationActionAck(
|
|
r.CorrelationId, Applied: false, "Parked message handler not available")),
|
|
|
|
// ── Inbound-API Route.To() relays → Deployment Manager singleton proxy ──
|
|
RouteToCallRequest => ToProxy(),
|
|
RouteToGetAttributesRequest => ToProxy(),
|
|
RouteToSetAttributesRequest => ToProxy(),
|
|
RouteToWaitForAttributeRequest => ToProxy(),
|
|
|
|
TriggerSiteFailover => throw new ArgumentException(
|
|
"TriggerSiteFailover is handled by PrepareFailover/HandleFailover, not ResolveRoute.",
|
|
nameof(command)),
|
|
|
|
_ => throw new ArgumentException(
|
|
$"'{command.GetType().Name}' is not a migrated site command.", nameof(command))
|
|
};
|
|
|
|
Route ToProxy() => Forwarded(_deploymentManagerProxy);
|
|
}
|
|
|
|
private static Route Forwarded(IActorRef target) => new(RouteDisposition.Forward, target, null);
|
|
|
|
private static Route Immediate(object reply) => new(RouteDisposition.ImmediateReply, null, reply);
|
|
|
|
/// <summary>
|
|
/// The actor's failover path: resolve the standby AND issue the leave in one step (today's
|
|
/// coupled behaviour over ClusterClient, where <c>Tell</c> merely enqueues the ack so leave
|
|
/// order is immaterial), then return the ack.
|
|
/// </summary>
|
|
/// <param name="msg">The failover command.</param>
|
|
/// <returns>The ack to send back.</returns>
|
|
public SiteFailoverAck HandleFailover(TriggerSiteFailover msg)
|
|
=> FailoverCore(msg, commitLeaveImmediately: true).Ack;
|
|
|
|
/// <summary>
|
|
/// The gRPC failover path: resolve the standby WITHOUT leaving, build the ack, and hand back a
|
|
/// deferred <see cref="FailoverOutcome.CommitLeave"/>. The caller must send the ack before
|
|
/// invoking the leave — otherwise a caller reaching the leaving node sees a broken stream
|
|
/// instead of its ack (the ack-before-Leave rule).
|
|
/// </summary>
|
|
/// <param name="msg">The failover command.</param>
|
|
/// <returns>The ack and the deferred leave (leave is <c>null</c> when refused).</returns>
|
|
public FailoverOutcome PrepareFailover(TriggerSiteFailover msg)
|
|
=> FailoverCore(msg, commitLeaveImmediately: false);
|
|
|
|
private FailoverOutcome FailoverCore(TriggerSiteFailover msg, bool commitLeaveImmediately)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(msg);
|
|
|
|
// A misrouted command must be refused, not silently acted on — acting would fail over a
|
|
// site the operator never selected. Checked before resolving so the resolver is untouched.
|
|
if (!string.Equals(msg.SiteId, _siteId, StringComparison.Ordinal))
|
|
{
|
|
return new FailoverOutcome(
|
|
new SiteFailoverAck(
|
|
msg.CorrelationId, Accepted: false, TargetAddress: null,
|
|
ErrorMessage: $"Command addressed to site '{msg.SiteId}' but this node serves '{_siteId}'."),
|
|
CommitLeave: null);
|
|
}
|
|
|
|
// Site singletons are scoped to the site-specific role, so failover must target that role.
|
|
var role = $"site-{_siteId}";
|
|
try
|
|
{
|
|
if (commitLeaveImmediately)
|
|
{
|
|
var target = _resolveFailover(role, false);
|
|
if (target is null)
|
|
{
|
|
return new FailoverOutcome(NoPeerAck(msg), CommitLeave: null);
|
|
}
|
|
|
|
return new FailoverOutcome(
|
|
new SiteFailoverAck(msg.CorrelationId, Accepted: true, target, ErrorMessage: null),
|
|
CommitLeave: null);
|
|
}
|
|
|
|
var resolved = _resolveFailover(role, true);
|
|
if (resolved is null)
|
|
{
|
|
return new FailoverOutcome(NoPeerAck(msg), CommitLeave: null);
|
|
}
|
|
|
|
return new FailoverOutcome(
|
|
new SiteFailoverAck(msg.CorrelationId, Accepted: true, resolved, ErrorMessage: null),
|
|
CommitLeave: () => _resolveFailover(role, false));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// A fault must be reported to the operator, never thrown into supervision (over
|
|
// ClusterClient) or surfaced as a broken stream (over gRPC).
|
|
return new FailoverOutcome(
|
|
new SiteFailoverAck(
|
|
msg.CorrelationId, Accepted: false, TargetAddress: null, ErrorMessage: ex.Message),
|
|
CommitLeave: null);
|
|
}
|
|
}
|
|
|
|
private static SiteFailoverAck NoPeerAck(TriggerSiteFailover msg) => new(
|
|
msg.CorrelationId, Accepted: false, TargetAddress: null,
|
|
ErrorMessage: "No standby available — failing over a lone node would be an outage.");
|
|
}
|