using Akka.Actor;
using Akka.Cluster.Tools.PublishSubscribe;
using Akka.Event;
using Grpc.Core;
using Microsoft.EntityFrameworkCore;
using ZB.MOM.WW.OtOpcUa.Cluster;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1;
using ZB.MOM.WW.OtOpcUa.Commons.Types;
using ZB.MOM.WW.OtOpcUa.Configuration;
// Grpc.Core (imported for RpcException on the dial path) declares a Status of its own. This file's
// bare Status is Akka's — the Status.Failure a faulted piped task arrives as; the gRPC one is only
// ever reached through an RpcException instance.
using Status = Akka.Actor.Status;
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
///
/// Central-side end of the central↔node command boundary (per-cluster mesh Phase 2). Routes
/// every central→node command out — over DistributedPubSub (pair-local) or gRPC (cross-mesh),
/// per — and forwards inbound s to
/// the deploy coordinator singleton.
///
///
///
/// Registered per node, NOT as a cluster singleton. A driver node posts its ack to
/// whichever central node answers first, so a live comm actor must exist on both. A singleton
/// would be reachable only through the one node hosting it, and the other central node's ack
/// ingress would have nothing to forward to.
///
///
/// The gRPC branch fans node by node over the contact load's target set. That load
/// selects exactly the enabled, non-maintenance ClusterNode rows carrying a
/// GrpcPort, and the fan dials every one of them, one call per node. It is deliberately
/// NOT narrower: the node side has no ClusterId or node filter at all — scoping happens
/// later, inside the artifact — so a node left out here silently keeps its old configuration
/// while the deployment seals green.
///
///
/// Fire-and-forget, on both transports. The gRPC branch pipes per-node results back to
/// itself for logging ONLY: a failure names the node and the status and is dropped. No retry,
/// no queue, no ordering guarantee across nodes. That is not an omission — it is the
/// DistributedPubSub semantic preserved exactly, and the backstops are unchanged (the deploy
/// coordinator's apply deadline, and a node's boot-time replay).
///
///
public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers
{
private const string RefreshTimerKey = "mesh-contact-refresh";
private const string BootstrapRetryTimerKey = "mesh-contact-bootstrap-retry";
// While the known-cluster gate is unpublished (PSK provider unresolved, or the first
// ClusterNode load failed) a single-shot RefreshContacts is re-armed every
// DefaultBootstrapRetryInterval, at most MaxBootstrapRetries times (~30 s), stopping at the
// first publish. Constants, not config. Steady state costs nothing extra.
private static readonly TimeSpan DefaultBootstrapRetryInterval = TimeSpan.FromSeconds(2);
private const int MaxBootstrapRetries = 15;
private readonly TimeSpan _bootstrapRetryInterval;
private int _bootstrapRetries;
private int _refreshAttempts;
private bool _gatePublished;
private readonly IDbContextFactory _dbFactory;
private readonly MeshTransportOptions _options;
private readonly Func _coordinator;
// Resolved in PreStart, never at construction. See the Props doc: the DI graph behind these two
// reaches actor-registry entries that do not exist yet while the Akka builder's starters are
// still running, and touching it there is what stopped every central node from booting.
private readonly Func? _dialerFactory;
private readonly Func? _pskProviderFactory;
private IMeshCommandDialer? _dialer;
private IMeshPskProvider? _pskProvider;
// Memoize-on-first-use, NOT PreStart-only: PreStart runs on a dispatcher thread concurrently
// with the builder's remaining starters, so its resolve can lose a millisecond-wide race
// against AuditWriterActorKey's registration (the fourth starter) and Resolve() would swallow that
// into a silent null — the known-cluster gate unpublished, fail-open. These properties
// retry on first use, but the first RefreshContacts is sent from PreStart (TimeSpan.Zero) and
// can ALSO land inside the starter window (a fast DB load finishes in ms), so first-use is not
// enough: while the gate is unpublished a bounded fast retry (ScheduleBootstrapRetry) re-arms
// RefreshContacts every couple of seconds until it publishes. Actor-thread
// only, so ??= needs no synchronisation; both are container singletons, so a repeat resolve
// is a dictionary hit.
private IMeshPskProvider? PskProvider =>
_pskProvider ??= Resolve(_pskProviderFactory, nameof(IMeshPskProvider), firstUse: true);
private IMeshCommandDialer? Dialer =>
_dialer ??= Resolve(_dialerFactory, nameof(IMeshCommandDialer), firstUse: true);
private readonly ILoggingAdapter _log = Context.GetLogger();
private readonly bool _useGrpc;
// Node ids already warned about for a null GrpcPort. A persistently portless row would otherwise
// produce one warning per refresh forever; same warn-once shape as TelemetryNodeSource. Actor
// thread only.
private readonly HashSet _warnedPortlessNodes = new(StringComparer.Ordinal);
// Set once PublishKnownClusters has reported a genuinely-absent PSK provider (a factory WAS
// supplied and still resolves null after the first-use retry). PublishKnownClusters runs on
// the contact-refresh timer for the actor's whole life, so an unguarded Error there would
// repeat every ContactRefreshSeconds forever. Same warn-once shape as _warnedPortlessNodes;
// actor thread only.
private bool _pskProviderAbsenceReported;
// The gRPC dispatch set, replaced wholesale on every contact load. Actor thread only; the fan
// captures the list reference, so a refresh mid-fan cannot mutate a dispatch already under way.
private IReadOnlyList _targets = [];
private CancellationTokenSource _lifecycle = new();
/// Gets the timer scheduler driving the periodic contact refresh.
public ITimerScheduler Timers { get; set; } = null!;
/// Creates the props for this actor.
/// Factory for the config database holding ClusterNode rows.
/// The bound mesh-transport options.
///
/// Resolves the deploy-coordinator singleton proxy. A rather than
/// an so this actor never depends on registration order at wiring
/// time — it resolves on first use and caches.
///
///
/// Resolves the thing that places one command call on one node, under
/// . Optional so the Dps transport needs nothing
/// new wired; a dialer under Grpc is a mis-wiring and is
/// reported loudly rather than silently dropping every command.
///
///
/// Resolves central's per-cluster key lookup, told on every contact refresh which clusters
/// exist (the known-cluster gate) and to drop its cache (so a rotated key takes effect within
/// one refresh). Optional for the same reason as .
///
///
/// Interval of the bounded fast retry run while the known-cluster gate is unpublished
/// (default 2 s, at most 15 attempts).
///
/// The props.
///
///
/// Both dependencies arrive as , and that is not a style
/// choice. Passing instances meant the caller resolved them while the Akka builder's
/// starters were still running, and the DI graph behind them reaches back into the actor
/// registry: dialer → IMeshPskProvider → ISecretResolver → AddZbSecrets
/// → IAuditWriter → ActorAuditWriter, whose constructor calls
/// registry.Get<AuditWriterActorKey>() — a singleton registered dozens of
/// lines LATER in the very same builder. Every central node failed to boot, in every
/// transport mode, on a MissingActorRegistryEntryException. Deferring to
/// moves the first resolve to a point where the host is fully
/// built, and matches the the coordinator proxy already uses
/// one argument earlier for the same class of reason.
///
///
/// This is the third instance of the same failure mode in this family — eager resolution
/// inside an AddAkka configurator, previously scadaproj#1 (a DI cycle that
/// deadlocked) and mesh Phase 6's IClusterRoleInfo boot StackOverflow. The
/// rule it keeps re-teaching: inside an Akka builder, resolve nothing you can defer.
///
///
public static Props Props(
IDbContextFactory dbFactory,
MeshTransportOptions options,
Func coordinator,
Func? dialer = null,
Func? pskProvider = null,
TimeSpan? bootstrapRetryInterval = null) =>
Akka.Actor.Props.Create(() =>
new CentralCommunicationActor(
dbFactory, options, coordinator, dialer, pskProvider, bootstrapRetryInterval));
/// Initializes a new instance of the class.
/// Factory for the config database.
/// The bound mesh-transport options.
/// Lazily resolves the deploy-coordinator singleton proxy.
/// Resolves the gRPC command dialer; invoked in .
/// Resolves the PSK provider; invoked in .
/// Fast-retry interval while the gate is unpublished; default 2 s.
public CentralCommunicationActor(
IDbContextFactory dbFactory,
MeshTransportOptions options,
Func coordinator,
Func? dialer = null,
Func? pskProvider = null,
TimeSpan? bootstrapRetryInterval = null)
{
_bootstrapRetryInterval = bootstrapRetryInterval ?? DefaultBootstrapRetryInterval;
_dbFactory = dbFactory;
_options = options;
_coordinator = coordinator;
_dialerFactory = dialer;
_pskProviderFactory = pskProvider;
_useGrpc = string.Equals(
options.Mode, MeshTransportOptions.ModeGrpc, StringComparison.OrdinalIgnoreCase);
Receive(HandleMeshCommand);
Receive(HandleApplyAck);
Receive(_ =>
{
_refreshAttempts++;
LoadContactsFromDb();
});
Receive(HandleContactsLoaded);
Receive(msg => _log.Debug("Mesh PSK refresh completed for {Count} cluster(s)", msg.ClusterCount));
Receive(msg => _log.Warning(msg.Error,
"Mesh PSK refresh failed; previously resolved keys stay in service until the next refresh"));
Receive(HandleCommandDispatched);
// A faulted LoadContactsFromDb task is piped here as a Status.Failure. Without this handler
// the failure is an unhandled message (debug level only) and the refresh fails silently —
// an operator cannot distinguish "no nodes configured" from "the database is down", and the
// contact set silently freezes at whatever it last held.
//
// The freeze extends to the known-cluster gate, deliberately: a failed load must neither
// widen the gate nor narrow it, and the previous set is the last thing central actually knew.
// The second line exists because that freeze is INVISIBLE otherwise, and it is fail-OPEN
// before the first successful load — a central that has never reached its ConfigDb refuses
// no cluster at all. Until the first publish a bounded fast retry (above) heals it.
Receive(f =>
{
// Gate still unpublished (boot with ConfigDb briefly down): retry fast, bounded.
if (!_gatePublished)
ScheduleBootstrapRetry();
_log.Warning(
f.Cause,
"Failed to load ClusterNode contact points; the mesh contact set was NOT refreshed "
+ "and may be stale or empty. Commands may be dropped until the next refresh");
_log.Warning(
"The known-cluster gate was NOT refreshed either: it still holds the set from the "
+ "last successful load, and if none has succeeded yet (boot, or central SQL Server "
+ "unreachable since start) it is INACTIVE — every cluster id reaches the secret "
+ "store, which is the pre-authentication cost the gate exists to remove");
});
Receive(_ => { /* DPS subscribe confirmation */ });
}
///
protected override void PreStart()
{
// First point at which the host is fully built — see the Props remark. Re-runs on every
// actor restart, which re-reads the container rather than caching a stale reference; both
// dependencies are container singletons, so a restart gets the same instances back.
//
// Quiet (firstUse: false — see Resolve): PreStart runs on a dispatcher thread concurrently
// with the builder's remaining starters and loses a millisecond-wide race against
// AuditWriterActorKey's registration (the fourth starter) on essentially every admin boot (live
// gates: 1A 2/3 boots, 1B 4/4 boots). That is the ORDINARY case, not a malfunction — the
// memoized PskProvider/Dialer properties retry on first use, and the bounded bootstrap
// retry re-arms RefreshContacts until the gate publishes (the first refresh itself can
// land inside the starter window). Logging it as
// Warning/Error twice a boot is what made a soak error-grep on a healthy rig impossible.
_dialer = Resolve(_dialerFactory, nameof(IMeshCommandDialer), firstUse: false);
_pskProvider = Resolve(_pskProviderFactory, nameof(IMeshPskProvider), firstUse: false);
if (!_useGrpc)
{
_log.Info(
"Mesh transport is {Mode}: commands are published on DistributedPubSub and nothing "
+ "is dialled. The contact refresh still runs — it is what keeps the known-cluster "
+ "gate current, and that gate belongs to the inbound ack endpoint, which is mapped "
+ "by port and does not care about this mode", _options.Mode);
}
// Deliberately NO null guard for either dependency here. At this point a still-null value
// is indistinguishable from the starter-ordering race Resolve just logged quietly above,
// so a loud report here cannot tell a lost race from a genuine absence — that decision
// belongs where a still-null result after the memoized properties' first-use retry
// actually costs something: DispatchOverGrpc for the dialer (every dropped command), and
// PublishKnownClusters for the PSK provider (once per incarnation, since that handler runs
// on the contact-refresh timer for the actor's whole life).
// Started in EVERY mode. The refresh is not transport state: it is central's ClusterNode
// inventory, and the known-cluster gate it feeds protects an endpoint mapped by PORT, not by
// MeshTransport:Mode. Gating the timer on the mode would mean a dark-switch rollback to
// Dps left that endpoint live with the gate permanently inactive — silently
// reopening both the timing oracle and the pre-auth store/audit flood. Under Dps this is a
// ConfigDb query every ContactRefreshSeconds that did not run before; central always has a
// ConfigDb, and one indexed read a minute is the cheaper half of the trade.
Timers.StartPeriodicTimer(
RefreshTimerKey,
new RefreshContacts(),
TimeSpan.Zero,
TimeSpan.FromSeconds(_options.ContactRefreshSeconds));
}
/// Re-arms a single-shot while the gate is unpublished.
/// when the retry budget is spent and nothing was scheduled.
private bool ScheduleBootstrapRetry()
{
if (_bootstrapRetries >= MaxBootstrapRetries)
return false;
_bootstrapRetries++;
Timers.StartSingleTimer(BootstrapRetryTimerKey, new RefreshContacts(), _bootstrapRetryInterval);
return true;
}
///
/// Invokes a dependency factory, converting a resolution failure into a logged absence.
///
/// The dependency's type.
/// The factory, or when nothing was supplied.
/// The dependency's name, for the log.
///
/// for the eager call,
/// for the retry the memoized / properties make
/// on first use. Controls wording only, deliberately at the SAME (Debug) severity either
/// way — see the remarks on why the loud signal lives elsewhere.
///
/// The resolved dependency, or .
///
///
/// Catching is the point. A throw out of is an actor failure, and
/// the supervisor's answer to that is to restart — which calls
/// again, and a broken DI graph is not a transient condition. That is an unbounded
/// restart loop on the node's command boundary, taking the transport-independent ack
/// path down with it. Logging and continuing leaves the ack path serving.
///
///
/// Never logs above Debug, on EITHER call site. From inside this method, a
/// PreStart resolve losing the starter-ordering race and a genuinely broken factory are
/// indistinguishable — both return null. Only the CALLER can tell them apart, because
/// only the caller knows whether a value came back after the retry the memoized
/// properties perform. So the loud, operator-facing signal is deliberately NOT here: it
/// is at the two places that consume a still-null result after that retry —
/// for the dialer (fires every dropped command) and
/// for the PSK provider (fires once per incarnation,
/// because that call site runs on a timer forever, where
/// only runs once per command).
///
///
private T? Resolve(Func? factory, string name, bool firstUse)
where T : class
{
if (factory is null)
return null; // Never supplied — not a resolve failure, nothing to report at either call site.
T? result;
try
{
result = factory();
}
catch (Exception ex)
{
LogUnresolved(name, firstUse, ex);
return null;
}
if (result is null)
LogUnresolved(name, firstUse, null);
return result;
}
private void LogUnresolved(string name, bool firstUse, Exception? ex)
{
if (firstUse)
{
_log.Debug(
ex,
"{Dependency} is still not resolvable on the first-use retry; continuing without "
+ "it for now. Whether this is worth an operator-facing signal is the caller's call",
name);
}
else
{
_log.Debug(
ex,
"{Dependency} was not yet resolvable at PreStart — likely a starter-ordering race "
+ "with a later starter that has not finished registering yet; will retry on first "
+ "use", name);
}
}
///
protected override void PostStop()
{
// Cancelling the lifecycle token also cancels every gRPC dial in flight. The dialer itself is
// deliberately NOT disposed here: it is a container-owned singleton shared with anything else
// that may dial, and an actor restart must not tear its channel pool down.
_lifecycle.Cancel();
_lifecycle.Dispose();
}
private void HandleMeshCommand(MeshCommand cmd)
{
// Checked first, so a Grpc-mode command never reaches DistributedPubSub. DPS is mesh-wide and
// cannot cross an application-Cluster boundary at all in the split topology.
if (_useGrpc)
{
DispatchOverGrpc(cmd);
return;
}
DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(cmd.Topic, cmd.Message));
}
///
/// Encodes one command once and fans it to every target node, one unary gRPC call each.
///
/// The command envelope carrying the domain record.
///
///
/// Encoded once, before the fan. Not per node — partly because the wire form is
/// identical for every recipient, but mostly because AlarmCommand has no
/// correlation member of its own and the envelope mints one: minting inside the loop
/// would give one operator action N unrelated correlation ids and make it untraceable.
///
///
/// Nothing captured from the actor. Each dial is a static call taking only locals,
/// so no per-node continuation can touch actor state off the actor's thread. Results come
/// back as ordinary messages via PipeTo and are handled on the actor thread like
/// anything else.
///
///
/// Every dial starts on the thread pool, not here. An async method runs
/// synchronously up to its first await, and the dialer's pre-await work includes
/// IMeshPskProvider.TryGetPsk, which blocks on the secret store (bounded, but
/// seconds) the first time a cluster is looked up — and again after every refresh, since
/// the refresh drops the cache to pick up rotations. Calling straight into it would park
/// this actor's mailbox, and deployment acks queue behind that mailbox.
///
///
private void DispatchOverGrpc(MeshCommand cmd)
{
var commandType = cmd.Message.GetType().Name;
var dialer = Dialer;
if (dialer is null)
{
_log.Error(
"Mesh transport is Grpc but no command dialer is wired — dropping {MessageType}. The "
+ "command is not buffered and will not be retried",
commandType);
return;
}
// Captured before the fan: a contact refresh between here and the last dial must not change
// the recipient set of a dispatch already under way.
var targets = _targets;
if (targets.Count == 0)
{
// App-level drop. The command is NOT queued and will NOT be retried: a deploy fails at
// the coordinator's apply deadline naming the silent nodes, which is a better operator
// signal than a command arriving minutes late against a config the DB already recorded
// as failed.
_log.Warning(
"No mesh gRPC targets — dropping {MessageType}. The last contact refresh produced no "
+ "enabled, non-maintenance ClusterNode row carrying a GrpcPort in any cluster; the "
+ "command is not buffered and will not be retried",
commandType);
return;
}
CommandEnvelope envelope;
try
{
envelope = MeshTransportDtoMapper.ToEnvelope(cmd.Message, new CorrelationId(Guid.NewGuid()));
}
catch (Exception ex)
{
// The mapper refuses a payload the far side could not decode. Dropping here is the same
// outcome the old transport gave for such a message (nothing on the node matched it), but
// loud instead of silent.
_log.Error(
ex,
"Cannot encode {MessageType} for the gRPC command plane; the command is dropped and "
+ "was NOT delivered to any node",
commandType);
return;
}
CancellationToken ct;
try
{
ct = _lifecycle.Token;
}
catch (ObjectDisposedException)
{
return; // Stopping.
}
var self = Self;
var correlationId = envelope.CorrelationId;
foreach (var target in targets)
{
var node = target;
// CancellationToken.None on Task.Run, deliberately: cancelling the SCHEDULING would make
// the task complete as canceled, which PipeTo converts into a Status.Failure — the one
// message this actor already reads as "the contact load failed". The lifecycle token is
// passed into the dial instead, where it cancels the call and comes back as an ordinary
// per-node outcome.
Task.Run(
() => DispatchOneAsync(dialer, node, envelope, commandType, correlationId, ct),
CancellationToken.None)
.PipeTo(self);
}
}
///
/// Dials one node and turns every possible outcome into a .
///
/// The dialer seam.
/// The node to dial.
/// The already-encoded command.
/// CLR type name of the command, for the log.
/// Transport correlation, for the log.
/// The actor's lifecycle token.
/// The outcome, always — this method does not fault.
///
/// Catching everything is what keeps the result on ONE channel. A faulting task piped to an
/// actor arrives as Status.Failure, which this actor already handles as "the contact
/// load failed" — so an unhandled dial fault would log a database-outage warning about a node
/// that simply did not answer.
///
private static async Task DispatchOneAsync(
IMeshCommandDialer dialer,
MeshNodeTarget target,
CommandEnvelope envelope,
string commandType,
string correlationId,
CancellationToken ct)
{
try
{
await dialer.DispatchAsync(target.Endpoint, envelope, target.ClusterId, ct)
.ConfigureAwait(false);
return new CommandDispatched(target.NodeId, commandType, correlationId, null, null);
}
catch (RpcException ex)
{
return new CommandDispatched(
target.NodeId,
commandType,
correlationId,
ex.StatusCode.ToString(),
Truncate(ex.Status.Detail));
}
catch (OperationCanceledException)
{
// Named rather than swept into the generic arm: this is the ordinary shape of a node that
// did not answer in time, and "Unknown / TaskCanceledException" would send an operator
// looking for a bug instead of at DispatchTimeoutSeconds.
return new CommandDispatched(
target.NodeId,
commandType,
correlationId,
"Cancelled",
"the dial did not complete within MeshTransport:DispatchTimeoutSeconds, or central "
+ "is shutting down");
}
catch (Exception ex)
{
// Type name only, never the message: this path also covers whatever the dialer's channel
// construction can throw, and a message is the one place a misconfigured endpoint string
// could carry something an operator did not mean to put in a log.
return new CommandDispatched(
target.NodeId, commandType, correlationId, "Unknown", ex.GetType().Name);
}
}
///
/// Bounds a status detail before it reaches the log. The string comes from the far side, and
/// the far side is a node — trusted, but not a reason to let it choose how many lines of
/// central's log one failed command occupies.
///
/// The received status detail.
/// The detail, at most 200 characters.
private static string Truncate(string? detail)
=> detail is null || detail.Length <= 200 ? detail ?? "" : detail[..200] + "…";
private void HandleCommandDispatched(CommandDispatched result)
{
if (result.FailureStatus is null)
{
_log.Debug(
"Mesh: {CommandType} (correlationId={CorrelationId}) accepted by {NodeId} over gRPC",
result.CommandType, result.CorrelationId, result.NodeId);
return;
}
_log.Warning(
"Mesh: {CommandType} (correlationId={CorrelationId}) was NOT delivered to {NodeId} "
+ "({Status}: {Detail}). It is dropped and will NOT be retried — the deployment apply "
+ "deadline and the node's boot-time replay are the backstops",
result.CommandType, result.CorrelationId, result.NodeId,
result.FailureStatus, result.FailureDetail);
}
private void HandleApplyAck(ApplyAck ack)
{
var coordinator = _coordinator();
if (coordinator is null)
{
_log.Warning(
"Received ApplyAck for {DeploymentId} from {NodeId} but the deploy coordinator is "
+ "not resolvable on this node; the ack is lost and the deployment will time out "
+ "naming a node that applied successfully",
ack.DeploymentId, ack.NodeId);
return;
}
// Forward, not Tell: preserves the original sender across the hop so the coordinator sees
// the node rather than this relay.
coordinator.Forward(ack);
}
private void LoadContactsFromDb()
{
var self = Self;
CancellationToken ct;
try
{
ct = _lifecycle.Token;
}
catch (ObjectDisposedException)
{
return; // Stopping.
}
var dbFactory = _dbFactory;
// Off the actor thread, result piped back as a message. A synchronous DB read here would
// block every command dispatch behind a slow or unreachable SQL Server.
Task.Run(async () =>
{
await using var db = await dbFactory.CreateDbContextAsync(ct).ConfigureAwait(false);
// UNFILTERED, with the eligibility flags projected and applied per row below. The two
// outputs of this read want different row sets: the dispatch set is the enabled,
// non-maintenance nodes central talks TO, while the known-cluster gate answers "is this a
// cluster central has ever heard of" and must include a cluster whose every node is
// disabled or in maintenance — such a cluster can still send an ack, and denying it would
// make maintenance mode an authentication switch.
var rows = await db.ClusterNodes
.AsNoTracking()
.Select(n => new
{
n.NodeId, n.ClusterId, n.Host, n.GrpcPort, n.Enabled, n.MaintenanceMode,
})
.ToListAsync(ct)
.ConfigureAwait(false);
// The gRPC dispatch set: one entry per eligible row that carries a port.
var targets = new List(rows.Count);
var portless = new List();
// Every cluster with ANY ClusterNode row — the known-cluster gate's membership set, and
// deliberately a superset of the fan set. Independent of Enabled, MaintenanceMode and
// GrpcPort alike: none of those describe whether the cluster exists.
var clusterIds = new HashSet(StringComparer.OrdinalIgnoreCase);
foreach (var row in rows)
{
clusterIds.Add(row.ClusterId);
// The eligibility filter that used to be the query's WHERE clause. Everything below
// this line — the gRPC targets — sees exactly the rows it saw before.
if (!row.Enabled || row.MaintenanceMode)
continue;
if (row.GrpcPort is { } grpcPort)
targets.Add(new MeshNodeTarget(
row.NodeId, row.ClusterId, $"http://{row.Host}:{grpcPort}"));
else
portless.Add(row.NodeId);
}
return new ContactsLoaded(targets, portless, clusterIds.ToList());
}, ct).PipeTo(self);
}
private void HandleContactsLoaded(ContactsLoaded msg)
{
// Order is load-bearing and is pinned by test: the dispatch set is adopted BEFORE the gate is
// published, so anything that observes the gate can rely on the targets already being current.
if (_useGrpc)
AdoptGrpcTargets(msg);
// EVERY mode. See PreStart — the gate protects an endpoint mapped by port, not by mode.
PublishKnownClusters(msg);
}
///
/// Adopts a contact load's gRPC dispatch set. only
/// — the other two transports have no use for it.
///
/// The load result.
///
/// A host that is unusable as a gRPC endpoint is deliberately NOT screened here: it surfaces
/// as a named per-node dial failure on every dispatch, which is a truer signal than a
/// refresh-time warning about a string that may well be fine.
///
private void AdoptGrpcTargets(ContactsLoaded msg)
{
// Rebuilt against this refresh's skip list, not merely added to. A node that gained a port
// and later lost it again must warn a second time — the operator's mental model is "the last
// refresh told me", not "you were told once, in a log that has since rotated away".
_warnedPortlessNodes.IntersectWith(msg.PortlessNodes);
foreach (var nodeId in msg.PortlessNodes)
{
// Warn-once per node id per continuous skip run: a persistently portless row is
// re-skipped every refresh, and that is the repetition TelemetryNodeSource suppresses.
if (_warnedPortlessNodes.Add(nodeId))
{
_log.Warning(
"ClusterNode {NodeId} has no GrpcPort; it exposes no command endpoint and is "
+ "skipped in this dispatch set (further skips of this node are silent). Commands "
+ "will not reach it until the row carries a port", nodeId);
}
}
_targets = msg.Targets;
// R03:1.3 — release the channels of endpoints that left the dispatch set (host renamed, port
// moved). Only the resolved dialer is asked; nothing is resolved just to prune it.
if (_dialer is { } dialer)
{
try
{
dialer.RetainOnly(_targets.Select(t => t.Endpoint).ToHashSet(StringComparer.Ordinal));
}
catch (Exception ex)
{
_log.Warning(ex, "Mesh dialer failed to release channels for retired endpoints; they stay cached until the next refresh");
}
}
if (_targets.Count == 0)
{
_log.Warning(
"No enabled, non-maintenance ClusterNode row carries a GrpcPort in any cluster; there "
+ "are no gRPC command targets and every central→node command will be dropped until a "
+ "refresh finds one");
}
}
///
/// Hands the PSK provider this refresh's cluster inventory and drops its key cache. Runs in
/// every transport mode.
///
/// The load result.
///
///
/// Not gated on , on purpose. The gate feeds
/// CentralCommandAuthInterceptor, which is registered on every admin node, and the
/// ack service it protects is mapped by listener port — neither consults the mode. Gating
/// this on Grpc would mean a dark-switch rollback to
/// Dps left that endpoint serving with a permanently inactive gate: the timing
/// oracle and the pre-authentication store/audit flood would both come back, silently,
/// as a side effect of a rollback nobody would connect to authentication.
///
///
/// InvalidateAll, not per-cluster Invalidate. The requirement is that
/// a rotated key takes effect within one refresh, and only a whole-cache drop delivers
/// that: a key can rotate without the cluster's membership changing at all, so
/// invalidating only departed clusters would serve the pre-rotation key forever. The cost
/// is one store read per cluster per refresh — the cache holds a single entry per cluster.
///
///
/// This is the PSK provider's "first use" call site. is
/// evaluated exactly once per call and its result reused below; evaluating it
/// per-statement would double the memoized property's retry rate for no benefit. A
/// still-null result here means the property's own retry, on top of PreStart's original
/// attempt, has ALSO failed — the genuinely permanent absence — so this is where that
/// gets reported loudly, guarded to once per incarnation because this handler runs on
/// the contact-refresh timer for the actor's entire life and an unguarded Error here
/// would repeat every ContactRefreshSeconds forever.
///
///
private void PublishKnownClusters(ContactsLoaded msg)
{
var pskProvider = PskProvider;
if (pskProvider is null)
{
// Gated on a factory having been supplied at all: a never-supplied factory is a
// deliberate wiring/test choice Resolve() already stayed silent about (see its
// remarks), not a resolve failure, so it earns no Error here either.
// Error only when the scheduler REFUSES (budget spent and this final attempt also
// failed), so a healed boot race stays quiet.
var budgetSpent = _pskProviderFactory is not null && !_gatePublished
&& !ScheduleBootstrapRetry();
if (budgetSpent && !_pskProviderAbsenceReported)
{
_pskProviderAbsenceReported = true;
_log.Error(
"No IMeshPskProvider is resolvable for this actor, even after a first-use retry and "
+ "the bounded bootstrap retries; the known-cluster gate will NEVER be published and cached per-cluster keys "
+ "will NEVER be dropped. Inbound mesh acks stay exposed to the "
+ "pre-authentication secret-store cost the gate removes, and a rotated key "
+ "needs a central restart. Check that the ControlPlane registration wires up "
+ "IMeshPskProvider (this will not be logged again this incarnation)");
}
return;
}
pskProvider.SetKnownClusters(msg.ClusterIds);
// R03:3.4 — refresh the per-cluster keys OFF the request path and swap them in atomically,
// instead of dropping the cache and making the first ack per cluster re-read the store on a
// gRPC thread. A provider that does not pre-resolve (the interface default, and every fake)
// still drops its cache here, synchronously, exactly as before.
CancellationToken ct;
try
{
ct = _lifecycle.Token;
}
catch (ObjectDisposedException)
{
return; // Stopping.
}
var self = Self;
Task refresh;
try
{
refresh = pskProvider.RefreshAsync(msg.ClusterIds, ct);
}
catch (Exception ex)
{
_log.Warning(ex, "Mesh PSK refresh threw synchronously; cached keys are unchanged until the next refresh");
refresh = Task.CompletedTask;
}
if (!refresh.IsCompletedSuccessfully)
{
refresh.PipeTo(self,
success: () => new PskRefreshed(msg.ClusterIds.Count),
failure: ex => new PskRefreshFailed(ex));
}
if (!_gatePublished)
{
_gatePublished = true;
Timers.Cancel(BootstrapRetryTimerKey);
_log.Info(
"Known-cluster gate published ({Count} clusters) after {Attempts} refresh attempt(s)",
msg.ClusterIds.Count, _refreshAttempts);
}
}
/// Internal tick asking for a contact-point refresh.
public sealed record RefreshContacts;
/// The asynchronous PSK refresh (R03:3.4) completed.
/// How many clusters were refreshed.
public sealed record PskRefreshed(int ClusterCount);
/// The asynchronous PSK refresh (R03:3.4) threw; the previous keys stay in service.
/// The failure.
public sealed record PskRefreshFailed(Exception Error);
/// Result of a contact-point load, piped back from the DB read.
///
/// The gRPC dispatch set: one entry per enabled, non-maintenance row that carries a
/// GrpcPort.
///
///
/// Node ids skipped because their row has no GrpcPort. Carried rather than logged in
/// the load task so the warn-once bookkeeping stays on the actor's thread.
///
///
/// Every application Cluster with any ClusterNode row — the PSK
/// provider's known-cluster gate membership, and deliberately a superset of
/// : a cluster whose nodes are all disabled or in maintenance can
/// still send an ack, and refusing it would make maintenance mode an authentication switch.
///
public sealed record ContactsLoaded(
IReadOnlyList Targets,
IReadOnlyList PortlessNodes,
IReadOnlyList ClusterIds);
/// One node the gRPC command plane can dial.
/// The ClusterNode row's id — what a failure log names.
///
/// The node's application Cluster. Selects the pre-shared key presented to it: a node
/// dialled with another cluster's id is graded against the wrong key and refuses the command.
///
/// The h2c base address, http://{Host}:{GrpcPort}.
public sealed record MeshNodeTarget(string NodeId, string ClusterId, string Endpoint);
///
/// The outcome of one node's dial, piped back to the actor for logging only. Nothing
/// retries, counts or waits on these — a deployment's completion is judged by the acks nodes
/// send back, never by whether central's outbound call succeeded.
///
/// The node dialled.
/// CLR type name of the command.
/// The envelope's transport correlation.
///
/// on success; otherwise the gRPC status code's name, or
/// Unknown for a non-gRPC fault.
///
///
/// The gRPC status detail, or the exception's type name for a non-gRPC fault. Never a key,
/// and never a raw exception message.
///
public sealed record CommandDispatched(
string NodeId,
string CommandType,
string CorrelationId,
string? FailureStatus,
string? FailureDetail);
}