using System.Collections.Immutable; using Akka.Actor; using Akka.Cluster.Tools.Client; using Akka.Cluster.Tools.PublishSubscribe; using Akka.Event; using Microsoft.EntityFrameworkCore; using ZB.MOM.WW.OtOpcUa.Cluster; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy; using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh; using ZB.MOM.WW.OtOpcUa.Configuration; namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Communication; /// /// Central-side end of the central↔node command boundary (per-cluster mesh Phase 2). Routes /// every central→node command out — over DistributedPubSub or ClusterClient, per /// — and forwards inbound s to the /// deploy coordinator singleton. /// /// /// /// Registered per node, NOT as a cluster singleton. A driver node's ClusterClient /// rotates across contact points and must find a live comm actor at whichever central node /// answers. A singleton would be reachable only through the one node hosting it, and the /// rotation would fail silently against the other. /// /// /// Exactly one ClusterClient, fleet-wide. See — this is /// the single most important thing to understand before changing this actor. /// /// public sealed class CentralCommunicationActor : ReceiveActor, IWithTimers { private const string RefreshTimerKey = "mesh-contact-refresh"; private readonly IDbContextFactory _dbFactory; private readonly MeshTransportOptions _options; private readonly IMeshClusterClientFactory _clientFactory; private readonly Func _coordinator; private readonly ILoggingAdapter _log = Context.GetLogger(); private readonly bool _useClusterClient; private IActorRef? _client; private ImmutableHashSet _currentContacts = ImmutableHashSet.Empty; private CancellationTokenSource _lifecycle = new(); /// Gets the timer scheduler driving the periodic contact refresh. public ITimerScheduler Timers { get; set; } = null!; /// Creates the props for this actor. /// Factory for the config database holding ClusterNode rows. /// The bound mesh-transport options. /// Creates the ClusterClient; substituted in tests. /// /// Resolves the deploy-coordinator singleton proxy. A rather than /// an so this actor never depends on registration order at wiring /// time — it resolves on first use and caches. /// /// The props. public static Props Props( IDbContextFactory dbFactory, MeshTransportOptions options, IMeshClusterClientFactory clientFactory, Func coordinator) => Akka.Actor.Props.Create(() => new CentralCommunicationActor(dbFactory, options, clientFactory, coordinator)); /// Initializes a new instance of the class. /// Factory for the config database. /// The bound mesh-transport options. /// Creates the ClusterClient. /// Lazily resolves the deploy-coordinator singleton proxy. public CentralCommunicationActor( IDbContextFactory dbFactory, MeshTransportOptions options, IMeshClusterClientFactory clientFactory, Func coordinator) { _dbFactory = dbFactory; _options = options; _clientFactory = clientFactory; _coordinator = coordinator; _useClusterClient = string.Equals( options.Mode, MeshTransportOptions.ModeClusterClient, StringComparison.OrdinalIgnoreCase); Receive(HandleMeshCommand); Receive(HandleApplyAck); Receive(_ => LoadContactsFromDb()); Receive(HandleContactsLoaded); // A faulted LoadContactsFromDb task is piped here as a Status.Failure. Without this handler // the failure is an unhandled message (debug level only) and the refresh fails silently — // an operator cannot distinguish "no nodes configured" from "the database is down", and the // contact set silently freezes at whatever it last held. Receive(f => _log.Warning( f.Cause, "Failed to load ClusterNode contact points; the mesh ClusterClient contact set was NOT " + "refreshed and may be stale or empty. Commands may be dropped until the next refresh")); Receive(_ => { /* DPS subscribe confirmation */ }); } /// protected override void PreStart() { if (!_useClusterClient) { _log.Info( "Mesh transport is {Mode}: commands are published on DistributedPubSub and no " + "ClusterClient is created", _options.Mode); return; } Timers.StartPeriodicTimer( RefreshTimerKey, new RefreshContacts(), TimeSpan.Zero, TimeSpan.FromSeconds(_options.ContactRefreshSeconds)); } /// protected override void PostStop() { _lifecycle.Cancel(); _lifecycle.Dispose(); } private void HandleMeshCommand(MeshCommand cmd) { if (!_useClusterClient) { DistributedPubSub.Get(Context.System).Mediator.Tell(new Publish(cmd.Topic, cmd.Message)); return; } if (_client is null) { // App-level drop, matching the sister project's decision and this repo's buffer-size = 0. // The command is NOT queued and will NOT be retried: a deploy fails at the coordinator's // apply deadline naming the silent nodes, which is a better operator signal than a // command arriving minutes late against a config the DB already recorded as failed. _log.Warning( "No mesh ClusterClient — dropping {MessageType}. The last contact refresh produced " + "no usable contact point from the enabled, non-maintenance ClusterNode rows; the " + "command is not buffered and will not be retried", cmd.Message.GetType().Name); return; } // SendToAll, NEVER Send. Today's DPS publish reaches EVERY DriverHostActor — there is no // ClusterId or node filter on the node side at all; scoping happens later, inside the // artifact (DeploymentArtifact.ResolveClusterScope). ClusterClient.Send delivers to exactly // ONE registered actor, which would deploy to a single node of the fleet while every other // node silently kept its old configuration — and the deployment would still seal green if // the ack set happened to be satisfied. _client.Tell(new ClusterClient.SendToAll(MeshPaths.NodeCommunication, cmd.Message)); } private void HandleApplyAck(ApplyAck ack) { var coordinator = _coordinator(); if (coordinator is null) { _log.Warning( "Received ApplyAck for {DeploymentId} from {NodeId} but the deploy coordinator is " + "not resolvable on this node; the ack is lost and the deployment will time out " + "naming a node that applied successfully", ack.DeploymentId, ack.NodeId); return; } // Forward, not Tell: preserves the original sender across the hop so the coordinator sees // the node rather than this relay. coordinator.Forward(ack); } private void LoadContactsFromDb() { var self = Self; CancellationToken ct; try { ct = _lifecycle.Token; } catch (ObjectDisposedException) { return; // Stopping. } var systemName = Context.System.Name; var dbFactory = _dbFactory; // Off the actor thread, result piped back as a message. A synchronous DB read here would // block every command dispatch behind a slow or unreachable SQL Server. Task.Run(async () => { await using var db = await dbFactory.CreateDbContextAsync(ct).ConfigureAwait(false); var rows = await db.ClusterNodes .AsNoTracking() .Where(n => n.Enabled && !n.MaintenanceMode) .Select(n => new { n.NodeId, n.Host, n.AkkaPort }) .ToListAsync(ct) .ConfigureAwait(false); var contacts = new List(rows.Count); var malformed = new List(); foreach (var row in rows) { var address = $"akka.tcp://{systemName}@{row.Host}:{row.AkkaPort}/system/receptionist"; // Parse up front, per row, inside the loop's own guard. A single malformed row must // not abort the whole refresh and leave the contact set half-built — the sister // project shipped that bug and its regression test deliberately orders the bad row // first. if (ActorPath.TryParse(address, out _)) contacts.Add(address); else malformed.Add($"{row.NodeId} -> {address}"); } return new ContactsLoaded(contacts, malformed); }, ct).PipeTo(self); } private void HandleContactsLoaded(ContactsLoaded msg) { foreach (var bad in msg.Malformed) { _log.Warning( "ClusterNode {Row} does not yield a usable receptionist address; skipping it in this " + "refresh (other nodes are unaffected). Check the row's Host and AkkaPort", bad); } var contacts = msg.Contacts.ToImmutableHashSet(); if (contacts.IsEmpty) { _log.Warning( "No usable ClusterNode contact points; the mesh ClusterClient was not created and " + "every central→node command will be dropped until a refresh finds one"); return; } if (_client is not null && _currentContacts.SetEquals(contacts)) return; RebuildClient(contacts); } /// Stops any existing client and creates a replacement for the new contact set. /// /// /// TODO(Phase 6): one client per application Cluster. Today this actor /// creates exactly one client whose contacts are the whole fleet, and that is not /// a compromise — it is the only correct shape while the fleet is a single Akka mesh. /// /// /// A ClusterClientReceptionist serves its entire cluster, so /// SendToAll reaches every registered node-comm actor in the mesh regardless of /// which node's address was used as the contact point. One client per Cluster would /// therefore fan each command out once per cluster — N× duplicate /// DispatchDeployment and N× duplicate ApplyAck. Phase 6 splits the meshes, /// at which point per-cluster clients become both correct and necessary. /// /// /// Corollary worth stating because it is easy to assume otherwise: the contact set /// does not scope delivery. Excluding a maintenance-mode node from the contacts does /// not stop it receiving commands — it still receives them, exactly as it does under /// today's DPS broadcast. The filter is about which receptionists are worth dialling /// (a node in maintenance may be switched off), not about who gets the message. /// /// /// The receptionist addresses to dial. private void RebuildClient(ImmutableHashSet contacts) { if (_client is not null) { _log.Info("Mesh contact set changed; replacing the ClusterClient"); Context.Stop(_client); // Cleared now: if the create below throws, a stale ref would route commands into a // stopping actor and they would vanish without a warning. _client = null; _currentContacts = ImmutableHashSet.Empty; } try { var paths = contacts.Select(ActorPath.Parse).ToImmutableHashSet(); _client = _clientFactory.Create(Context.System, paths); _currentContacts = contacts; _log.Info("Mesh ClusterClient created with {Count} contact point(s)", contacts.Count); } catch (Exception ex) { _log.Error(ex, "Failed to create the mesh ClusterClient; central→node commands are dropped until " + "the next contact refresh succeeds"); } } /// Internal tick asking for a contact-point refresh. public sealed record RefreshContacts; /// Result of a contact-point load, piped back from the DB read. /// Receptionist addresses that parsed cleanly. /// Rows that did not, described for the log. public sealed record ContactsLoaded( IReadOnlyList Contacts, IReadOnlyList Malformed); }