feat(mesh): one ClusterClient per Cluster in CentralCommunicationActor (Phase 6)

After the fleet splits into one Akka mesh per application Cluster, a ClusterClientReceptionist serves only its own mesh — so the single fleet-wide client's SendToAll reached only the mesh whose receptionist answered, leaving every other cluster silently on its old configuration. Central now holds one ClusterClient per ClusterId and fans SendToAll across all of them. LoadContactsFromDb selects ClusterId and groups the receptionist contacts per cluster (keeping the per-row TryParse guard and the enabled/non-maintenance filter); ContactsLoaded carries ContactsByCluster; HandleContactsLoaded diffs per cluster (rebuild changed, stop+drop vanished, warn-don't-create on empty); RebuildClient builds a per-cluster client under a clusterId-derived actor name. IMeshClusterClientFactory.Create gained a clusterId parameter so the per-cluster clients get distinct, diagnosable names. ApplyAck handling and the DPS branch are unchanged; the deploy path stays payload-free. Tests: focused unit coverage of the grouping + fan-out (one-client-per-cluster, fan-out SendToAll to every cluster client) via the recording factory double, plus the real two-mesh boundary test extended to central + two separate site meshes proving one dispatch reaches both and a client scoped to one cluster never crosses into another. Red-before-green verified: crippling the fan-out to a single client fails the reaches-both-meshes assertion. Claude-Session: https://claude.ai/code/session_01GASWkNEi68FSCtvr6rLoEW
2026-07-24 01:53:20 -04:00
parent 2535df019b
commit 6531ec1984
5 changed files with 358 additions and 75 deletions
@@ -31,13 +31,20 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase

        public List<ImmutableHashSet<ActorPath>> Calls { get; } = [];

+        public List<string> ClusterIds { get; } = [];
+
        public List<TestProbe> Probes { get; } = [];

-        public IActorRef Create(ActorSystem system, ImmutableHashSet<ActorPath> contacts)
+        /// <summary>Probe created for a given cluster id, for per-cluster fan-out assertions.</summary>
+        public Dictionary<string, TestProbe> ProbeByCluster { get; } = [];
+
+        public IActorRef Create(ActorSystem system, string clusterId, ImmutableHashSet<ActorPath> contacts)
        {
            Calls.Add(contacts);
+            ClusterIds.Add(clusterId);
            var probe = _newProbe();
            Probes.Add(probe);
+            ProbeByCluster[clusterId] = probe;
            return probe.Ref;
        }
    }
@@ -138,23 +145,54 @@ public class CentralCommunicationActorTests : ControlPlaneActorTestBase
    }

    [Fact]
-    public void Exactly_one_client_is_created_for_a_multi_cluster_fleet()
+    public void One_client_per_cluster_is_created_for_a_multi_cluster_fleet()
    {
-        // A receptionist serves its whole mesh, so one client per Cluster would fan each command out
-        // once per cluster while the fleet is still a single mesh — N x duplicate deployments.
+        // THE Phase 6 inversion. Each application Cluster is now its own Akka mesh, and a ClusterClient
+        // reaches only the mesh whose receptionist it dialled. Central must therefore build one client
+        // per cluster, each dialling only that cluster's nodes.
        var factory = NewInMemoryDbFactory();
        SeedCluster(factory, "C1", ("host-a:4053", "host-a", 4053, true, false));
        SeedCluster(factory, "C2", ("host-b:4053", "host-b", 4053, true, false));
        var clients = new RecordingClientFactory(() => CreateTestProbe());

        Spawn(factory, ClusterClientMode(), clients);
-        AwaitCondition(() => clients.Calls.Count == 1, TimeSpan.FromSeconds(5));
+        AwaitCondition(() => clients.Calls.Count == 2, TimeSpan.FromSeconds(5));

-        // ...and that one client dials BOTH clusters' nodes.
-        clients.Calls[0].Count.ShouldBe(2);
-        // Give a second refresh a chance to (wrongly) create another.
+        // Give a second refresh a chance to (wrongly) create more.
        ExpectNoMsg(TimeSpan.FromMilliseconds(300));
-        clients.Calls.Count.ShouldBe(1);
+        clients.Calls.Count.ShouldBe(2);
+        clients.ClusterIds.ShouldBe(new[] { "C1", "C2" }, ignoreOrder: true);
+
+        // Each client dials ONLY its own cluster's node.
+        clients.Calls[clients.ClusterIds.IndexOf("C1")].Single().ToString().ShouldContain("host-a");
+        clients.Calls[clients.ClusterIds.IndexOf("C2")].Single().ToString().ShouldContain("host-b");
+    }
+
+    [Fact]
+    public void A_command_fans_out_a_send_to_all_to_every_cluster_client()
+    {
+        // The other half of the inversion: one MeshCommand must reach EVERY cluster's mesh, so central
+        // fans SendToAll across all per-cluster clients. A single Send/one-client shape would leave
+        // every cluster but one silently on its old configuration.
+        var factory = NewInMemoryDbFactory();
+        SeedCluster(factory, "C1", ("host-a:4053", "host-a", 4053, true, false));
+        SeedCluster(factory, "C2", ("host-b:4053", "host-b", 4053, true, false));
+        var clients = new RecordingClientFactory(() => CreateTestProbe());
+
+        var actor = Spawn(factory, ClusterClientMode(), clients);
+        AwaitCondition(() => clients.Probes.Count == 2, TimeSpan.FromSeconds(5));
+
+        var dispatch = new DispatchDeployment(
+            new DeploymentId(Guid.NewGuid()), new RevisionHash("abc"), new CorrelationId(Guid.NewGuid()));
+        actor.Tell(new MeshCommand(DeploymentsTopic, dispatch));
+
+        foreach (var (clusterId, probe) in clients.ProbeByCluster)
+        {
+            var sent = probe.ExpectMsg<ClusterClient.SendToAll>(
+                TimeSpan.FromSeconds(5), $"cluster {clusterId} must receive the fanned SendToAll");
+            sent.Path.ShouldBe(MeshPaths.NodeCommunication);
+            sent.Message.ShouldBe(dispatch);
+        }
    }

    [Fact]
@@ -37,13 +37,13 @@ public class MeshClusterClientFactoryTests : TestKit
    {
        var factory = new DefaultMeshClusterClientFactory();

-        var first = factory.Create(Sys, UnreachableContact);
+        var first = factory.Create(Sys, "C1", UnreachableContact);
        Sys.Stop(first);

        // Stop is asynchronous: `first`'s name is still reserved right here. Without the generation
        // suffix this throws InvalidActorNameException — which is precisely what a contact-set
        // change would do in production, where stop-then-recreate happens in one handler.
-        var second = factory.Create(Sys, UnreachableContact);
+        var second = factory.Create(Sys, "C1", UnreachableContact);

        second.Path.Name.ShouldNotBe(first.Path.Name);
    }
@@ -54,12 +54,27 @@ public class MeshClusterClientFactoryTests : TestKit
        var factory = new DefaultMeshClusterClientFactory();

        var names = Enumerable.Range(0, 5)
-            .Select(_ => factory.Create(Sys, UnreachableContact).Path.Name)
+            .Select(_ => factory.Create(Sys, "C1", UnreachableContact).Path.Name)
            .ToList();

        names.Distinct().Count().ShouldBe(5);
    }

+    [Fact]
+    public void The_cluster_id_is_woven_into_the_client_actor_name()
+    {
+        // Per-cluster clients must be distinguishable in logs and the actor tree, and — more
+        // importantly — two clusters' clients created on one system must never collide on a name.
+        var factory = new DefaultMeshClusterClientFactory();
+
+        var a = factory.Create(Sys, "site-a", UnreachableContact);
+        var b = factory.Create(Sys, "site-b", UnreachableContact);
+
+        a.Path.Name.ShouldContain("site-a");
+        b.Path.Name.ShouldContain("site-b");
+        a.Path.Name.ShouldNotBe(b.Path.Name);
+    }
+
    // Contact-set propagation is deliberately NOT asserted here: reading initial contacts back
    // off a created ClusterClient is not possible, and asserting ClusterClientSettings directly
    // would test Akka rather than this factory. It is covered where it can actually fail — the
@@ -3,12 +3,16 @@ using Akka.Actor;
 using Akka.Cluster;
 using Akka.Cluster.Tools.Client;
 using Akka.Configuration;
+using Microsoft.EntityFrameworkCore;
 using Shouldly;
 using Xunit;
+using ZB.MOM.WW.OtOpcUa.Cluster;
 using ZB.MOM.WW.OtOpcUa.Commons.Messages.Admin;
 using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
 using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
 using ZB.MOM.WW.OtOpcUa.Commons.Types;
+using ZB.MOM.WW.OtOpcUa.Configuration;
+using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
 using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
 using ZB.MOM.WW.OtOpcUa.Runtime.Communication;
 using AkkaCluster = Akka.Cluster.Cluster;
@@ -86,12 +90,12 @@ public sealed class MeshClusterClientBoundaryTests : IAsyncLifetime
        ClusterClientReceptionist.Get(_central).RegisterService(centralComm);

        // Node: the REAL comm actor, dialling central through the REAL production client factory.
-        var nodeToCentral = _clientFactory.Create(_node, ReceptionistOf(_central));
+        var nodeToCentral = _clientFactory.Create(_node, "central", ReceptionistOf(_central));
        _nodeComm = _node.ActorOf(
            NodeCommunicationActor.Props(nodeToCentral), MeshPaths.NodeCommunicationName);
        ClusterClientReceptionist.Get(_node).RegisterService(_nodeComm);

-        _centralClient = _clientFactory.Create(_central, ReceptionistOf(_node));
+        _centralClient = _clientFactory.Create(_central, "site-a", ReceptionistOf(_node));
    }

    /// <inheritdoc />
@@ -179,7 +183,7 @@ public sealed class MeshClusterClientBoundaryTests : IAsyncLifetime
            var comm = deaf.ActorOf(NodeCommunicationActor.Props(null), MeshPaths.NodeCommunicationName);
            comm.ShouldNotBeNull();

-            var client = _clientFactory.Create(_central, ReceptionistOf(deaf));
+            var client = _clientFactory.Create(_central, "deaf", ReceptionistOf(deaf));
            var command = new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid());

            var deadline = DateTime.UtcNow + Silence;
@@ -199,6 +203,146 @@ public sealed class MeshClusterClientBoundaryTests : IAsyncLifetime
        }
    }

+    [Fact]
+    public async Task One_dispatch_through_central_reaches_a_node_in_every_cluster_mesh()
+    {
+        // THE Phase 6 assertion at the transport level, driving the REAL CentralCommunicationActor.
+        // Two genuinely separate site meshes (site-A = _node, site-B created here), each its own
+        // single-member cluster with its own receptionist. Central must build ONE ClusterClient per
+        // cluster and fan SendToAll across them, so a single MeshCommand reaches BOTH meshes. A single
+        // fleet-wide client would reach only the one mesh whose receptionist answered — the exact
+        // failure this test is red-before-green against.
+        var siteB = StartSelfSeededSystem(withReceptionist: true);
+        try
+        {
+            await AwaitSingleMemberUpAsync(siteB);
+            RegisterNodeComm(siteB);
+
+            var inboxA = NewInbox();
+            _node.EventStream.Subscribe(_node.ActorOf(CaptureActor.Props(inboxA)), typeof(RestartDriver));
+            var inboxB = NewInbox();
+            siteB.EventStream.Subscribe(siteB.ActorOf(CaptureActor.Props(inboxB)), typeof(RestartDriver));
+
+            var dbFactory = SeedTwoClusterDb(SiteAddress(_node), SiteAddress(siteB));
+            var actor = _central.ActorOf(CentralCommunicationActor.Props(
+                dbFactory, ClusterClientOptions(), new DefaultMeshClusterClientFactory(), () => null));
+
+            var command = new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid());
+            var deadline = DateTime.UtcNow + Arrival;
+            while (DateTime.UtcNow < deadline && !(inboxA.Task.IsCompleted && inboxB.Task.IsCompleted))
+            {
+                actor.Tell(new MeshCommand("deployments", command));
+                await Task.Delay(250, TestContext.Current.CancellationToken);
+            }
+
+            inboxA.Task.IsCompleted.ShouldBeTrue(
+                "site-A's mesh must receive the dispatch through its own per-cluster ClusterClient");
+            inboxB.Task.IsCompleted.ShouldBeTrue(
+                "site-B's mesh must receive the SAME dispatch through its own per-cluster ClusterClient; "
+                + "a single fleet-wide client would have reached only one of the two meshes");
+        }
+        finally
+        {
+            await ShutdownAsync(siteB);
+        }
+    }
+
+    [Fact]
+    public async Task A_client_built_for_one_cluster_does_not_deliver_into_another_clusters_mesh()
+    {
+        // The partition proof for the fan-out: a client dialling ONLY site-A's contacts must land in
+        // site-A's mesh and NEVER cross into site-B's. This is why the per-cluster fan-out is correct
+        // rather than accidentally N x broadcasting — each client is scoped to its own mesh.
+        var siteB = StartSelfSeededSystem(withReceptionist: true);
+        try
+        {
+            await AwaitSingleMemberUpAsync(siteB);
+            RegisterNodeComm(siteB);
+
+            var inboxA = NewInbox();
+            _node.EventStream.Subscribe(_node.ActorOf(CaptureActor.Props(inboxA)), typeof(RestartDriver));
+            var inboxB = NewInbox();
+            siteB.EventStream.Subscribe(siteB.ActorOf(CaptureActor.Props(inboxB)), typeof(RestartDriver));
+
+            var clientA = _clientFactory.Create(_central, "site-a", ReceptionistOf(_node));
+            var command = new RestartDriver("C1", "driver-1", "alice", Guid.NewGuid());
+
+            // Keep sending for the full Silence window so any erroneous cross-delivery into site-B has
+            // every chance to arrive — a pass means the boundary is shut, not that the clock ran out.
+            var deadline = DateTime.UtcNow + Silence;
+            while (DateTime.UtcNow < deadline)
+            {
+                clientA.Tell(new ClusterClient.SendToAll(MeshPaths.NodeCommunication, command));
+                await Task.Delay(250, TestContext.Current.CancellationToken);
+            }
+
+            inboxA.Task.IsCompleted.ShouldBeTrue("site-A's client must deliver into site-A's mesh");
+            inboxB.Task.IsCompleted.ShouldBeFalse(
+                "a client built for site-A's contacts must NOT deliver into site-B's mesh — the "
+                + "per-cluster fan-out must stay partitioned, one client per mesh");
+        }
+        finally
+        {
+            await ShutdownAsync(siteB);
+        }
+    }
+
+    /// <summary>Registers a real node-comm actor with a system's receptionist so central can reach it.</summary>
+    /// <param name="system">The site system to register into.</param>
+    private static void RegisterNodeComm(ActorSystem system)
+    {
+        var comm = system.ActorOf(NodeCommunicationActor.Props(null), MeshPaths.NodeCommunicationName);
+        ClusterClientReceptionist.Get(system).RegisterService(comm);
+    }
+
+    private static (string Host, int Port) SiteAddress(ActorSystem system)
+    {
+        var addr = AkkaCluster.Get(system).SelfAddress;
+        return (addr.Host!, addr.Port!.Value);
+    }
+
+    private static MeshTransportOptions ClusterClientOptions() => new()
+    {
+        Mode = MeshTransportOptions.ModeClusterClient,
+        CentralContactPoints = ["akka.tcp://otopcua@127.0.0.1:4053"],
+        ContactRefreshSeconds = 1,
+    };
+
+    private static IDbContextFactory<OtOpcUaConfigDbContext> SeedTwoClusterDb(
+        (string Host, int Port) siteA, (string Host, int Port) siteB)
+    {
+        var factory = new InMemoryConfigDbFactory(Guid.NewGuid().ToString("N"));
+        using var db = factory.CreateDbContext();
+        db.ClusterNodes.Add(NodeRow("SITE-A", "node-a", siteA));
+        db.ClusterNodes.Add(NodeRow("SITE-B", "node-b", siteB));
+        db.SaveChanges();
+        return factory;
+    }
+
+    private static ClusterNode NodeRow(string clusterId, string nodeId, (string Host, int Port) addr) => new()
+    {
+        NodeId = nodeId,
+        ClusterId = clusterId,
+        Host = addr.Host,
+        AkkaPort = addr.Port,
+        ApplicationUri = $"urn:{nodeId}",
+        Enabled = true,
+        MaintenanceMode = false,
+        CreatedBy = "test",
+    };
+
+    private sealed class InMemoryConfigDbFactory(string dbName)
+        : IDbContextFactory<OtOpcUaConfigDbContext>
+    {
+        public OtOpcUaConfigDbContext CreateDbContext()
+        {
+            var opts = new DbContextOptionsBuilder<OtOpcUaConfigDbContext>()
+                .UseInMemoryDatabase(dbName)
+                .Options;
+            return new OtOpcUaConfigDbContext(opts);
+        }
+    }
+
    /// <summary>
    ///     Starts a single-member cluster on a dynamic port, optionally without the receptionist.
    /// </summary>