dohertj2/scadalink-design
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: replace ActorSelection with ClusterClient for inter-cluster communication

Browse Source 
Central and site clusters now communicate via ClusterClient/
ClusterClientReceptionist instead of direct ActorSelection. Both
CentralCommunicationActor and SiteCommunicationActor are registered
with their cluster's receptionist. Central creates one ClusterClient
per site using NodeA/NodeB contact points from the DB. Sites configure
multiple CentralContactPoints for automatic failover between central
nodes. ISiteClientFactory enables test injection.
This commit is contained in:
Joseph Doherty
2026-03-18 00:08:47 -04:00
parent e5eb871961
commit 4f22ca2b1f
15 changed files with 287 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

129
tests/ScadaLink.Communication.Tests/CentralCommunicationActorTests.cs
View File

@@ -1,4 +1,6 @@
using System.Collections.Immutable;
using Akka.Actor;
using Akka.Cluster.Tools.Client;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.DependencyInjection;
using NSubstitute;
@@ -9,19 +11,21 @@ using ScadaLink.Commons.Messages.Deployment;
using ScadaLink.Commons.Messages.DebugView;
using ScadaLink.Commons.Messages.Health;
using ScadaLink.Communication.Actors;
using Akka.TestKit;
namespace ScadaLink.Communication.Tests;
/// <summary>
/// Tests for CentralCommunicationActor with database-driven site addressing.
/// WP-4: Message routing via site address cache loaded from DB.
/// Tests for CentralCommunicationActor with per-site ClusterClient routing.
/// WP-4: Message routing via ClusterClient instances created per site.
/// WP-5: Connection failure and failover handling.
/// </summary>
public class CentralCommunicationActorTests : TestKit
{
public CentralCommunicationActorTests() : base(@"akka.loglevel = DEBUG") { }
private (IActorRef actor, ISiteRepository mockRepo) CreateActorWithMockRepo(IEnumerable<Site>? sites = null)
private (IActorRef actor, ISiteRepository mockRepo, Dictionary<string, TestProbe> siteProbes) CreateActorWithMockRepo(
IEnumerable<Site>? sites = null)
{
var mockRepo = Substitute.For<ISiteRepository>();
mockRepo.GetAllSitesAsync(Arg.Any<CancellationToken>())
@@ -31,37 +35,76 @@ public class CentralCommunicationActorTests : TestKit
services.AddScoped(_ => mockRepo);
var sp = services.BuildServiceProvider();
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp)));
return (actor, mockRepo);
var siteProbes = new Dictionary<string, TestProbe>();
var mockFactory = Substitute.For<ISiteClientFactory>();
mockFactory.Create(Arg.Any<ActorSystem>(), Arg.Any<string>(), Arg.Any<ImmutableHashSet<ActorPath>>())
.Returns(callInfo =>
{
var siteId = callInfo.ArgAt<string>(1);
var probe = CreateTestProbe();
siteProbes[siteId] = probe;
return probe.Ref;
});
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, mockFactory)));
return (actor, mockRepo, siteProbes);
}
private Site CreateSite(string identifier, string? nodeAPath, string? nodeBPath = null) =>
new("Test Site", identifier) { NodeAAddress = nodeAPath, NodeBAddress = nodeBPath };
private (IActorRef actor, ISiteRepository mockRepo, Dictionary<string, TestProbe> siteProbes, ISiteClientFactory mockFactory) CreateActorWithFactory(
IEnumerable<Site>? sites = null)
{
var mockRepo = Substitute.For<ISiteRepository>();
mockRepo.GetAllSitesAsync(Arg.Any<CancellationToken>())
.Returns(sites?.ToList() ?? new List<Site>());
var services = new ServiceCollection();
services.AddScoped(_ => mockRepo);
var sp = services.BuildServiceProvider();
var siteProbes = new Dictionary<string, TestProbe>();
var mockFactory = Substitute.For<ISiteClientFactory>();
mockFactory.Create(Arg.Any<ActorSystem>(), Arg.Any<string>(), Arg.Any<ImmutableHashSet<ActorPath>>())
.Returns(callInfo =>
{
var siteId = callInfo.ArgAt<string>(1);
var probe = CreateTestProbe();
siteProbes[siteId] = probe;
return probe.Ref;
});
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, mockFactory)));
return (actor, mockRepo, siteProbes, mockFactory);
}
private Site CreateSite(string identifier, string? nodeAAddress, string? nodeBAddress = null) =>
new("Test Site", identifier) { NodeAAddress = nodeAAddress, NodeBAddress = nodeBAddress };
[Fact]
public void DatabaseDrivenRouting_RoutesToConfiguredSite()
public void ClusterClientRouting_RoutesToConfiguredSite()
{
var probe = CreateTestProbe();
var site = CreateSite("site1", probe.Ref.Path.ToString());
var (actor, _) = CreateActorWithMockRepo(new[] { site });
var site = CreateSite("site1", "akka.tcp://scadalink@host:8082");
var (actor, _, siteProbes) = CreateActorWithMockRepo(new[] { site });
// Send explicit refresh and wait for async DB load + PipeTo
actor.Tell(new RefreshSiteAddresses());
// Wait for auto-refresh (PreStart schedules with TimeSpan.Zero initial delay)
Thread.Sleep(1000);
var command = new DeployInstanceCommand(
"dep1", "inst1", "hash1", "{}", "admin", DateTimeOffset.UtcNow);
actor.Tell(new SiteEnvelope("site1", command));
probe.ExpectMsg<DeployInstanceCommand>(msg => msg.DeploymentId == "dep1");
// The site1 probe (acting as ClusterClient) should receive a ClusterClient.Send
var msg = siteProbes["site1"].ExpectMsg<ClusterClient.Send>();
Assert.Equal("/user/site-communication", msg.Path);
Assert.IsType<DeployInstanceCommand>(msg.Message);
Assert.Equal("dep1", ((DeployInstanceCommand)msg.Message).DeploymentId);
}
[Fact]
public void UnconfiguredSite_MessageIsDropped()
{
var (actor, _) = CreateActorWithMockRepo();
var (actor, _, _) = CreateActorWithMockRepo();
actor.Tell(new RefreshSiteAddresses());
// Wait for auto-refresh
Thread.Sleep(1000);
var command = new DeployInstanceCommand(
@@ -74,11 +117,10 @@ public class CentralCommunicationActorTests : TestKit
[Fact]
public void ConnectionLost_DebugStreamsKilled()
{
var siteProbe = CreateTestProbe();
var site = CreateSite("site1", siteProbe.Ref.Path.ToString());
var (actor, _) = CreateActorWithMockRepo(new[] { site });
var site = CreateSite("site1", "akka.tcp://scadalink@host:8082");
var (actor, _, siteProbes) = CreateActorWithMockRepo(new[] { site });
actor.Tell(new RefreshSiteAddresses());
// Wait for auto-refresh
Thread.Sleep(1000);
// Subscribe to debug view (tracks the subscription)
@@ -86,6 +128,9 @@ public class CentralCommunicationActorTests : TestKit
var subRequest = new SubscribeDebugViewRequest("inst1", "corr-123");
actor.Tell(new SiteEnvelope("site1", subRequest), subscriberProbe.Ref);
// The ClusterClient probe receives the routed message
siteProbes["site1"].ExpectMsg<ClusterClient.Send>();
// Simulate site disconnection
actor.Tell(new ConnectionStateChanged("site1", false, DateTimeOffset.UtcNow));
@@ -97,7 +142,6 @@ public class CentralCommunicationActorTests : TestKit
[Fact]
public void Heartbeat_ForwardedToParent()
{
// Actor still needs IServiceProvider even though this test doesn't use routing
var mockRepo = Substitute.For<ISiteRepository>();
mockRepo.GetAllSitesAsync(Arg.Any<CancellationToken>())
.Returns(new List<Site>());
@@ -106,9 +150,10 @@ public class CentralCommunicationActorTests : TestKit
services.AddScoped(_ => mockRepo);
var sp = services.BuildServiceProvider();
var siteClientFactory = Substitute.For<ISiteClientFactory>();
var parentProbe = CreateTestProbe();
var centralActor = parentProbe.ChildActorOf(
Props.Create(() => new CentralCommunicationActor(sp)));
Props.Create(() => new CentralCommunicationActor(sp, siteClientFactory)));
var heartbeat = new HeartbeatMessage("site1", "host1", true, DateTimeOffset.UtcNow);
centralActor.Tell(heartbeat);
@@ -119,24 +164,21 @@ public class CentralCommunicationActorTests : TestKit
[Fact]
public void RefreshSiteAddresses_UpdatesCache()
{
var probe1 = CreateTestProbe();
var probe2 = CreateTestProbe();
var site1 = CreateSite("site1", "akka.tcp://scadalink@host1:8082");
var (actor, mockRepo, siteProbes) = CreateActorWithMockRepo(new[] { site1 });
var site1 = CreateSite("site1", probe1.Ref.Path.ToString());
var (actor, mockRepo) = CreateActorWithMockRepo(new[] { site1 });
// Wait for initial load, then send explicit refresh
actor.Tell(new RefreshSiteAddresses());
// Wait for initial load
Thread.Sleep(1000);
// Verify routing to site1 works
var cmd1 = new DeployInstanceCommand(
"dep1", "inst1", "hash1", "{}", "admin", DateTimeOffset.UtcNow);
actor.Tell(new SiteEnvelope("site1", cmd1));
probe1.ExpectMsg<DeployInstanceCommand>(msg => msg.DeploymentId == "dep1");
var msg1 = siteProbes["site1"].ExpectMsg<ClusterClient.Send>();
Assert.Equal("dep1", ((DeployInstanceCommand)msg1.Message).DeploymentId);
// Update mock repo to return both sites
var site2 = CreateSite("site2", probe2.Ref.Path.ToString());
var site2 = CreateSite("site2", "akka.tcp://scadalink@host2:8082");
mockRepo.GetAllSitesAsync(Arg.Any<CancellationToken>())
.Returns(new List<Site> { site1, site2 });
@@ -148,23 +190,26 @@ public class CentralCommunicationActorTests : TestKit
var cmd2 = new DeployInstanceCommand(
"dep2", "inst2", "hash2", "{}", "admin", DateTimeOffset.UtcNow);
actor.Tell(new SiteEnvelope("site2", cmd2));
probe2.ExpectMsg<DeployInstanceCommand>(msg => msg.DeploymentId == "dep2");
var msg2 = siteProbes["site2"].ExpectMsg<ClusterClient.Send>();
Assert.Equal("dep2", ((DeployInstanceCommand)msg2.Message).DeploymentId);
}
[Fact]
public void NodeBFallback_WhenNodeANotConfigured()
public void BothContactPoints_UsedInSingleClient()
{
var probe = CreateTestProbe();
var site = CreateSite("site1", null, probe.Ref.Path.ToString());
var (actor, _) = CreateActorWithMockRepo(new[] { site });
var site = CreateSite("site1",
"akka.tcp://scadalink@host1:8082",
"akka.tcp://scadalink@host2:8082");
actor.Tell(new RefreshSiteAddresses());
var (actor, _, siteProbes, mockFactory) = CreateActorWithFactory(new[] { site });
// Wait for auto-refresh
Thread.Sleep(1000);
var command = new DeployInstanceCommand(
"dep1", "inst1", "hash1", "{}", "admin", DateTimeOffset.UtcNow);
actor.Tell(new SiteEnvelope("site1", command));
probe.ExpectMsg<DeployInstanceCommand>(msg => msg.DeploymentId == "dep1");
// Verify the factory was called with 2 contact paths
mockFactory.Received(1).Create(
Arg.Any<ActorSystem>(),
Arg.Is("site1"),
Arg.Is<ImmutableHashSet<ActorPath>>(paths => paths.Count == 2));
}
}