dohertj2/ScadaBridge
1
0
Fork 0
Code Issues 2 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat(comm): Phase 4 — delete Akka ClusterClient site↔central transport, gRPC-only

Browse Source 
ClusterClient→gRPC migration Phase 4 (docs/plans/2026-07-22-clusterclient-to-grpc-plan.md).
Phases 2/3 proved both directions on gRPC; this removes the Akka transport underneath.

Deleted:
- AkkaCentralTransport, AkkaSiteTransport (+ their dedicated tests)
- ISiteClientFactory + DefaultSiteClientFactory; CentralCommunicationActor legacy
  ctor + SelectTransport (Host now builds GrpcSiteTransport and injects it)
- ClusterClient creation + both ClusterClientReceptionist.RegisterService calls in
  AkkaHostedService; the RegisterCentralClient message + receive block
- CommunicationOptions.CentralContactPoints; the CentralTransport/SiteTransport
  coexistence flags; the CentralTransportMode/SiteTransportKind enums

gRPC is now the only site↔central transport (site→central CentralControlService via
GrpcCentralTransport; central→site SiteCommandService via GrpcSiteTransport), both
built unconditionally by the Host. NoOpCentralTransport is the fail-loud null-default
so TestKit command-dispatch suites still construct the site actor without a wired
transport; production always injects GrpcCentralTransport.

Config: CentralGrpcEndpoints is now unconditional — CommunicationOptionsValidator
rejects blank entries (role-agnostic), and StartupValidator requires a Site node to
list >=1 endpoint (fail-fast, mirrors GrpcPsk). Rig configs moved
CentralContactPoints -> CentralGrpcEndpoints (docker x6, docker-env2 x2, Host default,
deploy/wonder-app-vd03). Kept Akka.Cluster.Tools (ClusterSingleton still used).

Tests: build 0/0; Communication.Tests 640, Host.Tests 421 green. Removed the
ClusterClient.Send per-site-routing tests (covered by the transport suites), swapped
the ISiteClientFactory-based ctors to a substitute ISiteCommandTransport, converted
the audit-push integration relay to an in-process bridge transport.

Docs: Component-Communication/Host/StoreAndForward, components/Communication,
topology-guide, grpc_streams (SUPERSEDED note), the frame-size known-issue (retired
amendment), and CLAUDE.md transport decisions.

Not included: the dead IntegrationCallRequest path (#32) is a separate user-owned
behavioral decision — SiteEnvelope routing is transport-agnostic so it still compiles.
This commit is contained in:
Joseph Doherty
2026-07-23 12:54:32 -04:00
parent 3a8ddb7087
commit 7fd5cb2b56
42 changed files with 479 additions and 1295 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/ZB.MOM.WW.ScadaBridge.Communication.Tests/CentralCommunicationActorTests.cs
+22 -186
View File
@@ -1,6 +1,4 @@
using System.Collections.Immutable;
using Akka.Actor;
using Akka.Cluster.Tools.Client;
using Akka.TestKit.Xunit2;
using Microsoft.Extensions.DependencyInjection;
using NSubstitute;
@@ -18,15 +16,19 @@ using Akka.TestKit;
namespace ZB.MOM.WW.ScadaBridge.Communication.Tests;
/// <summary>
/// Tests for CentralCommunicationActor with per-site ClusterClient routing.
/// WP-4: Message routing via ClusterClient instances created per site.
/// WP-5: Connection failure and failover handling.
/// Tests for <see cref="CentralCommunicationActor"/> behaviour that is independent of the
/// central→site command transport: heartbeat/replica aggregation, notification-outbox proxying,
/// and the DB-refresh failure path. The per-site routing itself (envelope → transport → the right
/// site) is the transport's job and is covered by
/// <see cref="CentralCommunicationActorTransportTests"/> and the <c>GrpcSiteTransport</c> suites;
/// the old ClusterClient-routing tests here were removed with the Akka transport in the
/// ClusterClient→gRPC migration's Phase 4.
/// </summary>
public class CentralCommunicationActorTests : TestKit
{
public CentralCommunicationActorTests() : base(@"akka.loglevel = DEBUG") { }
private (IActorRef actor, ISiteRepository mockRepo, Dictionary<string, TestProbe> siteProbes) CreateActorWithMockRepo(
private (IActorRef actor, ISiteRepository mockRepo) CreateActorWithMockRepo(
IEnumerable<Site>? sites = null)
{
var mockRepo = Substitute.For<ISiteRepository>();
@@ -37,93 +39,11 @@ public class CentralCommunicationActorTests : TestKit
services.AddScoped(_ => mockRepo);
var sp = services.BuildServiceProvider();
var siteProbes = new Dictionary<string, TestProbe>();
var mockFactory = Substitute.For<ISiteClientFactory>();
mockFactory.Create(Arg.Any<ActorSystem>(), Arg.Any<string>(), Arg.Any<ImmutableHashSet<ActorPath>>())
.Returns(callInfo =>
{
var siteId = callInfo.ArgAt<string>(1);
var probe = CreateTestProbe();
siteProbes[siteId] = probe;
return probe.Ref;
});
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, mockFactory)));
return (actor, mockRepo, siteProbes);
var transport = Substitute.For<ISiteCommandTransport>();
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, transport, (TimeSpan?)null)));
return (actor, mockRepo);
}
private (IActorRef actor, ISiteRepository mockRepo, Dictionary<string, TestProbe> siteProbes, ISiteClientFactory mockFactory) CreateActorWithFactory(
IEnumerable<Site>? sites = null)
{
var mockRepo = Substitute.For<ISiteRepository>();
mockRepo.GetAllSitesAsync(Arg.Any<CancellationToken>())
.Returns(sites?.ToList() ?? new List<Site>());
var services = new ServiceCollection();
services.AddScoped(_ => mockRepo);
var sp = services.BuildServiceProvider();
var siteProbes = new Dictionary<string, TestProbe>();
var mockFactory = Substitute.For<ISiteClientFactory>();
mockFactory.Create(Arg.Any<ActorSystem>(), Arg.Any<string>(), Arg.Any<ImmutableHashSet<ActorPath>>())
.Returns(callInfo =>
{
var siteId = callInfo.ArgAt<string>(1);
var probe = CreateTestProbe();
siteProbes[siteId] = probe;
return probe.Ref;
});
var actor = Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, mockFactory)));
return (actor, mockRepo, siteProbes, mockFactory);
}
private Site CreateSite(string identifier, string? nodeAAddress, string? nodeBAddress = null) =>
new("Test Site", identifier) { NodeAAddress = nodeAAddress, NodeBAddress = nodeBAddress };
[Fact]
public void ClusterClientRouting_RefreshDeploymentCommand_RoutesToSite()
{
var site = CreateSite("site1", "akka.tcp://scadabridge@host:8082");
var (actor, _, siteProbes) = CreateActorWithMockRepo(new[] { site });
Thread.Sleep(1000);
var command = new RefreshDeploymentCommand(
"dep1", "inst1", "rev1", "admin", DateTimeOffset.UtcNow,
"https://central:9000", "tok1");
actor.Tell(new SiteEnvelope("site1", command));
var msg = siteProbes["site1"].ExpectMsg<ClusterClient.Send>();
Assert.Equal("/user/site-communication", msg.Path);
Assert.IsType<RefreshDeploymentCommand>(msg.Message);
Assert.Equal("dep1", ((RefreshDeploymentCommand)msg.Message).DeploymentId);
}
[Fact]
public void UnconfiguredSite_MessageIsDropped()
{
var (actor, _, _) = CreateActorWithMockRepo();
// Wait for auto-refresh
Thread.Sleep(1000);
var command = new RefreshDeploymentCommand(
"dep1", "inst1", "hash1", "admin", DateTimeOffset.UtcNow,
"https://central:9000", "tok1");
actor.Tell(new SiteEnvelope("unknown-site", command));
ExpectNoMsg(TimeSpan.FromMilliseconds(200));
}
// Communication-016: the prior `ConnectionLost_DebugStreamsKilled` test was
// removed alongside the dead HandleConnectionStateChanged handler. No
// production code ever emitted ConnectionStateChanged, so the test was
// exercising a workflow that never ran. Disconnect detection is owned by
// the gRPC keepalive (DebugStreamBridgeActor self-terminates) and by the
// Ask-timeout path at the CommunicationService layer (deploy callers see
// a failure).
[Fact]
public void Heartbeat_BumpsAggregatorTimestamp()
{
@@ -138,9 +58,9 @@ public class CentralCommunicationActorTests : TestKit
services.AddSingleton(aggregator);
var sp = services.BuildServiceProvider();
var siteClientFactory = Substitute.For<ISiteClientFactory>();
var transport = Substitute.For<ISiteCommandTransport>();
var centralActor = Sys.ActorOf(
Props.Create(() => new CentralCommunicationActor(sp, siteClientFactory)));
Props.Create(() => new CentralCommunicationActor(sp, transport, (TimeSpan?)null)));
var timestamp = DateTimeOffset.UtcNow;
centralActor.Tell(new HeartbeatMessage("site1", "host1", true, timestamp));
@@ -165,9 +85,9 @@ public class CentralCommunicationActorTests : TestKit
services.AddSingleton(aggregator);
var sp = services.BuildServiceProvider();
var siteClientFactory = Substitute.For<ISiteClientFactory>();
var transport = Substitute.For<ISiteCommandTransport>();
var centralActor = Sys.ActorOf(
Props.Create(() => new CentralCommunicationActor(sp, siteClientFactory)));
Props.Create(() => new CentralCommunicationActor(sp, transport, (TimeSpan?)null)));
var ts = DateTimeOffset.UtcNow;
centralActor.Tell(new SiteHeartbeatReplica(new HeartbeatMessage("site-1", "host-a", true, ts)));
@@ -175,41 +95,6 @@ public class CentralCommunicationActorTests : TestKit
AwaitAssert(() => aggregator.Received(1).MarkHeartbeat("site-1", ts));
}
[Fact]
public void RefreshSiteAddresses_UpdatesCache()
{
var site1 = CreateSite("site1", "akka.tcp://scadabridge@host1:8082");
var (actor, mockRepo, siteProbes) = CreateActorWithMockRepo(new[] { site1 });
// Wait for initial load
Thread.Sleep(1000);
// Verify routing to site1 works
var cmd1 = new RefreshDeploymentCommand(
"dep1", "inst1", "hash1", "admin", DateTimeOffset.UtcNow,
"https://central:9000", "tok1");
actor.Tell(new SiteEnvelope("site1", cmd1));
var msg1 = siteProbes["site1"].ExpectMsg<ClusterClient.Send>();
Assert.Equal("dep1", ((RefreshDeploymentCommand)msg1.Message).DeploymentId);
// Update mock repo to return both sites
var site2 = CreateSite("site2", "akka.tcp://scadabridge@host2:8082");
mockRepo.GetAllSitesAsync(Arg.Any<CancellationToken>())
.Returns(new List<Site> { site1, site2 });
// Refresh again
actor.Tell(new RefreshSiteAddresses());
Thread.Sleep(1000);
// Verify routing to site2 now works
var cmd2 = new RefreshDeploymentCommand(
"dep2", "inst2", "hash2", "admin", DateTimeOffset.UtcNow,
"https://central:9000", "tok2");
actor.Tell(new SiteEnvelope("site2", cmd2));
var msg2 = siteProbes["site2"].ExpectMsg<ClusterClient.Send>();
Assert.Equal("dep2", ((RefreshDeploymentCommand)msg2.Message).DeploymentId);
}
[Fact]
public void LoadSiteAddressesFailure_IsLoggedNotSilentlySwallowed()
{
@@ -226,56 +111,26 @@ public class CentralCommunicationActorTests : TestKit
services.AddScoped(_ => mockRepo);
var sp = services.BuildServiceProvider();
var mockFactory = Substitute.For<ISiteClientFactory>();
var transport = Substitute.For<ISiteCommandTransport>();
// The fix logs a Warning carrying the InvalidOperationException as the cause.
EventFilter.Warning(contains: "Failed to load site addresses from the database").ExpectOne(() =>
{
Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, mockFactory)));
Sys.ActorOf(Props.Create(() => new CentralCommunicationActor(sp, transport, (TimeSpan?)null)));
});
}
[Fact]
public void MalformedSiteAddress_DoesNotAbortRefresh_OtherSitesStillRegistered()
{
// Regression test for Communication-009. HandleSiteAddressCacheLoaded calls
// ActorPath.Parse for every site in a single loop. A malformed NodeAAddress
// throws inside that loop; before the fix the whole refresh aborted partway
// through, leaving the cache half-updated (some sites registered, others not).
// The fix wraps the parse in a try/catch that logs and skips the bad site so
// a single garbage row cannot starve every other site of its ClusterClient.
var goodSite = CreateSite("good-site", "akka.tcp://scadabridge@host1:8082");
// A garbage address that ActorPath.Parse rejects.
var badSite = CreateSite("bad-site", "this is not a valid actor path !!!");
// Order the bad site first so a non-resilient loop aborts before reaching good-site.
var (actor, _, siteProbes) = CreateActorWithMockRepo(new[] { badSite, goodSite });
Thread.Sleep(1000);
// good-site must still be registered and routable despite bad-site failing to parse.
var cmd = new RefreshDeploymentCommand(
"dep1", "inst1", "hash1", "admin", DateTimeOffset.UtcNow,
"https://central:9000", "tok1");
actor.Tell(new SiteEnvelope("good-site", cmd));
Assert.True(siteProbes.ContainsKey("good-site"),
"good-site should have a ClusterClient even though bad-site's address is malformed");
var msg = siteProbes["good-site"].ExpectMsg<ClusterClient.Send>();
Assert.Equal("dep1", ((RefreshDeploymentCommand)msg.Message).DeploymentId);
}
private NotificationSubmit CreateSubmit(string id = "notif1") =>
new(id, "ops-list", "Subject", "Body", "site1", "inst1", "script.cs", DateTimeOffset.UtcNow);
[Fact]
public void NotificationSubmit_ForwardedToOutboxProxy_AckRoutesBackToSite()
{
var (actor, _, _) = CreateActorWithMockRepo();
var (actor, _) = CreateActorWithMockRepo();
var outboxProbe = CreateTestProbe();
actor.Tell(new RegisterNotificationOutbox(outboxProbe.Ref));
// A second probe stands in for the site's ClusterClient (the original Sender).
// A second probe stands in for the site (the original Sender).
var siteProbe = CreateTestProbe();
var submit = CreateSubmit();
actor.Tell(submit, siteProbe.Ref);
@@ -290,7 +145,7 @@ public class CentralCommunicationActorTests : TestKit
[Fact]
public void NotificationStatusQuery_ForwardedToOutboxProxy_ResponseRoutesBackToSite()
{
var (actor, _, _) = CreateActorWithMockRepo();
var (actor, _) = CreateActorWithMockRepo();
var outboxProbe = CreateTestProbe();
actor.Tell(new RegisterNotificationOutbox(outboxProbe.Ref));
@@ -308,7 +163,7 @@ public class CentralCommunicationActorTests : TestKit
[Fact]
public void NotificationSubmit_NoOutboxConfigured_RepliesNonAccepted()
{
var (actor, _, _) = CreateActorWithMockRepo();
var (actor, _) = CreateActorWithMockRepo();
// No RegisterNotificationOutbox sent — the proxy is null.
var submit = CreateSubmit();
@@ -323,7 +178,7 @@ public class CentralCommunicationActorTests : TestKit
[Fact]
public void NotificationStatusQuery_NoOutboxConfigured_RepliesNotFound()
{
var (actor, _, _) = CreateActorWithMockRepo();
var (actor, _) = CreateActorWithMockRepo();
// No RegisterNotificationOutbox sent — the proxy is null.
var query = new NotificationStatusQuery("corr1", "notif1");
@@ -333,23 +188,4 @@ public class CentralCommunicationActorTests : TestKit
Assert.Equal("corr1", response.CorrelationId);
Assert.False(response.Found);
}
[Fact]
public void BothContactPoints_UsedInSingleClient()
{
var site = CreateSite("site1",
"akka.tcp://scadabridge@host1:8082",
"akka.tcp://scadabridge@host2:8082");
var (actor, _, siteProbes, mockFactory) = CreateActorWithFactory(new[] { site });
// Wait for auto-refresh
Thread.Sleep(1000);
// Verify the factory was called with 2 contact paths
mockFactory.Received(1).Create(
Arg.Any<ActorSystem>(),
Arg.Is("site1"),
Arg.Is<ImmutableHashSet<ActorPath>>(paths => paths.Count == 2));
}
}