Joseph Doherty
6a86201105
MeshGrpcCommandDialer places one NodeCommandService.Execute call on one node: a GrpcChannel cached per endpoint (the GrpcDeploymentArtifactFetcher shape), the TARGET node's cluster's bearer plus x-otopcua-cluster naming that cluster, and a per-call deadline linked to the caller's token. A cluster with no resolvable key is refused locally with Unauthenticated — there is no arm on which a command leaves central unauthenticated, even one the far side would have rejected. CentralCommunicationActor gains the third mode branch. The contact load now also projects GrpcPort and, from the SAME rows in the SAME pass, builds the dispatch set — which is what makes 'the gRPC fan reaches exactly the SendToAll recipient set' structural rather than two queries that have to keep agreeing. A row with a null GrpcPort is skipped warn-once (the TelemetryNodeSource precedent). Per-node results are piped back for LOGGING ONLY: named node, gRPC status, dropped. No retry, no queue. Every dial is scheduled on the thread pool rather than called from the receive. Found by review and pinned red-first: an async method runs synchronously to its first await, and the dialer's pre-await work is TryGetPsk, which blocks on the secret store — bounded but seconds, and repeated after every refresh since the refresh drops the cache. Dialing inline parked the actor's mailbox, so deployment acks queued behind a slow node; the regression test tells an ApplyAck while a dial is held and fails without the hop. The contact refresh also wires the Phase 1A register: it publishes the loaded cluster ids as the PSK provider's known-cluster set and drops the key cache, so a rotated key takes effect within one ContactRefreshSeconds. Neither happens under Dps or ClusterClient. Both existing mode branches are behaviourally untouched and every existing fact passes unmodified. The one edit to shared text: the Status.Failure warning drops the word 'ClusterClient', since that handler now serves all three transports. ControlPlane.Tests 205 -> 230, Host.Tests 92, Cluster.Tests 184, 0 warnings.
871 lines
37 KiB
C#
871 lines
37 KiB
C#
using System.Collections.Immutable;
|
|
using System.Diagnostics.CodeAnalysis;
|
|
using Akka.Actor;
|
|
using Akka.Cluster.Tools.Client;
|
|
using Akka.Cluster.Tools.PublishSubscribe;
|
|
using Akka.TestKit;
|
|
using Microsoft.EntityFrameworkCore;
|
|
using Shouldly;
|
|
using Xunit;
|
|
using ZB.MOM.WW.OtOpcUa.Cluster;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Deploy;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Messages.Mesh;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.OpcUa;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Protos.MeshTransport.V1;
|
|
using ZB.MOM.WW.OtOpcUa.Commons.Types;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Entities;
|
|
using ZB.MOM.WW.OtOpcUa.Configuration.Enums;
|
|
using ZB.MOM.WW.OtOpcUa.ControlPlane.Communication;
|
|
using ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Harness;
|
|
|
|
namespace ZB.MOM.WW.OtOpcUa.ControlPlane.Tests.Communication;
|
|
|
|
public class CentralCommunicationActorTests : ControlPlaneActorTestBase
|
|
{
|
|
private const string DeploymentsTopic = "deployments";
|
|
|
|
/// <summary>Records every client the actor asks for, and hands back a probe in its place.</summary>
|
|
private sealed class RecordingClientFactory : IMeshClusterClientFactory
|
|
{
|
|
private readonly Func<TestProbe> _newProbe;
|
|
|
|
public RecordingClientFactory(Func<TestProbe> newProbe) => _newProbe = newProbe;
|
|
|
|
public List<ImmutableHashSet<ActorPath>> Calls { get; } = [];
|
|
|
|
public List<string> ClusterIds { get; } = [];
|
|
|
|
public List<TestProbe> Probes { get; } = [];
|
|
|
|
/// <summary>Probe created for a given cluster id, for per-cluster fan-out assertions.</summary>
|
|
public Dictionary<string, TestProbe> ProbeByCluster { get; } = [];
|
|
|
|
public IActorRef Create(ActorSystem system, string clusterId, ImmutableHashSet<ActorPath> contacts)
|
|
{
|
|
Calls.Add(contacts);
|
|
ClusterIds.Add(clusterId);
|
|
var probe = _newProbe();
|
|
Probes.Add(probe);
|
|
ProbeByCluster[clusterId] = probe;
|
|
return probe.Ref;
|
|
}
|
|
}
|
|
|
|
private static MeshTransportOptions ClusterClientMode(int refreshSeconds = 60) => new()
|
|
{
|
|
Mode = MeshTransportOptions.ModeClusterClient,
|
|
CentralContactPoints = ["akka.tcp://otopcua@central-1:4053"],
|
|
ContactRefreshSeconds = refreshSeconds,
|
|
};
|
|
|
|
private static MeshTransportOptions GrpcMode(int refreshSeconds = 60) => new()
|
|
{
|
|
Mode = MeshTransportOptions.ModeGrpc,
|
|
ContactRefreshSeconds = refreshSeconds,
|
|
DispatchTimeoutSeconds = 10,
|
|
};
|
|
|
|
private static void SeedCluster(
|
|
IDbContextFactory<OtOpcUaConfigDbContext> factory,
|
|
string clusterId,
|
|
params (string NodeId, string Host, int AkkaPort, bool Enabled, bool Maintenance)[] nodes)
|
|
{
|
|
using var db = factory.CreateDbContext();
|
|
db.ServerClusters.Add(new ServerCluster
|
|
{
|
|
ClusterId = clusterId,
|
|
Name = clusterId,
|
|
Enterprise = "ZB",
|
|
Site = "Site",
|
|
// Warm/Hot require NodeCount = 2 (CK_ServerCluster_RedundancyMode_NodeCount); these
|
|
// fixtures only need the row to exist, so use the standalone shape.
|
|
RedundancyMode = RedundancyMode.None,
|
|
NodeCount = 1,
|
|
CreatedBy = "test",
|
|
});
|
|
foreach (var n in nodes)
|
|
{
|
|
db.ClusterNodes.Add(new ClusterNode
|
|
{
|
|
NodeId = n.NodeId,
|
|
ClusterId = clusterId,
|
|
Host = n.Host,
|
|
AkkaPort = n.AkkaPort,
|
|
ApplicationUri = $"urn:{n.NodeId}",
|
|
Enabled = n.Enabled,
|
|
MaintenanceMode = n.Maintenance,
|
|
CreatedBy = "test",
|
|
});
|
|
}
|
|
|
|
db.SaveChanges();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Seeds a cluster whose nodes carry a <c>GrpcPort</c> — the gRPC command plane's shape.
|
|
/// Kept separate from <see cref="SeedCluster"/> so the ClusterClient facts, which have no
|
|
/// opinion about the port, stay exactly as they were.
|
|
/// </summary>
|
|
private static void SeedGrpcCluster(
|
|
IDbContextFactory<OtOpcUaConfigDbContext> factory,
|
|
string clusterId,
|
|
params (string NodeId, string Host, int? GrpcPort, bool Enabled, bool Maintenance)[] nodes)
|
|
{
|
|
using var db = factory.CreateDbContext();
|
|
db.ServerClusters.Add(new ServerCluster
|
|
{
|
|
ClusterId = clusterId,
|
|
Name = clusterId,
|
|
Enterprise = "ZB",
|
|
Site = "Site",
|
|
RedundancyMode = RedundancyMode.None,
|
|
NodeCount = 1,
|
|
CreatedBy = "test",
|
|
});
|
|
foreach (var n in nodes)
|
|
{
|
|
db.ClusterNodes.Add(new ClusterNode
|
|
{
|
|
NodeId = n.NodeId,
|
|
ClusterId = clusterId,
|
|
Host = n.Host,
|
|
AkkaPort = 4053,
|
|
GrpcPort = n.GrpcPort,
|
|
ApplicationUri = $"urn:{n.NodeId}",
|
|
Enabled = n.Enabled,
|
|
MaintenanceMode = n.Maintenance,
|
|
CreatedBy = "test",
|
|
});
|
|
}
|
|
|
|
db.SaveChanges();
|
|
}
|
|
|
|
private IActorRef Spawn(
|
|
IDbContextFactory<OtOpcUaConfigDbContext> factory,
|
|
MeshTransportOptions options,
|
|
IMeshClusterClientFactory clientFactory,
|
|
IActorRef? coordinator = null) =>
|
|
Sys.ActorOf(CentralCommunicationActor.Props(factory, options, clientFactory, () => coordinator));
|
|
|
|
[Fact]
|
|
public void Dps_mode_publishes_on_the_carried_topic_and_creates_no_client()
|
|
{
|
|
var factory = NewInMemoryDbFactory();
|
|
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
|
var mediatorProbe = CreateTestProbe();
|
|
// Subscribe on the probe's behalf: SubscribeAck goes to the SENDER, so telling the mediator
|
|
// without an explicit sender would ack to TestActor and this wait would hang.
|
|
DistributedPubSub.Get(Sys).Mediator.Tell(
|
|
new Subscribe(DeploymentsTopic, mediatorProbe.Ref), mediatorProbe.Ref);
|
|
mediatorProbe.ExpectMsg<SubscribeAck>(TimeSpan.FromSeconds(5));
|
|
|
|
var actor = Spawn(factory, new MeshTransportOptions(), clients);
|
|
var dispatch = new DispatchDeployment(
|
|
new DeploymentId(Guid.NewGuid()), new RevisionHash("abc"), new CorrelationId(Guid.NewGuid()));
|
|
|
|
actor.Tell(new MeshCommand(DeploymentsTopic, dispatch));
|
|
|
|
mediatorProbe.ExpectMsg<DispatchDeployment>(TimeSpan.FromSeconds(5)).ShouldBe(dispatch);
|
|
clients.Calls.ShouldBeEmpty();
|
|
}
|
|
|
|
[Fact]
|
|
public void ClusterClient_mode_sends_to_all_not_to_one()
|
|
{
|
|
// THE assertion of this phase. ClusterClient.Send delivers to exactly ONE registered actor;
|
|
// today's DPS publish reaches EVERY DriverHostActor, and the node side has no ClusterId or
|
|
// node filter to compensate. Send would deploy to a single node while every other node
|
|
// silently kept its old configuration.
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedCluster(factory, "C1", ("host-a:4053", "host-a", 4053, true, false));
|
|
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
|
|
|
var actor = Spawn(factory, ClusterClientMode(), clients);
|
|
AwaitCondition(() => clients.Probes.Count == 1, TimeSpan.FromSeconds(5));
|
|
|
|
var dispatch = new DispatchDeployment(
|
|
new DeploymentId(Guid.NewGuid()), new RevisionHash("abc"), new CorrelationId(Guid.NewGuid()));
|
|
actor.Tell(new MeshCommand(DeploymentsTopic, dispatch));
|
|
|
|
var sent = clients.Probes[0].ExpectMsg<ClusterClient.SendToAll>(TimeSpan.FromSeconds(5));
|
|
sent.Path.ShouldBe(MeshPaths.NodeCommunication);
|
|
sent.Message.ShouldBe(dispatch);
|
|
}
|
|
|
|
[Fact]
|
|
public void One_client_per_cluster_is_created_for_a_multi_cluster_fleet()
|
|
{
|
|
// THE Phase 6 inversion. Each application Cluster is now its own Akka mesh, and a ClusterClient
|
|
// reaches only the mesh whose receptionist it dialled. Central must therefore build one client
|
|
// per cluster, each dialling only that cluster's nodes.
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedCluster(factory, "C1", ("host-a:4053", "host-a", 4053, true, false));
|
|
SeedCluster(factory, "C2", ("host-b:4053", "host-b", 4053, true, false));
|
|
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
|
|
|
Spawn(factory, ClusterClientMode(), clients);
|
|
AwaitCondition(() => clients.Calls.Count == 2, TimeSpan.FromSeconds(5));
|
|
|
|
// Give a second refresh a chance to (wrongly) create more.
|
|
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
|
clients.Calls.Count.ShouldBe(2);
|
|
clients.ClusterIds.ShouldBe(new[] { "C1", "C2" }, ignoreOrder: true);
|
|
|
|
// Each client dials ONLY its own cluster's node.
|
|
clients.Calls[clients.ClusterIds.IndexOf("C1")].Single().ToString().ShouldContain("host-a");
|
|
clients.Calls[clients.ClusterIds.IndexOf("C2")].Single().ToString().ShouldContain("host-b");
|
|
}
|
|
|
|
[Fact]
|
|
public void A_command_fans_out_a_send_to_all_to_every_cluster_client()
|
|
{
|
|
// The other half of the inversion: one MeshCommand must reach EVERY cluster's mesh, so central
|
|
// fans SendToAll across all per-cluster clients. A single Send/one-client shape would leave
|
|
// every cluster but one silently on its old configuration.
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedCluster(factory, "C1", ("host-a:4053", "host-a", 4053, true, false));
|
|
SeedCluster(factory, "C2", ("host-b:4053", "host-b", 4053, true, false));
|
|
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
|
|
|
var actor = Spawn(factory, ClusterClientMode(), clients);
|
|
AwaitCondition(() => clients.Probes.Count == 2, TimeSpan.FromSeconds(5));
|
|
|
|
var dispatch = new DispatchDeployment(
|
|
new DeploymentId(Guid.NewGuid()), new RevisionHash("abc"), new CorrelationId(Guid.NewGuid()));
|
|
actor.Tell(new MeshCommand(DeploymentsTopic, dispatch));
|
|
|
|
foreach (var (clusterId, probe) in clients.ProbeByCluster)
|
|
{
|
|
var sent = probe.ExpectMsg<ClusterClient.SendToAll>(
|
|
TimeSpan.FromSeconds(5), $"cluster {clusterId} must receive the fanned SendToAll");
|
|
sent.Path.ShouldBe(MeshPaths.NodeCommunication);
|
|
sent.Message.ShouldBe(dispatch);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void Disabled_and_maintenance_nodes_are_not_dialled()
|
|
{
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedCluster(
|
|
factory,
|
|
"C1",
|
|
("host-a:4053", "host-a", 4053, true, false),
|
|
("host-b:4053", "host-b", 4053, false, false),
|
|
("host-c:4053", "host-c", 4053, true, true));
|
|
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
|
|
|
Spawn(factory, ClusterClientMode(), clients);
|
|
AwaitCondition(() => clients.Calls.Count == 1, TimeSpan.FromSeconds(5));
|
|
|
|
clients.Calls[0].Count.ShouldBe(1);
|
|
clients.Calls[0].Single().ToString().ShouldContain("host-a");
|
|
}
|
|
|
|
[Fact]
|
|
public void A_malformed_node_row_does_not_abort_the_refresh()
|
|
{
|
|
// The bad row is ordered FIRST on purpose: the failure this guards against is an exception
|
|
// that aborts the loop and leaves the contact set half-built.
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedCluster(
|
|
factory,
|
|
"C1",
|
|
("bad", "not a host!!", 4053, true, false),
|
|
("host-a:4053", "host-a", 4053, true, false));
|
|
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
|
|
|
Spawn(factory, ClusterClientMode(), clients);
|
|
AwaitCondition(() => clients.Calls.Count == 1, TimeSpan.FromSeconds(5));
|
|
|
|
clients.Calls[0].ShouldContain(p => p.ToString().Contains("host-a"));
|
|
}
|
|
|
|
[Fact]
|
|
public void A_command_with_no_client_is_dropped_with_a_warning()
|
|
{
|
|
// No ClusterNode rows at all: nothing to dial, so the command must be dropped loudly rather
|
|
// than queued. buffer-size = 0 covers the case where a client exists but cannot connect;
|
|
// this covers the case where there is no client at all.
|
|
var factory = NewInMemoryDbFactory();
|
|
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
|
var actor = Spawn(factory, ClusterClientMode(), clients);
|
|
|
|
EventFilter.Warning(contains: "dropping").ExpectOne(TimeSpan.FromSeconds(5), () =>
|
|
actor.Tell(new MeshCommand(DeploymentsTopic, "anything")));
|
|
}
|
|
|
|
[Fact]
|
|
public void ApplyAck_is_forwarded_to_the_coordinator_preserving_the_sender()
|
|
{
|
|
var factory = NewInMemoryDbFactory();
|
|
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
|
var coordinator = CreateTestProbe();
|
|
var node = CreateTestProbe();
|
|
var actor = Spawn(factory, ClusterClientMode(), clients, coordinator.Ref);
|
|
|
|
var ack = new ApplyAck(
|
|
new DeploymentId(Guid.NewGuid()),
|
|
new NodeId("host-a:4053"),
|
|
ApplyAckOutcome.Applied,
|
|
null,
|
|
new CorrelationId(Guid.NewGuid()));
|
|
|
|
actor.Tell(ack, node.Ref);
|
|
|
|
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).ShouldBe(ack);
|
|
// Forward, not Tell: the coordinator must see the node, not this relay.
|
|
coordinator.LastSender.ShouldBe(node.Ref);
|
|
}
|
|
|
|
[Fact]
|
|
public void An_unresolvable_coordinator_logs_rather_than_losing_the_ack_silently()
|
|
{
|
|
var factory = NewInMemoryDbFactory();
|
|
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
|
var actor = Spawn(factory, ClusterClientMode(), clients, coordinator: null);
|
|
|
|
var ack = new ApplyAck(
|
|
new DeploymentId(Guid.NewGuid()),
|
|
new NodeId("host-a:4053"),
|
|
ApplyAckOutcome.Applied,
|
|
null,
|
|
new CorrelationId(Guid.NewGuid()));
|
|
|
|
EventFilter.Warning(contains: "not resolvable").ExpectOne(TimeSpan.FromSeconds(5), () =>
|
|
actor.Tell(ack));
|
|
}
|
|
|
|
[Fact]
|
|
public void A_faulted_contact_load_warns_and_leaves_the_actor_alive()
|
|
{
|
|
var factory = NewInMemoryDbFactory();
|
|
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
|
var actor = Spawn(factory, ClusterClientMode(), clients);
|
|
|
|
// Filter on a phrase unique to the DB-failure warning. EventFilter matches case-INSENSITIVELY,
|
|
// so a looser "was NOT" also caught this actor's other warning ("the mesh ClusterClient was
|
|
// not created") and the assertion failed with two matches instead of one.
|
|
EventFilter.Warning(contains: "contact set was NOT refreshed").ExpectOne(
|
|
TimeSpan.FromSeconds(5),
|
|
() => actor.Tell(new Status.Failure(new InvalidOperationException("db down"))));
|
|
|
|
// Still serving: a transient DB outage must not take the command boundary down with it.
|
|
actor.Tell(new MeshCommand(DeploymentsTopic, "anything"));
|
|
ExpectNoMsg(TimeSpan.FromMilliseconds(200));
|
|
}
|
|
|
|
// ------------------------------------------------------------------ Mode = Grpc
|
|
//
|
|
// The third transport. The fan-out must reach EXACTLY the recipient set SendToAll reaches today
|
|
// — every enabled, non-maintenance node in every cluster — because the node side still has no
|
|
// ClusterId or node filter; scoping happens later, inside the artifact. A narrower fan would
|
|
// leave nodes silently on their old configuration while the deployment sealed green.
|
|
|
|
[Fact]
|
|
public void Grpc_mode_fans_one_command_to_every_enabled_node_across_all_clusters()
|
|
{
|
|
// THE assertion of this task, and the direct counterpart of
|
|
// A_command_fans_out_a_send_to_all_to_every_cluster_client.
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedGrpcCluster(
|
|
factory,
|
|
"C1",
|
|
("host-a:4053", "host-a", 4056, true, false),
|
|
("host-b:4053", "host-b", 4056, true, false));
|
|
SeedGrpcCluster(factory, "C2", ("host-c:4053", "host-c", 4056, true, false));
|
|
var rig = SpawnGrpcRig(factory);
|
|
AwaitContactsLoaded(rig);
|
|
|
|
rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()));
|
|
|
|
AwaitCondition(() => rig.Dialer.Calls.Count == 3, TimeSpan.FromSeconds(5));
|
|
rig.Dialer.Calls.Select(c => c.Endpoint).ShouldBe(
|
|
new[] { "http://host-a:4056", "http://host-b:4056", "http://host-c:4056" }, ignoreOrder: true);
|
|
|
|
// Each node is dialled with ITS OWN cluster's id — that is what selects the pre-shared key,
|
|
// so getting it wrong turns every command into a PermissionDenied at the node.
|
|
rig.Dialer.Calls.Single(c => c.Endpoint.Contains("host-c")).ClusterId.ShouldBe("C2");
|
|
rig.Dialer.Calls.Where(c => !c.Endpoint.Contains("host-c")).ShouldAllBe(c => c.ClusterId == "C1");
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_mode_creates_no_ClusterClient_and_publishes_nothing_on_pubsub()
|
|
{
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false));
|
|
var clients = new RecordingClientFactory(() => CreateTestProbe());
|
|
var mediatorProbe = CreateTestProbe();
|
|
DistributedPubSub.Get(Sys).Mediator.Tell(
|
|
new Subscribe(DeploymentsTopic, mediatorProbe.Ref), mediatorProbe.Ref);
|
|
mediatorProbe.ExpectMsg<SubscribeAck>(TimeSpan.FromSeconds(5));
|
|
|
|
var rig = SpawnGrpcRig(factory, clientFactory: clients);
|
|
AwaitContactsLoaded(rig);
|
|
|
|
rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()));
|
|
|
|
AwaitCondition(() => rig.Dialer.Calls.Count == 1, TimeSpan.FromSeconds(5));
|
|
clients.Calls.ShouldBeEmpty();
|
|
mediatorProbe.ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_mode_does_not_dial_disabled_or_maintenance_nodes()
|
|
{
|
|
// Same recipient filter as the ClusterClient contact load, asserted independently so the two
|
|
// transports cannot drift on who counts as a node.
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedGrpcCluster(
|
|
factory,
|
|
"C1",
|
|
("host-a:4053", "host-a", 4056, true, false),
|
|
("host-b:4053", "host-b", 4056, false, false),
|
|
("host-c:4053", "host-c", 4056, true, true));
|
|
var rig = SpawnGrpcRig(factory);
|
|
AwaitContactsLoaded(rig);
|
|
|
|
rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()));
|
|
|
|
AwaitCondition(() => rig.Dialer.Calls.Count == 1, TimeSpan.FromSeconds(5));
|
|
rig.Dialer.Calls[0].Endpoint.ShouldContain("host-a");
|
|
|
|
// And no late arrival from the two excluded rows.
|
|
ExpectNoMsg(TimeSpan.FromMilliseconds(300));
|
|
rig.Dialer.Calls.Count.ShouldBe(1);
|
|
}
|
|
|
|
[Fact]
|
|
public void A_node_with_no_GrpcPort_is_skipped_with_a_warning_and_the_others_are_still_dialled()
|
|
{
|
|
// Mirrors TelemetryNodeSource: a row with no port exposes no endpoint yet. It must cost that
|
|
// node its commands, never the whole fan.
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedGrpcCluster(
|
|
factory,
|
|
"C1",
|
|
("host-a:4053", "host-a", null, true, false),
|
|
("host-b:4053", "host-b", 4056, true, false));
|
|
|
|
GrpcRig? rig = null;
|
|
EventFilter.Warning(contains: "has no GrpcPort").ExpectOne(
|
|
TimeSpan.FromSeconds(5), () => rig = SpawnGrpcRig(factory));
|
|
|
|
rig.ShouldNotBeNull();
|
|
AwaitContactsLoaded(rig);
|
|
rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()));
|
|
|
|
AwaitCondition(() => rig.Dialer.Calls.Count == 1, TimeSpan.FromSeconds(5));
|
|
rig.Dialer.Calls[0].Endpoint.ShouldBe("http://host-b:4056");
|
|
}
|
|
|
|
[Fact]
|
|
public void One_failing_node_does_not_stop_the_fan_and_is_logged_by_node_id()
|
|
{
|
|
// Fire-and-forget: a failure is logged and DROPPED. No retry, no queue, and above all no
|
|
// abort of the remaining nodes — the deploy path's apply deadline is the backstop.
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedGrpcCluster(
|
|
factory,
|
|
"C1",
|
|
("host-a:4053", "host-a", 4056, true, false),
|
|
("host-b:4053", "host-b", 4056, true, false));
|
|
var dialer = new RecordingDialer
|
|
{
|
|
FailEndpointsContaining = "host-a",
|
|
Fault = new Grpc.Core.RpcException(
|
|
new Grpc.Core.Status(Grpc.Core.StatusCode.Unavailable, "node down")),
|
|
};
|
|
var rig = SpawnGrpcRig(factory, dialer);
|
|
AwaitContactsLoaded(rig);
|
|
|
|
// The node id, not the endpoint: an operator reading the log is looking at ClusterNode rows.
|
|
EventFilter.Warning(contains: "host-a:4053").ExpectOne(TimeSpan.FromSeconds(5), () =>
|
|
rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())));
|
|
|
|
// Both were attempted; the healthy one is unaffected by its neighbour's failure.
|
|
AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5));
|
|
|
|
// Still serving after a per-node failure — no supervision restart, no state loss.
|
|
rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()));
|
|
AwaitCondition(() => rig.Dialer.Calls.Count == 4, TimeSpan.FromSeconds(5));
|
|
}
|
|
|
|
[Fact]
|
|
public void A_psk_miss_costs_only_that_node_and_never_becomes_a_fan_level_abort()
|
|
{
|
|
// The dialer refuses a keyless cluster locally, so it arrives here as an ordinary per-node
|
|
// failure. The fan-level assertion is that the other cluster is still dialled.
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false));
|
|
SeedGrpcCluster(factory, "C2", ("host-b:4053", "host-b", 4056, true, false));
|
|
var dialer = new RecordingDialer
|
|
{
|
|
FailClusters = ["C2"],
|
|
Fault = new Grpc.Core.RpcException(
|
|
new Grpc.Core.Status(Grpc.Core.StatusCode.Unauthenticated, "no key")),
|
|
};
|
|
var rig = SpawnGrpcRig(factory, dialer);
|
|
AwaitContactsLoaded(rig);
|
|
|
|
EventFilter.Warning(contains: "Unauthenticated").ExpectOne(TimeSpan.FromSeconds(5), () =>
|
|
rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())));
|
|
|
|
AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5));
|
|
}
|
|
|
|
[Fact]
|
|
public void A_command_with_no_grpc_target_is_dropped_with_a_warning()
|
|
{
|
|
var factory = NewInMemoryDbFactory();
|
|
var rig = SpawnGrpcRig(factory);
|
|
AwaitContactsLoaded(rig);
|
|
|
|
EventFilter.Warning(contains: "dropping").ExpectOne(TimeSpan.FromSeconds(5), () =>
|
|
rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())));
|
|
|
|
rig.Dialer.Calls.ShouldBeEmpty();
|
|
}
|
|
|
|
[Fact]
|
|
public void A_payload_that_is_not_a_mesh_command_record_is_dropped_with_an_error()
|
|
{
|
|
// The mapper refuses to encode an unknown payload rather than shipping something the node
|
|
// cannot decode. That refusal must cost the command, not the actor.
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false));
|
|
var rig = SpawnGrpcRig(factory);
|
|
AwaitContactsLoaded(rig);
|
|
|
|
EventFilter.Error(contains: "Cannot encode").ExpectOne(TimeSpan.FromSeconds(5), () =>
|
|
rig.Actor.Tell(new MeshCommand(DeploymentsTopic, "not a command record")));
|
|
|
|
rig.Dialer.Calls.ShouldBeEmpty();
|
|
|
|
// Still serving.
|
|
rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()));
|
|
AwaitCondition(() => rig.Dialer.Calls.Count == 1, TimeSpan.FromSeconds(5));
|
|
}
|
|
|
|
[Fact]
|
|
public void The_envelope_carries_the_commands_own_correlation_and_the_whole_fan_shares_it()
|
|
{
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedGrpcCluster(
|
|
factory,
|
|
"C1",
|
|
("host-a:4053", "host-a", 4056, true, false),
|
|
("host-b:4053", "host-b", 4056, true, false));
|
|
var rig = SpawnGrpcRig(factory);
|
|
AwaitContactsLoaded(rig);
|
|
|
|
var correlationId = new CorrelationId(Guid.NewGuid());
|
|
rig.Actor.Tell(new MeshCommand(DeploymentsTopic, new DispatchDeployment(
|
|
new DeploymentId(Guid.NewGuid()), new RevisionHash("abc"), correlationId)));
|
|
|
|
AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5));
|
|
rig.Dialer.Calls.ShouldAllBe(c => c.Envelope.CorrelationId == correlationId.ToString());
|
|
// One envelope object, encoded once and fanned — not re-encoded per node.
|
|
rig.Dialer.Calls.Select(c => c.Envelope).Distinct().Count().ShouldBe(1);
|
|
}
|
|
|
|
[Fact]
|
|
public void An_alarm_command_gets_one_minted_transport_correlation_shared_by_the_whole_fan()
|
|
{
|
|
// AlarmCommand is the one record with no correlation member of its own, so the envelope's is
|
|
// the only handle an operator has on it. Minting per fan rather than per node is what makes
|
|
// "this alarm command" one traceable event instead of N unrelated ones.
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedGrpcCluster(
|
|
factory,
|
|
"C1",
|
|
("host-a:4053", "host-a", 4056, true, false),
|
|
("host-b:4053", "host-b", 4056, true, false));
|
|
var rig = SpawnGrpcRig(factory);
|
|
AwaitContactsLoaded(rig);
|
|
|
|
rig.Actor.Tell(new MeshCommand(
|
|
"alarms", new AlarmCommand("alarm-1", "Acknowledge", "operator", null, null)));
|
|
|
|
AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5));
|
|
rig.Dialer.Calls.Select(c => c.Envelope.CorrelationId).Distinct().Count().ShouldBe(1);
|
|
rig.Dialer.Calls[0].Envelope.CorrelationId.ShouldNotBeNullOrWhiteSpace();
|
|
rig.Dialer.Calls[0].Envelope.AlarmCommand.AlarmId.ShouldBe("alarm-1");
|
|
}
|
|
|
|
[Fact]
|
|
public void The_contact_refresh_publishes_the_known_cluster_set_and_drops_cached_psks()
|
|
{
|
|
// Phase 1A security register item 1, wired: the gate is only a control if something keeps it
|
|
// current, and the refresh is the moment central's picture of the fleet is up to date. The
|
|
// InvalidateAll on the same tick is what makes a rotated key take effect within one refresh
|
|
// instead of requiring a central restart.
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false));
|
|
SeedGrpcCluster(factory, "C2", ("host-b:4053", "host-b", null, true, false));
|
|
var rig = SpawnGrpcRig(factory);
|
|
|
|
AwaitContactsLoaded(rig);
|
|
|
|
// C2 is included even though its only node has no GrpcPort: the gate is about which clusters
|
|
// central KNOWS, not which it can currently dial. Excluding it would deny that cluster's acks.
|
|
rig.Psk.KnownClusterPublications[0].ShouldBe(new[] { "C1", "C2" }, ignoreOrder: true);
|
|
rig.Psk.InvalidateAllCount.ShouldBeGreaterThanOrEqualTo(1);
|
|
}
|
|
|
|
[Fact]
|
|
public void A_slow_dial_does_not_park_the_actors_mailbox()
|
|
{
|
|
// The dialer's first act is a PSK lookup, which blocks on the secret store — bounded, but
|
|
// seconds, and repeated after every refresh because the refresh drops the cache. An async
|
|
// method runs synchronously to its first await, so dialing straight from the receive would
|
|
// stall this actor for that long PER NODE, and deployment acks queue behind that mailbox.
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false));
|
|
var coordinator = CreateTestProbe();
|
|
var dialer = new RecordingDialer { BlockBeforeFirstAwait = true };
|
|
var rig = SpawnGrpcRig(factory, dialer, coordinator: coordinator.Ref);
|
|
AwaitContactsLoaded(rig);
|
|
|
|
try
|
|
{
|
|
rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()));
|
|
|
|
// The ack must be handled while the dial is still parked.
|
|
var ack = new ApplyAck(
|
|
new DeploymentId(Guid.NewGuid()),
|
|
new NodeId("host-a:4053"),
|
|
ApplyAckOutcome.Applied,
|
|
null,
|
|
new CorrelationId(Guid.NewGuid()));
|
|
rig.Actor.Tell(ack);
|
|
|
|
coordinator.ExpectMsg<ApplyAck>(TimeSpan.FromSeconds(5)).ShouldBe(ack);
|
|
}
|
|
finally
|
|
{
|
|
dialer.Release();
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void A_node_added_after_start_joins_the_fan_on_the_next_refresh()
|
|
{
|
|
// The refresh is not decoration: operators add and retire nodes, and central must pick them
|
|
// up without a restart. Under Grpc that is the ONLY thing keeping the dispatch set current —
|
|
// there is no receptionist to notice a new registration on its own.
|
|
//
|
|
// Driven by telling the actor its own RefreshContacts tick rather than by shortening
|
|
// ContactRefreshSeconds: a one-second timer would run for the whole fact, adding DB load to
|
|
// every test sharing the runner, and would make this assertion depend on wall-clock timing.
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false));
|
|
var rig = SpawnGrpcRig(factory);
|
|
AwaitContactsLoaded(rig);
|
|
|
|
using (var db = factory.CreateDbContext())
|
|
{
|
|
db.ClusterNodes.Add(new ClusterNode
|
|
{
|
|
NodeId = "host-b:4053",
|
|
ClusterId = "C1",
|
|
Host = "host-b",
|
|
AkkaPort = 4053,
|
|
GrpcPort = 4056,
|
|
ApplicationUri = "urn:host-b",
|
|
Enabled = true,
|
|
CreatedBy = "test",
|
|
});
|
|
db.SaveChanges();
|
|
}
|
|
|
|
rig.Actor.Tell(new CentralCommunicationActor.RefreshContacts());
|
|
AwaitCondition(
|
|
() => rig.Psk.KnownClusterPublications.Count >= 2, TimeSpan.FromSeconds(5));
|
|
|
|
rig.Actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch()));
|
|
|
|
AwaitCondition(() => rig.Dialer.Calls.Count == 2, TimeSpan.FromSeconds(5));
|
|
rig.Dialer.Calls.Select(c => c.Endpoint).ShouldBe(
|
|
new[] { "http://host-a:4056", "http://host-b:4056" }, ignoreOrder: true);
|
|
}
|
|
|
|
[Fact]
|
|
public void The_known_cluster_set_is_not_published_by_the_other_two_modes()
|
|
{
|
|
// The gate is a gRPC-command-plane control. Publishing it under Dps or ClusterClient would
|
|
// start refusing acks on a transport whose ack path never consults the provider at all —
|
|
// and, worse, would do so from a code path those modes' tests do not cover.
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedCluster(factory, "C1", ("host-a:4053", "host-a", 4053, true, false));
|
|
var psk = new RecordingPskProvider();
|
|
|
|
Sys.ActorOf(CentralCommunicationActor.Props(
|
|
factory,
|
|
ClusterClientMode(),
|
|
new RecordingClientFactory(() => CreateTestProbe()),
|
|
() => null,
|
|
dialer: null,
|
|
pskProvider: psk));
|
|
|
|
ExpectNoMsg(TimeSpan.FromMilliseconds(500));
|
|
psk.KnownClusterPublications.ShouldBeEmpty();
|
|
psk.InvalidateAllCount.ShouldBe(0);
|
|
}
|
|
|
|
[Fact]
|
|
public void Grpc_mode_with_no_dialer_wired_drops_the_command_with_an_error()
|
|
{
|
|
// Mis-wiring, not misconfiguration: Mode=Grpc validates fine but the DI graph did not supply
|
|
// a dialer. Silence here would be a fleet-wide command outage with no symptom until a deploy
|
|
// stalled at its apply deadline two minutes later.
|
|
var factory = NewInMemoryDbFactory();
|
|
SeedGrpcCluster(factory, "C1", ("host-a:4053", "host-a", 4056, true, false));
|
|
var actor = Sys.ActorOf(CentralCommunicationActor.Props(
|
|
factory,
|
|
GrpcMode(),
|
|
new RecordingClientFactory(() => CreateTestProbe()),
|
|
() => null,
|
|
dialer: null,
|
|
pskProvider: null));
|
|
|
|
// Filtered on the DISPATCH message's exact phrase: PreStart logs its own error about the same
|
|
// mis-wiring, and a looser filter races it into a second match.
|
|
EventFilter.Error(contains: "no command dialer is wired").ExpectOne(
|
|
TimeSpan.FromSeconds(5),
|
|
() => actor.Tell(new MeshCommand(DeploymentsTopic, NewDispatch())));
|
|
}
|
|
|
|
private static DispatchDeployment NewDispatch() => new(
|
|
new DeploymentId(Guid.NewGuid()), new RevisionHash("abc"), new CorrelationId(Guid.NewGuid()));
|
|
|
|
/// <summary>A spawned gRPC-mode actor plus the two seams the tests observe it through.</summary>
|
|
private sealed record GrpcRig(IActorRef Actor, RecordingDialer Dialer, RecordingPskProvider Psk);
|
|
|
|
private GrpcRig SpawnGrpcRig(
|
|
IDbContextFactory<OtOpcUaConfigDbContext> factory,
|
|
RecordingDialer? dialer = null,
|
|
IMeshClusterClientFactory? clientFactory = null,
|
|
IActorRef? coordinator = null)
|
|
{
|
|
dialer ??= new RecordingDialer();
|
|
var psk = new RecordingPskProvider();
|
|
var actor = Sys.ActorOf(CentralCommunicationActor.Props(
|
|
factory,
|
|
GrpcMode(),
|
|
clientFactory ?? new RecordingClientFactory(() => CreateTestProbe()),
|
|
() => coordinator,
|
|
dialer,
|
|
psk));
|
|
return new GrpcRig(actor, dialer, psk);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Waits until the actor has applied a contact load. The known-cluster publication is the
|
|
/// signal because it happens in the same handler, immediately after the target set is
|
|
/// adopted — and unlike the target set itself it is observable from outside the actor. A
|
|
/// command told before this point would be dropped as "no targets" and never retried, which
|
|
/// is correct behaviour and a flaky test.
|
|
/// </summary>
|
|
private void AwaitContactsLoaded(GrpcRig rig) =>
|
|
AwaitCondition(
|
|
() => rig.Psk.KnownClusterPublications.Count >= 1, TimeSpan.FromSeconds(5));
|
|
|
|
/// <summary>Records every dial and answers however the test asks; never opens a socket.</summary>
|
|
private sealed class RecordingDialer : IMeshCommandDialer
|
|
{
|
|
private readonly object _gate = new();
|
|
private readonly List<Dial> _calls = [];
|
|
private readonly ManualResetEventSlim _released = new(false);
|
|
|
|
/// <summary>
|
|
/// When true, every dial blocks its calling thread synchronously until
|
|
/// <see cref="Release"/>. Models the dialer's pre-await PSK lookup.
|
|
/// </summary>
|
|
public bool BlockBeforeFirstAwait { get; set; }
|
|
|
|
/// <summary>Unblocks every dial parked by <see cref="BlockBeforeFirstAwait"/>.</summary>
|
|
public void Release() => _released.Set();
|
|
|
|
/// <summary>Every dial made. Copied under the lock — the fan runs on thread-pool threads.</summary>
|
|
public IReadOnlyList<Dial> Calls
|
|
{
|
|
get { lock (_gate) return _calls.ToList(); }
|
|
}
|
|
|
|
/// <summary>Endpoints whose substring match makes the dial fault.</summary>
|
|
public string? FailEndpointsContaining { get; set; }
|
|
|
|
/// <summary>Cluster ids whose dials fault.</summary>
|
|
public string[] FailClusters { get; set; } = [];
|
|
|
|
/// <summary>
|
|
/// The fault raised for a matching dial. Fully qualified because this file's bare
|
|
/// <c>Status</c> is Akka's — an existing fact drives the actor's <c>Status.Failure</c>
|
|
/// handler, and importing Grpc.Core here would make that reference ambiguous.
|
|
/// </summary>
|
|
public Grpc.Core.RpcException? Fault { get; set; }
|
|
|
|
public Task DispatchAsync(
|
|
string nodeEndpoint, CommandEnvelope envelope, string clusterId, CancellationToken ct)
|
|
{
|
|
lock (_gate) _calls.Add(new Dial(nodeEndpoint, envelope, clusterId));
|
|
|
|
if (BlockBeforeFirstAwait)
|
|
{
|
|
// Blocks the CALLING thread, exactly as IMeshPskProvider.TryGetPsk does on a cold
|
|
// cache: an async method runs synchronously to its first await, so a dial invoked
|
|
// straight from the actor would park the mailbox here.
|
|
_released.Wait(TimeSpan.FromSeconds(30));
|
|
}
|
|
|
|
var shouldFail =
|
|
(FailEndpointsContaining is not null && nodeEndpoint.Contains(FailEndpointsContaining))
|
|
|| FailClusters.Contains(clusterId, StringComparer.OrdinalIgnoreCase);
|
|
|
|
return shouldFail && Fault is not null
|
|
? Task.FromException(Fault)
|
|
: Task.CompletedTask;
|
|
}
|
|
|
|
/// <summary>One recorded dial.</summary>
|
|
public sealed record Dial(string Endpoint, CommandEnvelope Envelope, string ClusterId);
|
|
}
|
|
|
|
/// <summary>Records the gate and invalidation calls the contact refresh is supposed to make.</summary>
|
|
private sealed class RecordingPskProvider : IMeshPskProvider
|
|
{
|
|
private readonly object _gate = new();
|
|
private readonly List<IReadOnlyList<string>> _publications = [];
|
|
private int _invalidateAllCount;
|
|
|
|
/// <summary>Every known-cluster set published, in order.</summary>
|
|
public IReadOnlyList<IReadOnlyList<string>> KnownClusterPublications
|
|
{
|
|
get { lock (_gate) return _publications.ToList(); }
|
|
}
|
|
|
|
/// <summary>How many times the whole cache was dropped.</summary>
|
|
public int InvalidateAllCount => Volatile.Read(ref _invalidateAllCount);
|
|
|
|
public bool TryGetPsk(string clusterId, [NotNullWhen(true)] out string? psk)
|
|
{
|
|
psk = "psk";
|
|
return true;
|
|
}
|
|
|
|
public bool Matches(string clusterId, string presented) => true;
|
|
|
|
public void SetKnownClusters(IReadOnlyCollection<string> clusterIds)
|
|
{
|
|
lock (_gate) _publications.Add(clusterIds.ToList());
|
|
}
|
|
|
|
public void Invalidate(string clusterId) { }
|
|
|
|
public void InvalidateAll() => Interlocked.Increment(ref _invalidateAllCount);
|
|
}
|
|
}
|